def get_data(self, *, catalog: CatalogName, entity_type: str, file_url_func: FileUrlFunc, item_id: Optional[str], filters: MutableFilters, pagination: Optional[Pagination]) -> JSON: """ Returns data for a particular entity type of single item. :param catalog: The name of the catalog to query :param entity_type: Which index to search (i.e. 'projects', 'specimens', etc.) :param pagination: A dictionary with pagination information as return from `_get_pagination()` :param filters: parsed JSON filters from the request :param item_id: If item_id is specified, only a single item is searched for :param file_url_func: A function that is used only when getting a *list* of files data. It creates the files URL based on info from the request. It should have the type signature `(uuid: str, **params) -> str` :return: The Elasticsearch JSON response """ if item_id is not None: validate_uuid(item_id) filters.explicit['entryId'] = {'is': [item_id]} response = self.transform_request(catalog=catalog, filters=filters, pagination=pagination, entity_type=entity_type) for hit in response['hits']: entity = one(hit[entity_type]) source_id = one(hit['sources'])['sourceId'] entity['accessible'] = source_id in filters.source_ids def inject_file_urls(node: AnyMutableJSON, *path: str) -> None: if node is None: pass elif isinstance(node, (str, int, float, bool)): pass elif isinstance(node, list): for child in node: inject_file_urls(child, *path) elif isinstance(node, dict): if path: try: next_node = node[path[0]] except KeyError: # Not all node trees will match the given path. (e.g. a # response from the 'files' index won't have a # 'matrices' in its 'hits[].projects' inner entities. pass else: inject_file_urls(next_node, *path[1:]) else: try: url = node['url'] version = node['version'] uuid = node['uuid'] except KeyError: for child in node.values(): inject_file_urls(child, *path) else: if url is None: node['url'] = file_url_func(catalog=catalog, fetch=False, file_uuid=uuid, version=version) else: assert False inject_file_urls(response['hits'], 'projects', 'contributedAnalyses') inject_file_urls(response['hits'], 'projects', 'matrices') inject_file_urls(response['hits'], 'files') if item_id is not None: response = one(response['hits'], too_short=EntityNotFoundError(entity_type, item_id)) return response
from more_itertools import one from .._utils import names_and_abbrevs from ..Unit import CURRENT, Dimension, LENGTH, MASS, TEMPERATURE, TIME, Unit base_unit_map = {(names, abbrevs): Unit(dim, name=one(abbrevs)) for (names, abbrevs), dim in { names_and_abbrevs(item): { val: 1 } if isinstance(val, Dimension) else val for item, val in { (('meter', 'metre'), 'm'): LENGTH, 'second': TIME, ('kilogram', 'kg'): MASS, 'Ampere': CURRENT, 'Kelvin': TEMPERATURE, }.items() }.items()}
def _get_cute_score_for_action(observation: Observation, legal_move_action: Action) -> int: next_state = observation.state. \ get_next_state_from_actions({observation.letter: legal_move_action}) return more_itertools.one(next_state.player_id_to_observation.values()).cute_score
def get_next_state_from_actions(self, player_id_to_action: Mapping[PlayerId, Action]) \ -> SinglePlayerState: return self.get_next_state_from_action(more_itertools.one(player_id_to_action.values()))
def __init__(self, pattern_ll, cell_vect, nb_cells, offset, attractors, soft_mat, name): """ Contrat : Créer un maillage pour des RVE 2D, plans, comportant au plus 2 matériaux constitutifs et pouvant contenir plusieurs cellules. cell_vect : Vecteurs de périodicité en colonne, définit également le parallélogramme qui contient la cellule. #! La cellule doit être un parallélogramme. Parameters ---------- pattern_ll : list Instances of LineLoop that define the contours of the microstructure. cell_vect : 2D array dimensions of the unit cell and directions of periodicity. (given in a 2D cartesian coordinate system) nb_cells : 1D array Numbers of cells in each direction of repetition/periodicity. offset : 1D array Relative position inside a cell of the point that will coincide with the origin of the global domain. attractors : list Instances of Point. Can also be = None or empty. It represent the points that will be used as attractors in the definition of the element characteristic length fields. Attractors are geometrical elements of the cell around which mesh refinement constraints will be set. name : string or Path """ self.name = name.stem if isinstance(name, PurePath) else name model.add(self.name) model.setCurrent(self.name) if offset.any(): nb_pattern = [ math.ceil(val + 1) if offset[i] != 0 else math.ceil(val) for i, val in enumerate(nb_cells) ] nb_pattern = np.array(nb_pattern, dtype=np.int8) pattern_ll = offset_pattern(pattern_ll, offset, cell_vect) else: nb_pattern = np.int8(np.ceil(nb_cells)) if not np.equal(nb_pattern, 1).all(): duplicate_pattern(pattern_ll, nb_pattern, cell_vect) rve_vect = cell_vect * np.asarray(nb_cells) pt_o = np.zeros((3, )) macro_vtcs = [ pt_o, rve_vect[:, 0], rve_vect[:, 0] + rve_vect[:, 1], rve_vect[:, 1], ] macro_ll = geo.LineLoop([geo.Point(c) for c in macro_vtcs]) macro_s = geo.PlaneSurface(macro_ll) if attractors: if not all(isinstance(e, geo.Point) for e in attractors): _err_msg = """Use of curves as attractors for the refinement of the mesh is not yet fully supported in our python library for gmsh.""" raise TypeError(_err_msg) # TODO : toujours d'actualité ? if offset.any(): attractors = offset_pattern(attractors, offset, cell_vect) if not np.equal(nb_pattern, 1).all(): duplicate_pattern(attractors, nb_pattern, cell_vect) logger.info("Start boolean operations on surfaces") _msg_ok = "The main material domain of the RVE is connected (topological property)." # noqa _msg_no_surface = "The boolean operation for creating the main material domain of the RVE return 0 surfaces." _msg_not_connected = "The main material domain of the RVE obtained by a boolean operation is disconnected (topological property)." phy_surf = list() pattern_s = [geo.PlaneSurface(ll) for ll in pattern_ll] rve_s = geo.surface_bool_cut(macro_s, pattern_s) if len(rve_s) == 1: logger.info(_msg_ok) elif len(rve_s) == 0: logger.warning(_msg_no_surface) else: logger.warning(_msg_not_connected) rve_s_phy = geo.PhysicalGroup(rve_s, 2, "microstruct_domain") phy_surf.append(rve_s_phy) if soft_mat: soft_s = geo.surface_bool_cut(macro_s, rve_s) soft_s_phy = geo.PhysicalGroup(soft_s, 2, "soft_domain") phy_surf.append(soft_s_phy) logger.info("Done boolean operations on surfaces") if attractors: need_sync = False for entity in attractors: if not entity.tag: entity.add_gmsh() need_sync = True if need_sync: factory.synchronize() # ? Pourrait être enlevé ? for gp in phy_surf: gp.add_gmsh() factory.synchronize() data = model.getPhysicalGroups() details = [ f"Physical group id : {dimtag[1]}, " + f"dimension : {dimtag[0]}, " + f"name : {model.getPhysicalName(*dimtag)}, " + f"nb of entitities {len(model.getEntitiesForPhysicalGroup(*dimtag))} \n" for dimtag in data ] logger.debug(f"All physical groups in the model : {data}") logger.debug(f"Physical groups details : \n {details}") logger.info("Done generating the gmsh geometrical model") if isinstance(name, PurePath): gmsh.write(str(name.with_suffix(".brep"))) logger.info(f"Saving brep at {str(name.with_suffix('.brep'))}") else: gmsh.write(f"{name}.brep") logger.info(f"Saving brep at {name}.brep") macro_bndry = macro_ll.sides if soft_mat: boundary = geo.AbstractSurface.get_surfs_boundary(rve_s + soft_s) else: try: s = one(rve_s) boundary = geo.AbstractSurface.get_surfs_boundary( s, recursive=False) except ValueError: boundary = geo.AbstractSurface.get_surfs_boundary( rve_s, recursive=False) for b in boundary: b.get_boundary(get_coords=True) # factory.synchronize() micro_bndry = list() for macro_line in macro_bndry: fragments = geo.macro_line_fragments(boundary, macro_line) for c in fragments: c.gmsh_type = model.getType(1, c.tag) lines_only = [c for c in fragments if c.gmsh_type == "Line"] micro_bndry.append(lines_only) macro_dir = list() for i in range(len(macro_bndry) // 2): macro_line = macro_bndry[i] direction = macro_line.def_pts[-1].coord - macro_line.def_pts[ 0].coord macro_dir.append(direction) for i, crvs in enumerate(micro_bndry): msh.order_curves(crvs, macro_dir[i % 2], orientation=True) msh.set_periodicity_pairs(micro_bndry[0], micro_bndry[2]) msh.set_periodicity_pairs(micro_bndry[1], micro_bndry[3]) logger.info("Done defining a mesh periodicity constraint") tags = [ f"per_pair_{k}" for k in ("1_slave", "2_slave", "1_mast", "2_mast") ] per_pair_phy = list() for crvs, tag in zip(micro_bndry, tags): per_pair_phy.append(geo.PhysicalGroup(crvs, 1, tag)) for gp in per_pair_phy: gp.add_gmsh() self.gen_vect = rve_vect self.nb_cells = nb_cells self.attractors = attractors if attractors else [] self.phy_surf = phy_surf self.mesh_fields = [] self.mesh_abs_path = ""
class State(_BaseGrid, gamey.State): Observation = Observation Action = Action is_end = False def __init__(self, culture: Culture, *, board_size: int, player_id_to_observation: ImmutableDict[str, Observation], food_positions: FrozenSet[Position], bullets: ImmutableDict[Position, FrozenSet[Bullet]] = ImmutableDict()) -> None: self.culture = culture assert len(self.culture.core_strategies) == N_CORE_STRATEGIES self.player_id_to_observation = player_id_to_observation self.bullets = bullets assert all(bullets.values()) # No empty sets in this bad boy. self.all_bullets = frozenset(itertools.chain.from_iterable(bullets.values())) self.food_positions = food_positions self.board_size = board_size self.position_to_observation = ImmutableDict( {observation.position: observation for observation in player_id_to_observation.values()} ) def _reduce(self) -> tuple: return ( type(self), frozenset( (letter, observation.position, observation.score, observation.reward, observation.last_action) for letter, observation in self.player_id_to_observation.items() ), self.bullets, self.food_positions, self.board_size, ) def __eq__(self, other: Any) -> bool: return isinstance(other, State) and self._reduce() == other._reduce() def __hash__(self) -> int: return hash(self._reduce()) @staticmethod def make_initial(culture: Culture, *, board_size: int, starting_score: int = 0, concurrent_food_tiles: int = 40) -> State: n_players = len(culture.strategies) random_positions_firehose = utils.iterate_deduplicated( State.iterate_random_positions(board_size=board_size)) random_positions = tuple( more_itertools.islice_extended( random_positions_firehose)[:(n_players + concurrent_food_tiles)] ) player_positions = random_positions[:n_players] food_positions = frozenset(random_positions[n_players:]) assert len(food_positions) == concurrent_food_tiles player_id_to_observation = {} for letter, player_position in zip(LETTERS, player_positions): player_id_to_observation[letter] = Observation(state=None, position=player_position, score=starting_score, letter=letter, last_action=None) state = State( culture=culture, board_size=board_size, player_id_to_observation=ImmutableDict(player_id_to_observation), food_positions=food_positions, ) for observation in player_id_to_observation.values(): observation.state = state return state def get_next_state_from_actions(self, player_id_to_action: Mapping[Position, Action]) -> State: new_player_position_to_olds = collections.defaultdict(set) for letter, action in player_id_to_action.items(): action: Action old_observation = self.player_id_to_observation[letter] assert action in old_observation.legal_actions old_player_position = old_observation.position if action.move is not None: new_player_position_to_olds[old_player_position + action.move].add(old_player_position) else: new_player_position_to_olds[old_player_position].add(old_player_position) ############################################################################################ ### Figuring out which players collided into each other: ################################### # # # There are three types of collisions: # 1. Two or more players that try to move into the same position. # 2. Two players that are trying to move into each other's positions. # 3. Any players that are trying to move into the old position of a player that had one # of the two collisions above, and is therefore still occupying that position. collided_player_positions = set() while True: for new_player_position, old_player_positions in new_player_position_to_olds.items(): if len(old_player_positions) >= 2: # Yeehaw, we have a collision! This is either type 1 or type 3. Let's punish # everyone! collided_player_positions |= old_player_positions del new_player_position_to_olds[new_player_position] for old_player_position in old_player_positions: new_player_position_to_olds[old_player_position].add(old_player_position) # We modified the dict while iterating, let's restart the loop: break elif (len(old_player_positions) == 1 and ((old_player_position := more_itertools.one(old_player_positions)) != new_player_position) and new_player_position_to_olds.get( old_player_position, None) == {new_player_position}): collided_player_positions |= {old_player_position, new_player_position} new_player_position_to_olds[new_player_position] = {new_player_position} new_player_position_to_olds[old_player_position] = {old_player_position} # We modified the dict while iterating, let's restart the loop: break else: # We already found all collisions, if any. break # # ### Finished figuring out which players collided into each other. ########################## ############################################################################################ new_player_position_to_old = { new_player_position: more_itertools.one(old_player_positions) for new_player_position, old_player_positions in new_player_position_to_olds.items() } del new_player_position_to_olds # Prevent confusion ############################################################################################ ### Figuring out bullets: ################################################################## # # # Todo: This section needs a lot of tests! wip_bullets = collections.defaultdict(set) # Continuing trajectory for existing bullets: for bullet in self.all_bullets: new_bullet = bullet.get_next_bullet() wip_bullets[new_bullet.position].add(new_bullet) # Processing new bullets that were just fired: for letter, action in player_id_to_action.items(): if action.shoot is not None: player_position = self.player_id_to_observation[letter].position new_bullet = Bullet(player_position + action.shoot, action.shoot) wip_bullets[new_bullet.position].add(new_bullet) # Clearing bullets out of board: for position in [position for position, bullets in wip_bullets.items() if (position not in self)]: del wip_bullets[position] # Figuring out which players were shot, removing these bullets: new_player_positions_that_were_shot = set() for new_player_position, old_player_position in new_player_position_to_old.items(): if wip_bullets.get(new_player_position, None): # Common shooting case. del wip_bullets[new_player_position] new_player_positions_that_were_shot.add(new_player_position) elif translation := new_player_position - old_player_position: # Player moved oncoming_bullet_direction = - Step(*translation) oncoming_bullets = {bullet for bullet in wip_bullets.get(old_player_position, ()) if bullet.direction == oncoming_bullet_direction} if oncoming_bullets: # Less-common shooting case: The player walked towards an oncoming bullet, # switching places with it. (oncoming_bullet,) = oncoming_bullets wip_bullets[old_player_position].remove(oncoming_bullet) new_player_positions_that_were_shot.add(new_player_position)
def get_riff_by_id(conn, id): result = conn.execute(sa.text("SELECT * FROM riffs WHERE id=:id"), id=id) return one(result.fetchall())
async def get_presents_stat(self, import_id: str) -> Dict[str, Any]: aggregation = PresentsAggregation(import_id) cursor = self.client[self.db_name][self.collection_name].aggregate( aggregation.get()) return one([presents_stat async for presents_stat in cursor])
def test_readDbEntitiesOfClsToMerge(self) -> None: person_1 = schema.StatePerson(person_id=1, state_code=_STATE_CODE) sentence_group_1 = schema.StateSentenceGroup( sentence_group_id=1, external_id=_EXTERNAL_ID, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_STATE_CODE, person=person_1, ) person_1.sentence_groups = [sentence_group_1] person_2 = schema.StatePerson(person_id=2, state_code=_STATE_CODE) sentence_group_1_dup = schema.StateSentenceGroup( sentence_group_id=2, external_id=_EXTERNAL_ID, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_STATE_CODE, person=person_2, ) sentence_group_2 = schema.StateSentenceGroup( sentence_group_id=3, external_id=_EXTERNAL_ID_2, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_STATE_CODE, person=person_2, ) placeholder_sentence_group = schema.StateSentenceGroup( sentence_group_id=4, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_STATE_CODE, person=person_2, ) person_2.sentence_groups = [ sentence_group_1_dup, sentence_group_2, placeholder_sentence_group, ] with SessionFactory.using_database(self.database_key, autocommit=False) as session: session.add(person_1) session.add(person_2) session.flush() # Act trees_to_merge = read_db_entity_trees_of_cls_to_merge( session, _STATE_CODE, schema.StateSentenceGroup) sentence_group_trees_to_merge = one(trees_to_merge) self.assertEqual(len(sentence_group_trees_to_merge), 2) sentence_group_ids = set() for entity_tree in sentence_group_trees_to_merge: self.assertIsInstance(entity_tree, EntityTree) entity = entity_tree.entity if not isinstance(entity, schema.StateSentenceGroup): self.fail(f"Expected StateSentenceGroup. Found {entity}") self.assertEqual(entity.external_id, _EXTERNAL_ID) sentence_group_ids.add(entity.sentence_group_id) self.assertEqual(sentence_group_ids, {1, 2})
def test_get_ingest_view_metadata_for_most_recent_valid_job(self) -> None: with freeze_time("2015-01-02T03:05:05"): self.metadata_manager.register_ingest_file_export_job( GcsfsIngestViewExportArgs( ingest_view_name="file_tag", upper_bound_datetime_prev=None, upper_bound_datetime_to_export=datetime.datetime( 2015, 1, 2, 2, 2, 2, 2), )) with freeze_time("2015-01-02T03:06:06"): self.metadata_manager.register_ingest_file_export_job( GcsfsIngestViewExportArgs( ingest_view_name="file_tag", upper_bound_datetime_prev=datetime.datetime( 2015, 1, 2, 2, 2, 2, 2), upper_bound_datetime_to_export=datetime.datetime( 2015, 1, 2, 3, 3, 3, 3), )) with freeze_time("2015-01-02T03:07:07"): self.metadata_manager.register_ingest_file_export_job( GcsfsIngestViewExportArgs( ingest_view_name="another_tag", upper_bound_datetime_prev=datetime.datetime( 2015, 1, 2, 3, 3, 3, 3), upper_bound_datetime_to_export=datetime.datetime( 2015, 1, 2, 3, 4, 4, 4), )) most_recent_valid_job = ( self.metadata_manager. get_ingest_view_metadata_for_most_recent_valid_job("file_tag")) self.assertIsNotNone(most_recent_valid_job) if most_recent_valid_job is None: self.fail("most_recent_valid_job is unexpectedly None") self.assertEqual("file_tag", most_recent_valid_job.file_tag) self.assertEqual( datetime.datetime(2015, 1, 2, 2, 2, 2, 2), most_recent_valid_job.datetimes_contained_lower_bound_exclusive, ) self.assertEqual( datetime.datetime(2015, 1, 2, 3, 3, 3, 3), most_recent_valid_job.datetimes_contained_upper_bound_inclusive, ) # Invalidate the row that was just returned session = SessionFactory.for_schema_base(OperationsBase) results = (session.query( schema.DirectIngestIngestFileMetadata).filter_by( file_id=most_recent_valid_job.file_id).all()) result = one(results) result.is_invalidated = True session.commit() most_recent_valid_job = ( self.metadata_manager. get_ingest_view_metadata_for_most_recent_valid_job("file_tag")) if most_recent_valid_job is None: self.fail("most_recent_valid_job is unexpectedly None") self.assertEqual("file_tag", most_recent_valid_job.file_tag) self.assertEqual( None, most_recent_valid_job.datetimes_contained_lower_bound_exclusive) self.assertEqual( datetime.datetime(2015, 1, 2, 2, 2, 2, 2), most_recent_valid_job.datetimes_contained_upper_bound_inclusive, )
def _init_objects_to_attributes( self ) -> ImmutableSetMultiDict[ObjectSemanticNode, AttributeSemanticNode]: return immutablesetmultidict( (one(attribute.slot_fillings.values()), attribute) for attribute in self.attributes)
def project_id(self): path = '{namespace}/{project}'.format(**self.options) return one(x for x in self.projects if x['path_with_namespace'] == path)['id']
def site_config(self, site_id): site_root = self._site_dir(site_id) config = list(site_root.glob('*.lektorproject')) if config: return inifile.IniFile(one(config)) return collections.defaultdict(type(None))
def simulate_timecourse_objective( parent_petab_problem: petab.Problem, timecourse_id: str, problem_parameters: Dict[str, float], parameter_mapping, sensi_orders: Tuple[int, ...] = (0, ), return_all: bool = False, max_abs_grad: float = None, **kwargs, ): #unscaled_problem_parameters = unscale_parameters( # scaled_parameters=problem_parameters, # petab_problem=parent_petab_problem, #) unscaled_problem_parameters = parent_petab_problem.unscale_parameters( scaled_parameters=problem_parameters, ) #print('in') results = simulate_timecourse( parent_petab_problem, timecourse_id, problem_parameters=unscaled_problem_parameters, parameter_mapping=parameter_mapping, sensi_orders=sensi_orders, **kwargs, ) #print('out') if kwargs.get('initial_states', None) is not None: pass #breakpoint() #sensitivity_parameter_ids = results[0]['sllh'].keys() sensitivity_parameter_ids = problem_parameters.keys() #print('l1') #print(problem_parameters) #breakpoint() #sensitivity_parameter_ids = parent_petab_problem.x_ids for result in results: if result['sllh'].keys() != sensitivity_parameter_ids: # FIXME reimplement so this still holds? #raise NotImplementedError( # 'All conditions must provide sensitivities for the same set ' # 'of parameters.' #) pass #print('l2') accumulated_result = { FVAL: sum(-result['llh'] for result in results), GRAD: [ sum(-result['sllh'][k] for result in results if k in result['sllh']) for k in sensitivity_parameter_ids #if k in result['sllh'] ] #GRAD: { # k: sum(-result['sllh'][k] for result in results) # for k in sensitivity_parameter_ids #} } #print('l3') if return_all: # TODO magic constant accumulated_result[FINAL_STATES] = \ one(results[-1]['rdatas']).x[-1].flatten() #accumulated_result[RESULTS] = results #print('l4') #print(accumulated_result) #print(accumulated_result) #breakpoint() #print(accumulated_result) if max_abs_grad is not None: pass #accumulated_result[GRAD] = list(np.nan_to_num( # np.array(accumulated_result[GRAD]), # nan=np.nan, # posinf=max_abs_grad, # neginf=-max_abs_grad, #)) #print(accumulated_result) return accumulated_result
def test_no_overlap(self): """ Tests that decoders do not overlap. If a byte of the response is processed by a bit decoder, then it can only be processed by bit decoders, and for different bits (multiple decoders may not test the same bits). If a byte of the response is processed by another decoder (byte, word or longer) then it may only be tested by a single decoder. """ for prefix, decoders in decoding.serial_page_prefix_to_decoders.items( ): non_bit_tested_byte_to_decoder = {} bit_tested_bytes_to_bit_to_decoders = defaultdict(dict) for decoder in decoders: # Find out which, if any, of the decode_bits canned decoders are used, # i.e. which bits are tested by this decoder. This only works if we only # use the canned decoders! tested_bits = { i for i, decode_fn in decoding.decode_bits.items() if decode_fn is decoder.decode_fn } # We assume that if no decode_bits decoders are used, the decoder must process # entire bytes, and thus conflict with any other decoder that processes the # same bytes: self.assertIn(len(tested_bits), [0, 1]) if len(tested_bits) == 1: # Which bits is it testing? Ensure that it doesn't overlap any decoder # that we've already checked, and allocate those bits to this one: tested_byte = decoder.start_position overlapping_decoder = non_bit_tested_byte_to_decoder.get( tested_byte) self.assertIsNone( overlapping_decoder, f"{decoder} and {overlapping_decoder} overlap on {prefix} " f"byte {tested_byte}") tested_bit = one(tested_bits) overlapping_decoder = ( bit_tested_bytes_to_bit_to_decoders[tested_byte].get( tested_bit)) self.assertIsNone( overlapping_decoder, f"{decoder} overlaps on {prefix} byte {tested_byte} " f"bit {tested_bit}") bit_tested_bytes_to_bit_to_decoders[tested_byte][ tested_bit] = decoder else: # Which bytes is it testing? Ensure that it doesn't overlap any decoder # that we've already checked, and allocate those bytes to this one: overlapping_decoders = { other_decoder for position, other_decoder in bit_tested_bytes_to_bit_to_decoders.items() if (position < decoder.end_position and position >= decoder.start_position) } self.assertEqual( set(), overlapping_decoders, f"{decoder} and {overlapping_decoders} overlap on {prefix} " f"bytes {decoder.start_position}-{decoder.end_position - 1}" ) overlapping_decoders = { other_decoder for position, other_decoder in non_bit_tested_byte_to_decoder.items() if (position < decoder.end_position and position >= decoder.start_position) } self.assertEqual( set(), overlapping_decoders, f"{decoder} overlaps on {prefix} bytes " f"{decoder.start_position}-{decoder.end_position - 1}") for i in range(decoder.start_position, decoder.end_position): non_bit_tested_byte_to_decoder[i] = decoder
def is_folder(self) -> bool: return one(unique_everseen(vp.is_folder for vp in self.generating_vps))
def get_qs_for_observation( self, observation: Observation) -> Mapping[Action, numbers.Real]: return more_itertools.one(self.get_qs_for_observations( (observation, )))
def process_form_submission(self, form, editor): people_allowed_to_sign_up = list( filter( lambda person: self.can_signup(editor, person), gefolge_web.login.Mensch if self.event.location is not None and self.event.location.is_online else self.event.signups)) if len(people_allowed_to_sign_up) > 0: if len(people_allowed_to_sign_up) == 1: person_to_signup = more_itertools.one( people_allowed_to_sign_up) else: person_to_signup = form.person_to_signup.data if not self.can_signup(editor, person_to_signup): flask.flash( jinja2.Markup( 'Du bist nicht berechtigt, {} für diesen Programmpunkt anzumelden.' .format(person_to_signup.__html__())), 'error') return flask.redirect( flask.url_for('event_programmpunkt', event=self.event.event_id, programmpunkt=self.url_part)) if 'challonge' in self.data: try: if form.challonge_username.data: challonge.participants.create( self.data['challonge'], name=person_to_signup.name, challonge_username=form.challonge_username.data, misc='id{}'.format(person_to_signup.snowflake)) else: challonge.participants.create( self.data['challonge'], name=person_to_signup.name, misc='id{}'.format(person_to_signup.snowflake)) except challonge.api.ChallongeException as e: flask.flash( jinja2.Markup( 'Bei der Anmeldung auf Challonge ist ein Fehler aufgetreten. Bitte versuche es nochmal. Falls du Hilfe brauchst, wende dich bitte an {}. Fehlermeldung: {}' .format( gefolge_web.login.Mensch.admin().__html__(), jinja2.escape(e))), 'error') return flask.redirect( flask.url_for('event_programmpunkt', event=self.event.event_id, programmpunkt=self.url_part)) if 'smashgg' in self.data: if form.smashgg_slug.data: #TODO validate that the user exists if 'smashggSlugs' not in self.data: self.data['smashggSlugs'] = {} self.data['smashggSlugs'][str( person_to_signup.snowflake)] = form.smashgg_slug.data if self.orga is not None: peter.msg( self.orga, '<@{}> ({}) hat sich für {} auf {} angemeldet. (smash.gg-Profil: {})' .format(person_to_signup.snowflake, person_to_signup, self, self.event, form.smashgg_slug.data) ) #TODO fix recipient if guest, fix formatting for EventGuests (dm_mention) else: if self.orga is not None: peter.msg( self.orga, '<@{}> ({}) hat sich für {} auf {} angemeldet. (kein smash.gg-Profil)' .format(person_to_signup.snowflake, person_to_signup, self, self.event) ) #TODO fix recipient if guest, fix formatting for EventGuests (dm_mention) self.signup(person_to_signup) self.process_form_details(form, editor)
def main(self): self.sources = self.get_source() self.targets = self.get_target() moved_ok = 0 moved_err = 0 deleted_ok = 0 deleted_err = 0 linked_ok = 0 linked_err = 0 logger.trace( json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=2)) for item in self.sources: if [ t for t in self.targets if t["path"] == item["path"] ] and one([t for t in self.targets if t["path"] == item["path"] ])["inode"] != item["inode"]: if self.recycle_path: if self.move(os.path.join(self.target_path, item["path"]), os.path.join(self.recycle_path, item["path"])): moved_ok += 1 else: moved_err += 1 else: if self.delete(os.path.join(self.target_path, item["path"])): deleted_ok += 1 else: deleted_err += 1 if not [t for t in self.targets if t["path"] == item["path"]]: if self.link(item["src"], os.path.join(self.target_path, item["path"])): linked_ok += 1 else: linked_err += 1 for item in self.targets: if not [s for s in self.sources if s["path"] == item["path"]]: if self.recycle_path: if self.move(os.path.join(self.target_path, item["path"]), os.path.join(self.recycle_path, item["path"])): moved_ok += 1 else: moved_err += 1 else: if self.delete(os.path.join(self.target_path, item["path"])): deleted_ok += 1 else: deleted_err += 1 if sum([ moved_ok, moved_err, deleted_ok, deleted_err, linked_ok, linked_err ]) > 0: logger.success("Run completed:") if sum([moved_ok, moved_err]) > 0: logger.success( f" Moved {moved_ok} successfully, {moved_err} failed.") if sum([deleted_ok, deleted_err]) > 0: logger.success( f" Deleted {deleted_ok} successfully, {deleted_err} failed." ) if sum([linked_ok, linked_err]) > 0: logger.success( f" Linked {linked_ok} successfully, {linked_err} failed.")
setup_logging(filename=args.logfile) log = get_logger('import_container_data') log.info('start_ingest') aspace = ASpace() def get_jsons(repo_id, id_set): '''Get chunk of Archival Object JSONModelObjects''' return aspace.client.get(f'repositories/{repo_id}/archival_objects', params={ 'id_set': ",".join(map(str, id_set)) }).json() repo_id_key_fn = lambda x: x['Repo ID'] rows = list(dictify_sheet(one(args.excel))) ao_ids_by_repo_id = groupby(sorted(rows, key=repo_id_key_fn), key=repo_id_key_fn) def get_id_num(uri): return int(uri[uri.rindex('/') + 1:]) def get_uri_prefix(uri): return uri[:uri.rindex('/') + 1] ao_jsons_by_id = {} for repo_id, aos in ao_ids_by_repo_id: for chunk in chunked((el['Archival Object'] for el in aos), 250): ao_jsons_by_id.update({ get_id_num(ao['uri']): ao for ao in get_jsons(repo_id, chunk)
def pluralize(**thing): name, value = one(thing.items()) if name.endswith('y') and name[-2] not in 'aeiou': name = f'{name[:-1]}ies' if value != 1 else name return f'{value} {name}' return f'{value} {name}{"s" * (value != 1)}'
def get_neurons_of_sample_states_and_best_actions(self) -> Tuple[np.ndarray, Tuple[Action, ...]]: culture = Culture(n_players=1, core_strategies=(self,) * N_CORE_STRATEGIES) def make_state(player_position: Position, food_positions: Iterable[Position]) -> State: observation = Observation(None, player_position, letter=LETTERS[0], score=0, reward=0, last_action=None) player_id_to_observation = ImmutableDict({observation.letter: observation}) state = State( culture, board_size=self.culture.board_size, player_id_to_observation=player_id_to_observation, food_positions=frozenset(food_positions) ) observation.state = state return state def make_states(player_position: Position, food_positions: Iterable[Position]) -> Tuple[State]: return tuple( make_state(rotated_player_position, rotated_food_positions) for rotated_player_position, *rotated_food_positions in zip( *map(lambda position: position.iterate_rotations_in_board( board_size=self.culture.board_size), itertools.chain((player_position,), food_positions)) ) ) player_positions = [Position(x, y) for x, y in itertools.product((5, 11, 16), repeat=2)] states = [] for player_position in player_positions: distances_lists = tuple( itertools.chain.from_iterable( itertools.combinations(range(1, 4 + 1), i) for i in range(1, 3 + 1) ) ) for distances, step in itertools.product(distances_lists, Step.all_steps): states.extend( make_states(player_position, [player_position + distance * step for distance in distances]) ) observations = [more_itertools.one(state.player_id_to_observation.values()) for state in states] def _get_cute_score_for_action(observation: Observation, legal_move_action: Action) -> int: next_state = observation.state. \ get_next_state_from_actions({observation.letter: legal_move_action}) return more_itertools.one(next_state.player_id_to_observation.values()).cute_score return tuple(( np.concatenate([observation.to_neurons()[np.newaxis, :] for observation in observations]), tuple( max( observation.legal_move_actions, key=lambda legal_action: _get_cute_score_for_action(observation, legal_action) ) for observation in observations ) ))
def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: """ Returns an `enum` schema for the given items. By default, the schema type of the items is inferred, but a type may be passed explicitly to override that. However, the current implementation cannot detect some cases in which the types of the enum values contradict the explicit type. >>> from azul.doctests import assert_json >>> assert_json(enum('foo', 'bar', type_=str)) { "type": "string", "enum": [ "foo", "bar" ] } >>> assert_json(enum(2, 5, 7)) { "type": "integer", "format": "int64", "enum": [ 2, 5, 7 ] } >>> assert_json(enum('x', type_={'type': 'string'})) { "type": "string", "enum": [ "x" ] } >>> enum('foo', 1.0) Traceback (most recent call last): ... ValueError: too many items in iterable (expected 1) >>> enum('foo', 'bar', type_=int) Traceback (most recent call last): ... AssertionError >>> assert_json(enum('foo', 'bar', type_="integer")) { "type": "integer", "enum": [ "foo", "bar" ] } """ if isinstance(type_, type): assert all(isinstance(item, type_) for item in items) else: inferred_type = one(set(map(type, items))) if type_ is None: type_ = inferred_type else: # Can't easily verify type when passed as string or mapping pass return {**make_type(type_), 'enum': items}
def get_ancestor_class_sequence( class_name: 'AncestorClassName', ancestor_chain: Dict['AncestorClassName', str] = None, enforced_ancestor_choices: Dict['AncestorChoiceKey', 'AncestorClassName'] = None) \ -> Sequence['AncestorClassName']: """Returns the sequence of ancestor classes leading from the root of the object graph, a Person type, all the way to the given |class_name|. This handles graphs with multiple inbound edges to a particular type. For example, StateCharge can be a child of IncarcerationSentence, SupervisionSentence, and Fine. To determine exactly which sequence to return, context is derived from the other parameters. |ancestor_chain| is the set of ids of known ancestor instances for which we are trying to find the ancestor sequence. If |class_name| is state_charge and the |ancestor_chain| is a dict containing a key of 'state_incarceration_sentence', then the returned sequence will be ('state_person', 'state_sentence_group', 'state_incarceration_sentence'). |enforced_ancestor_choices| is mapping of ancestor classes to specifically choose where there are multiple options. If |class_name| is state_charge and |enforced_ancestor_choices| is {'state_sentence': 'state_supervision_sentence'}, then the returned sequence will be ('state_person', 'state_sentence_group', 'state_supervision_sentence'). If there is a valid key in the |ancestor_chain| and there are also |enforced_ancestor_choices|, then the key in |ancestor_chain| wins. Args: class_name: The name of the class to find the sequence for ancestor_chain: A dictionary with keys of ancestor class types and values of specific ancestor instance ids enforced_ancestor_choices: A dictionary with keys of kinds of ancestor choices, e.g. `state_sentence` for any of the state sentence types, and values of the specific choice for that kind of choice, e.g. `state_incarceration_sentence` """ if not ancestor_chain: ancestor_chain = {} if not enforced_ancestor_choices: enforced_ancestor_choices = {} hierarchy_sequence = [] for step in _HIERARCHY_MAP[class_name]: if isinstance(step, str): hierarchy_sequence.append(step) elif isinstance(step, AncestorTypeChoices): if step.key not in VALID_ANCESTOR_CHOICE_KEYS: raise ValueError( f"Invalid ancestor choice key of [{step.key}], must be one " f"of [{VALID_ANCESTOR_CHOICE_KEYS}]") # First, pick if we've selected one of the choices via the # ancestor chain choices_ancestor_chain_overlap = \ step.ancestor_choices.intersection(ancestor_chain.keys()) if choices_ancestor_chain_overlap: if len(choices_ancestor_chain_overlap) > 1: raise ValueError( "There are multiple valid ancestor choices in the " "given ancestor chain. Valid choices are: " f"[{step.ancestor_choices}]. Ancestor chain includes: " f"[{ancestor_chain.keys()}]") hierarchy_sequence.append(one(choices_ancestor_chain_overlap)) continue # Next, check the enforced_ancestor_choices if not enforced_ancestor_choices: raise ValueError( "For possible ancestor choices of " f"[{step.ancestor_choices}], there is neither overlap with " f"the ancestor chain [{ancestor_chain.keys()}] nor a " f"declared choice. We don't have enough information to " f"construct the ancestor hierarchy for this object.") if step.key not in enforced_ancestor_choices: raise ValueError( f"The enforced choices [{enforced_ancestor_choices}] don't " f"contain a mapping for [{step.key}]. We don't have enough " "information to construct the ancestor hierarchy for this " "object.") choice = enforced_ancestor_choices[step.key] if choice not in _HIERARCHY_MAP: raise ValueError( f"Invalid ancestor choice value of [{choice}], must be a " "valid type listed in the hierarchy map.") hierarchy_sequence.append(choice) else: raise ValueError(f"Unknown type [{type(step)}] in hierarchy map.") return tuple(hierarchy_sequence)
async def validator(): cache_data = validator_TTCache.get(VALIDATOR_CACHE_KEY) if cache_data: resp: ValidatorsResponse = cache_data else: async with lock.lock: cache_data = validator_TTCache.get(VALIDATOR_CACHE_KEY) if cache_data: return cache_data else: latest_block_number_tasks = [] for validator in setting.validator_list: latest_block_number_tasks.append( get_latest_block(validator)) latest_infos = await asyncio.gather(*latest_block_number_tasks, return_exceptions=True) latest_infos_no_exception = list( filter(lambda x: x.block_number != NO_LATEST_BLOCK, latest_infos)) latest_num_dict: Dict[str, LatestInfo] = { i.validator.host: i for i in latest_infos } # get latest blocks from all the validators failed then randomly return the `nextToPropose` if len(latest_infos_no_exception) == 0: best = random.choice(setting.validator_list) max_block_numbers = NO_LATEST_BLOCK else: max_block_numbers = max( [i.block_number for i in latest_infos_no_exception]) latest = first_true( latest_infos_no_exception, lambda x: x.block_number == max_block_numbers) index = one( locate(setting.validator_list, lambda x: x.pub_key == latest.sender)) # why +2 ? # actually index validator should be the latest proposed validator # but it is possible that at this moment, the next validator is already trying # to propose a new block. So choosing the +2 validator is more reliable best = nth(ncycles(setting.validator_list, 2), index + 2) split_validators = list( split_before(setting.validator_list, lambda x: x.host == best.host)) if len(split_validators) == 1: sorted_validators = one(split_validators) else: sorted_validators = last(split_validators) + first( split_validators) validators = list( map( lambda x: Validator(host=x.host, grpc_port=x.grpc_port, http_port=x.http_port, latestBlockNumber=latest_num_dict. get(x.host).block_number, timestamp=latest_num_dict.get( x.host).timestamp), sorted_validators)) nextToPropose = NextToPropose( host=best.host, grpcPort=best.grpc_port, httpPort=best.http_port, latestBlockNumber=max_block_numbers) resp = ValidatorsResponse(nextToPropose=nextToPropose, validators=validators) validator_TTCache[VALIDATOR_CACHE_KEY] = resp return resp.dict()
def section(self): return more_itertools.one(section for section in self.location.data['rooms'] if self.name in section).key
def open_file(self, flags): return one(self.generating_vps).open_file(flags)
def _data(self): response = self._vk.api.messages.getById(message_ids=self.id) return mit.one(response["items"])
def path(self) -> str: return one( unique_everseen(vp.partial_path for vp in self.generating_vps))
def consensus(values): from itertools import groupby from more_itertools import one return one((x[0] for x in groupby(values)))
def fix_sam_read_groups(sam_input, out_path, progress_bar=False): """ This was written to deal with the result of read group tagging by BWA. The produced SAM has a single read group in the header, under a line like: @RG ID:PLTF.RUN.FLOWCELL.LANE1 LB:LB PL:ILLUMINA PU:PU SM:SM But then reads might come from different lanes than LANE1, which can be seen in the Illumina read IDs: PLTF:RUN:FLOWCELL:LANE1:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 PLTF:RUN:FLOWCELL:LANE1:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 PLTF:RUN:FLOWCELL:LANE2:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 <- :( PLTF:RUN:FLOWCELL:LANE2:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 <- :( The idea is to produce a new SAM fixing this situation in two ways: - The header of the new SAM must have a new @RG line for each different read group inferred from the read IDs. In this case: @RG ID:PLTF.RUN.FLOWCELL.LANE1 LB:LB PL:ILLUMINA PU:PU SM:SM @RG ID:PLTF.RUN.FLOWCELL.LANE2 LB:LB PL:ILLUMINA PU:PU SM:SM - The read group assigned to each read in the RG:Z: tag should match its true read grop, inferred from the first fields of the read ID: PLTF:RUN:FLOWCELL:LANE1:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 PLTF:RUN:FLOWCELL:LANE1:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE1 PLTF:RUN:FLOWCELL:LANE2:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE2 <- ok PLTF:RUN:FLOWCELL:LANE2:X:Y:FOO ... RG:Z:PLTF:RUN:FLOWCELL:LANE2 <- ok Returns the path to the new SAM produced with those fixes. """ # NOTE: the SAM files will be HEAVY, specially for exome data (say, 20Gb, # 50 million lines). # if os.path.isfile(out_path): # os.remove(out_path) assert '.sam' in out_path # accepts files like "myfile.sam-luigi-tmp-1234" # Write the header of the new SAM, with one @RG entry for each lane headers = [] with open(sam_input) as f_in: for line in f_in: if not line.startswith("@"): # When the header is over break headers.append(line) headers_not_RG = [l for l in headers if not l.startswith('@RG')] headers_RG = [l for l in headers if l.startswith('@RG')] # This entire function assumes the following structure for read group IDs: # INSTRUMENT.RUN.FLOWCELL.1-2-3 # where 1-2-3 are the merged lane numbers lane_numbers = [] for RG_line in headers_RG: RG_header_parts = RG_line.split('\t') # [@RG, ID:INSTRUMENT.RUN.FLOWCELL.1-2-3, LB:LIB, PL:ILLUMINA, ...] RG_ID = one(chunk for chunk in RG_header_parts if 'ID:' in chunk) # ID:INSTRUMENT.RUN.FLOWCELL.1-2-3 lane_numbers_merged = RG_ID.split('.')[-1] # 1-2-3 for lane_number in lane_numbers_merged.split('-'): # [1, 2, 3] lane_numbers.append(lane_number) # One RG line for each lane number seen: new_RG_headers = [] template_RG_line = headers_RG[0] for lane_number in sorted(unique_everseen(lane_numbers)): new_RG_header = re.sub(r'(ID:)(.+?\.)(.+?\.)(.+?\.)(.+?)(\s)', rf'\1\2\3\g<4>{lane_number}\6', template_RG_line) new_RG_headers.append(new_RG_header) with open(out_path, 'w') as f_out: for header in headers_not_RG + new_RG_headers: f_out.write(header) # Write the body of the SAM, fixing read groups read by read: regex_id = re.compile(r'^(.+?:.+?:.+?:.+?):.+?:.+?:.+?\s') regex_rg = re.compile(r'\sRG:Z:(.+)\s') inferred_read_groups = set() with open(sam_input) as f_in, open(out_path, 'a') as f_out: input_iterable = tqdm(f_in) if progress_bar else f_in # NOTE: this will loop over ~50M lines for exome data! for line in input_iterable: if line.startswith('@'): continue fixed_line = line match_id = regex_id.match(line) match_rg = regex_rg.search(line) inferred_read_group = match_id.group(1).replace(':', '.') seen_read_group = match_rg.group(1) if seen_read_group != inferred_read_group: fixed_line = fixed_line.replace(seen_read_group, inferred_read_group) inferred_read_groups.add(inferred_read_group) f_out.write(fixed_line) return out_path