def test_trace_with_geometric_objects_1(self): """ Only one of the geometric objects has a transformation applied. """ root = Node(name="Root", geometry=Sphere(radius=10.0)) a = Node(name="A", parent=root, geometry=Sphere(radius=1.0)) b = Node(name="B", parent=root, geometry=Sphere(radius=1.0)) b.translate((5.0, 0.0, 0.0)) scene = Scene(root) tracer = PhotonTracer(scene) position = (-3.0, 0.0, 0.0) direction = (1.0, 0.0, 0.0) initial_ray = Ray( position=position, direction=direction, wavelength=555.0, is_alive=True ) expected_history = [ initial_ray, # Starting ray replace(initial_ray, position=(-1.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(1.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(4.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(6.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(10.0, 0.0, 0.0), is_alive=False), # Exit ray ] history = tracer.follow(initial_ray) for pair in zip(history, expected_history): a, b = pair print("Testing {} {}".format(a.position, b.position)) assert np.allclose(a.position, b.position)
def structural(p): print(len(p)) dataclasses.fields(p) dataclasses.asdict(p) dataclasses.astuple(p) dataclasses.replace(p)
def test_trace_with_material_object(self): """ Root node and test object has a material attached. """ np.random.seed(1) # No reflections # np.random.seed(2) # Reflection at last inteface root = Node(name="Root", geometry=Sphere(radius=10.0, material=Dielectric.make_constant((400, 800), 1.0))) b = Node(name="B", parent=root, geometry=Sphere(radius=1.0, material=Dielectric.make_constant((400, 800), 1.5))) b.translate((5.0, 0.0, 0.0)) scene = Scene(root) tracer = PhotonTracer(scene) position = (-3.0, 0.0, 0.0) direction = (1.0, 0.0, 0.0) initial_ray = Ray( position=position, direction=direction, wavelength=555.0, is_alive=True ) expected_history = [ initial_ray, # Starting ray replace(initial_ray, position=(4.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(4.0, 0.0, 0.0)), # Refracted into A replace(initial_ray, position=(6.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(6.0, 0.0, 0.0)), # Refracted out of A replace(initial_ray, position=(10.0, 0.0, 0.0), is_alive=False), # Exit ray ] history = tracer.follow(initial_ray) for pair in zip(history, expected_history): a, b = pair print("Testing {} {}".format(a.position, b.position)) assert np.allclose(a.position, b.position)
def trace_path(self, ray: Ray, container_node: Node, intersection_node: Node, intersection_point: tuple) -> Tuple[Ray, Node]: """ Determines if the ray is absorbed or scattered along it's path and returns a new ray is these processes occur. If not, or the container node does not have a material attached the ray is moved to the next intersection point. """ # Exit early if possible if any([node.geometry is None for node in (container_node, intersection_node)]): raise TraceError("Node is missing a geometry.") elif container_node.geometry.material is None: logger.debug("Container node is missing a material. Will propagate ray the full path length.") new_ray = replace(ray, position=intersection_point) hit_node = intersection_node return (new_ray, hit_node) # Have a proper material root = self.scene.root distance = distance_between(ray.position, intersection_point) volume = make_volume(container_node, distance) new_ray = volume.trace(ray) position_in_intersection_node_frame = root.point_to_node(new_ray.position, intersection_node) if intersection_node.geometry.is_on_surface(position_in_intersection_node_frame): # Hit interface between container node and intersection node hit_node = intersection_node else: # Hit molecule in container node (i.e. was absorbed or scattered) hit_node = container_node return new_ray, hit_node
def _update_row_diff(self, rowid: int, **kwds: Any) -> None: row = self.get_row(rowid) if not row: raise RowLookupError(rowid) upd = dataclasses.replace(row, **kwds) upd.id = rowid upd.state = row.state self._diff[rowid] = upd
def test_no_interaction(self): np.random.seed(0) mat = Dielectric.make_constant((400, 800), 1.0) root = Node(name="Root", parent=None) root.geometry = Sphere(radius=10.0, material=mat) a = Node(name="A", parent=root) a.geometry = Sphere(radius=1.0, material=mat) ray = Ray(position=(-1.0, 0.0, 0.0), direction=(1.0, 0.0, 0.0), wavelength=555.0, is_alive=True) volume = Volume(a, 2.0) new_ray = volume.trace(ray) expected = replace(ray, position=(1.0, 0.0, 0.0)) assert new_ray == expected
def transform(self, ray: Ray) -> Ray: """ Transform ray according to the physics of the interaction. """ context = self.context normal = np.array(context.normal) ray_ = ray.representation(context.normal_node.root, context.normal_node) vec = np.array(ray_.direction) d = np.dot(normal, vec) reflected_direction = vec - 2 * d * normal new_ray_ = replace(ray_, direction=tuple(reflected_direction.tolist())) new_ray = new_ray_.representation(context.normal_node, context.normal_node.root) return new_ray # back to world node
def transform(self, ray: Ray) -> Ray: """ Transform ray according to the physics of the interaction. """ context = self.context material = context.interaction_material if not isinstance(material, Emissive): AppError("Need an emissive material.") new_wavelength = material.redshift_wavelength(ray.wavelength) new_direction = material.emission_direction() logger.debug("Wavelength was {} and is now {}".format(ray.wavelength, new_wavelength)) new_ray = replace(ray, wavelength=new_wavelength, direction=new_direction) return new_ray
def test_trace_with_translated_geometric_object(self): """ Single translated geometric objects. """ root = Node(name="Root", geometry=Sphere(radius=10.0)) a = Node(name="A", parent=root, geometry=Sphere(radius=1.0)) a.translate((5.0, 0.0, 0.0)) scene = Scene(root) tracer = PhotonTracer(scene) position = (-2.0, 0.0, 0.0) direction = (1.0, 0.0, 0.0) initial_ray = Ray( position=position, direction=direction, wavelength=555.0, is_alive=True ) expected_history = [ initial_ray, # Starting ray replace(initial_ray, position=(4.0, 0.0, 0.0)), # First intersection replace(initial_ray, position=(6.0, 0.0, 0.0)), # Second intersection replace(initial_ray, position=(10.0, 0.0, 0.0), is_alive=False), # Exit ray ] history = tracer.follow(initial_ray) for pair in zip(history, expected_history): assert pair[0] == pair[1]
def test_trace_with_geometric_object(self): """ Contains a single object without an attached material: a geometric object. In this case we expect the tracer to just return the intersection points with the scene. """ root = Node(name="Root", geometry=Sphere(radius=10.0)) a = Node(name="A", parent=root, geometry=Sphere(radius=1.0)) scene = Scene(root) tracer = PhotonTracer(scene) position = (-2.0, 0.0, 0.0) direction = (1.0, 0.0, 0.0) initial_ray = Ray( position=position, direction=direction, wavelength=555.0, is_alive=True ) expected_history = [ initial_ray, # Starting ray replace(initial_ray, position=(-1.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(1.0, 0.0, 0.0)), # Moved to intersection replace(initial_ray, position=(10.0, 0.0, 0.0), is_alive=False), # Exit ray ] history = tracer.follow(initial_ray) for pair in zip(history, expected_history): assert pair[0] == pair[1]
def propagate(self, distance: float) -> Ray: """ Returns a new ray which has been moved the specified distance along its direction. Parameters ---------- distance : float The distance to move the ray. Can be negative in which case the new ray will be moved backwards. """ if not self.is_alive: raise ValueError('Ray is not alive.') new_position = np.array(self.position) + np.array(self.direction) * distance new_position = tuple(new_position.tolist()) new_ray = replace(self, position=new_position) return new_ray
def test_trace_without_objects(self): """ Trace empty scene. Should return intersections with root node. """ root = Node(name="Root", geometry=Sphere(radius=10.0)) scene = Scene(root) tracer = PhotonTracer(scene) position = (-2.0, 0.0, 0.0) direction = (1.0, 0.0, 0.0) initial_ray = Ray( position=position, direction=direction, wavelength=555.0, is_alive=True ) expected_history = [ initial_ray, # Starting ray replace(initial_ray, position=(10.0, 0.0, 0.0), is_alive=False), # Exit ray ] history = tracer.follow(initial_ray) for pair in zip(history, expected_history): assert pair[0] == pair[1]
def _get_login_response_authn(self, ticket: SSOLoginData, user: IdPUser) -> AuthnInfo: """ Figure out what AuthnContext to assert in the SAML response. The 'highest' Assurance-Level (AL) asserted is basically min(ID-proofing-AL, Authentication-AL). What AuthnContext is asserted is also heavily influenced by what the SP requested. :param ticket: State for this request :param user: The user for whom the assertion will be made :return: Authn information """ self.logger.debug('MFA credentials logged in the ticket: {}'.format(ticket.mfa_action_creds)) self.logger.debug('External MFA credential logged in the ticket: {}'.format(ticket.mfa_action_external)) self.logger.debug('Credentials used in this SSO session:\n{}'.format(self.sso_session.authn_credentials)) self.logger.debug('User credentials:\n{}'.format(user.credentials.to_list())) # Decide what AuthnContext to assert based on the one requested in the request # and the authentication performed req_authn_context = get_requested_authn_context(self.context.idp, ticket.saml_req, self.logger) try: resp_authn = eduid_idp.assurance.response_authn(req_authn_context, user, self.sso_session, self.logger) except WrongMultiFactor as exc: self.logger.info('Assurance not possible: {!r}'.format(exc)) raise eduid_idp.error.Forbidden('SWAMID_MFA_REQUIRED') except MissingMultiFactor as exc: self.logger.info('Assurance not possible: {!r}'.format(exc)) raise eduid_idp.error.Forbidden('MFA_REQUIRED') except AssuranceException as exc: self.logger.info('Assurance not possible: {!r}'.format(exc)) raise MustAuthenticate() self.logger.debug("Response Authn context class: {!r}".format(resp_authn)) try: self.logger.debug("Asserting AuthnContext {!r} (requested: {!r})".format( resp_authn, req_authn_context)) except AttributeError: self.logger.debug("Asserting AuthnContext {!r} (none requested)".format(resp_authn)) # Augment the AuthnInfo with the authn_timestamp before returning it return replace(resp_authn, instant=self.sso_session.authn_timestamp)
def representation(self, from_node: Node, to_node: Node) -> Ray: """ Representation of the ray in another coordinate system. Parameters ---------- from_node : Node The node which represents the ray's current coordinate system to_node : Node The node in which the new ray should be represented. Notes ----- Use this method to express the ray location and direction as viewed in the `to_node` coordinate system. """ new_position = from_node.point_to_node(self.position, to_node) new_direction = from_node.vector_to_node(self.direction, to_node) new_ray = replace(self, position=new_position, direction=new_direction) return new_ray
def transform(self, ray: Ray) -> Ray: """ Transform ray according to the physics of the interaction. """ context = self.context n1 = context.n1 n2 = context.n2 ray_ = ray.representation(context.normal_node.root, context.normal_node) normal = np.array(context.normal) vector = np.array(ray_.direction) n = n1/n2 dot = np.dot(vector, normal) c = np.sqrt(1 - n**2 * (1 - dot**2)) sign = 1 if dot < 0.0: sign = -1 refracted_direction = n * vector + sign*(c - sign*n*dot) * normal new_ray_ = replace(ray_, direction=tuple(refracted_direction.tolist())) new_ray = new_ray_.representation(context.normal_node, context.normal_node.root) return new_ray
def step(self, ray: Ray) -> Ray: """ Steps the ray one event forward. """ ctx = self.make_step_context(ray) logger.debug('Ray is in {}.'.format(ctx.container)) initial_ray = ray next_intersection = ctx.next_intersection() logger.debug("Tracing along path.") ray, hit_node = self.trace_path(ray, ctx.container, next_intersection.hit, next_intersection.point) if hit_node == self.scene.root: ray = replace(ray, is_alive=False) logger.debug("Ray died {}".format(ray)) yield ray else: logger.debug('Ray moved and hit or is inside {}.'.format(hit_node)) yield ray # yield ray motion step ray_on_hit_node = hit_node.geometry.is_on_surface(self.scene.root.point_to_node(ray.position, hit_node)) if ray_on_hit_node: if np.allclose(ray.direction, initial_ray.direction): interface = (ctx.container, hit_node) if ctx.container == hit_node: interface = (ctx.container, ctx.all_intersections[ctx.next_index+1].hit) else: logger.debug('Direction changed. Re-calculating intersections.') ctx = self.make_step_context(ray) interface = (ctx.container, ctx.next_intersection().hit) logger.debug("Interface {}".format(interface)) # Check that a material exists on both sides of the interface has_material = [side.geometry.material is not None for side in interface] if all(has_material): logger.debug("Tracing interface.") ray = self.trace_interface(ray, interface) yield ray # yield ray interface step and exit elif all(np.logical_not(has_material)): logger.debug("Interface does not have materials.") elif len(set(has_material)) == 2: logger.debug(traceback.format_exc()) raise TraceError("Both interface nodes must have a material. At interface {} {}".format(interface[0].geometry.material, interface[1].geometry.material))
def refresh_settings(self, ovrIP=False): self.settings = dataclasses.replace( self.settings, **ValidateSettings.get_current_settings(self.launcher, ovrIP=ovrIP)) self.ipPortCombo = f'{self.settings.PublicIP}:{self.settings.Port}'
def clamp(message: AnalysisMessage): if not samefile(message.filename, cls_file): return replace(message, filename=cls_file, line=cls_start_line) else: return message
def transform(self, ray: Ray) -> Ray: new_ray = replace(ray, is_alive=False) return new_ray
async def main_writer(self): """ Main coroutine for submitting commands. """ LOG.info("Writer loop for party %s is starting...", self.party) ledger_fut = ensure_future(self._pool.ledger()) client = await self._client_fut # type: LedgerClient metadata = await ledger_fut # type: LedgerMetadata validator = ValidateSerializer(self.parent.lookup) self._writer.pending_commands.start() # Asynchronously iterate over all pending commands that a user has (or will) send. # This asynchronous loop "blocks" when pending_commands is empty and is "woken up" # immediately when new data is added to it. The loop terminates when the pending_commands # ServiceQueue is stopped. async for p in self._writer.pending_commands: LOG.debug("Sending a command: %s", p) command_payloads = [ ] # type: List[Tuple[_PendingCommand, Sequence[CommandPayload]]] if p.future.done(): # PendingCommand instances that are already marked as done have either been marked # as a failure or cancelled by the caller. Do NOT send the corresponding Ledger API # command because the PendingCommand() has effectively been aborted. continue self._writer.inflight_commands.append(p) try: defaults = CommandDefaults( default_party=self.party, default_ledger_id=metadata.ledger_id, default_workflow_id=None, default_application_id=self._config.application_name, default_command_id=None, ) cps = p.command.build(defaults) if cps: commands = await metadata.package_loader.do_with_retry( lambda: [ replace(cp, commands=validator.serialize_commands( cp.commands)) for cp in cps ]) command_payloads.append((p, commands)) await submit_command_async(client, p, commands) else: # This is a "null command"; don't even bother sending to the server. Immediately # resolve the future successfully and discard if not p.future.done(): p.future.set_result(None) except Exception as ex: LOG.exception("Tried to send a command and failed!") p.notify_read_fail(ex) LOG.info("Writer loop for party %s is winding down.", self.party) # After the pending command list is fully empty (and never to be filled again), wait for # all outstanding commands. done, pending = await wait( [pc.future for pc in self._writer.inflight_commands], timeout=5, return_when=ALL_COMPLETED, ) if pending: LOG.warning( "Writer loop for party %s has NOT fully finished, " "but will be terminated anyway (%d futures still pending).", self.party, len(pending), ) else: LOG.info("Writer loop for party %s is finished.", self.party)
def union2(p: Union[Type[A], Type[B]]): dataclasses.fields(p) dataclasses.asdict(<warning descr="'dataclasses.asdict' method should be called on dataclass instances">p</warning>) dataclasses.astuple(<warning descr="'dataclasses.astuple' method should be called on dataclass instances">p</warning>) dataclasses.replace(<warning descr="'dataclasses.replace' method should be called on dataclass instances">p</warning>)
async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> HelmChart: dependencies, source_files, metadata = await MultiGet( Get(Targets, DependenciesRequest(request.field_set.dependencies)), Get( HelmChartSourceFiles, HelmChartSourceFilesRequest, HelmChartSourceFilesRequest.for_field_set( request.field_set, include_metadata=False, include_resources=True, include_files=True, ), ), Get(HelmChartMetadata, HelmChartMetaSourceField, request.field_set.chart), ) third_party_artifacts = await Get( FetchedHelmArtifacts, FetchHelmArfifactsRequest, FetchHelmArfifactsRequest.for_targets( dependencies, description_of_origin=request.field_set.address.spec), ) first_party_subcharts = await MultiGet( Get(HelmChart, HelmChartRequest, HelmChartRequest.from_target(target)) for target in dependencies if HelmChartFieldSet.is_applicable(target)) third_party_charts = await MultiGet( Get(HelmChart, FetchedHelmArtifact, artifact) for artifact in third_party_artifacts) subcharts = [*first_party_subcharts, *third_party_charts] subcharts_digest = EMPTY_DIGEST if subcharts: logger.debug( f"Found {pluralize(len(subcharts), 'subchart')} as direct dependencies on Helm chart at: {request.field_set.address}" ) merged_subcharts = await Get( Digest, MergeDigests([chart.snapshot.digest for chart in subcharts])) subcharts_digest = await Get(Digest, AddPrefix(merged_subcharts, "charts")) # Update subchart dependencies in the metadata and re-render it. remotes = subsystem.remotes() subchart_map: dict[str, HelmChart] = { chart.metadata.name: chart for chart in subcharts } updated_dependencies: OrderedSet[HelmChartDependency] = OrderedSet() for dep in metadata.dependencies: updated_dep = dep if not dep.repository and remotes.default_registry: # If the dependency hasn't specified a repository, then we choose the registry with the 'default' alias. default_remote = remotes.default_registry updated_dep = dataclasses.replace( updated_dep, repository=default_remote.address) elif dep.repository and dep.repository.startswith("@"): remote = next(remotes.get(dep.repository)) updated_dep = dataclasses.replace(updated_dep, repository=remote.address) if dep.name in subchart_map: updated_dep = dataclasses.replace( updated_dep, version=subchart_map[dep.name].metadata.version) updated_dependencies.add(updated_dep) # Include the explicitly provided subchats in the set of dependencies if not already present. updated_dependencies_names = {dep.name for dep in updated_dependencies} remaining_subcharts = [ chart for chart in subcharts if chart.metadata.name not in updated_dependencies_names ] for chart in remaining_subcharts: if chart.artifact: dependency = HelmChartDependency( name=chart.artifact.name, version=chart.artifact.version, repository=chart.artifact.location_url, ) else: dependency = HelmChartDependency( name=chart.metadata.name, version=chart.metadata.version) updated_dependencies.add(dependency) # Update metadata with the information about charts' dependencies. metadata = dataclasses.replace( metadata, dependencies=tuple(updated_dependencies)) # Re-render the Chart.yaml file with the updated dependencies. metadata_digest, sources_without_metadata = await MultiGet( Get(Digest, HelmChartMetadata, metadata), Get( Digest, DigestSubset( source_files.snapshot.digest, PathGlobs([ "**/*", *(f"!**/{filename}" for filename in HELM_CHART_METADATA_FILENAMES) ]), ), ), ) # Merge all digests that conform chart's content. content_digest = await Get( Digest, MergeDigests( [metadata_digest, sources_without_metadata, subcharts_digest])) chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, metadata.name)) return HelmChart(address=request.field_set.address, metadata=metadata, snapshot=chart_snapshot)
def replace(self, **changes) -> 'CircuitOperation': """Returns a copy of this operation with the specified changes.""" return dataclasses.replace(self, **changes)
def decode_single(player_index: int, num_players: int, game_modifications: dict, configuration: LayoutConfiguration) -> GamePatches: """ Decodes a dict created by `serialize` back into a GamePatches. :param game_modifications: :param player_index: :param num_players: :param configuration: :return: """ game = data_reader.decode_data(configuration.game_data) game_specific = dataclasses.replace( game.game_specific, energy_per_tank=configuration.energy_per_tank, beam_configurations=configuration.beam_configuration.create_game_specific(game.resource_database)) world_list = game.world_list # Starting Location starting_location = _area_name_to_area_location(world_list, game_modifications["starting_location"]) # Initial items starting_items = { find_resource_info_with_long_name(game.resource_database.item, resource_name): quantity for resource_name, quantity in game_modifications["starting_items"].items() } # Elevators elevator_connection = {} for source_name, target_name in game_modifications["elevators"].items(): source_area = _area_name_to_area_location(world_list, source_name) target_area = _area_name_to_area_location(world_list, target_name) potential_source_nodes = [ node for node in world_list.area_by_area_location(source_area).nodes if isinstance(node, TeleporterNode) ] assert len(potential_source_nodes) == 1 source_node = potential_source_nodes[0] elevator_connection[source_node.teleporter_instance_id] = target_area # Translator Gates translator_gates = { _find_gate_with_name(gate_name): find_resource_info_with_long_name(game.resource_database.item, resource_name) for gate_name, resource_name in game_modifications["translators"].items() } # Pickups target_name_re = re.compile(r"(.*) for Player \d+") index_to_pickup_name = {} for world_name, world_data in game_modifications["locations"].items(): for area_node_name, target_name in world_data.items(): if target_name == _ETM_NAME: continue pickup_name_match = target_name_re.match(target_name) if pickup_name_match is not None: pickup_name = pickup_name_match.group(1) else: pickup_name = target_name node = world_list.node_from_name(f"{world_name}/{area_node_name}") assert isinstance(node, PickupNode) index_to_pickup_name[node.pickup_index] = pickup_name decoder = BitPackDecoder(base64.b64decode(game_modifications["_locations_internal"].encode("utf-8"), validate=True)) pickup_assignment = dict(BitPackPickupEntryList.bit_pack_unpack(decoder, { "index_mapping": index_to_pickup_name, "num_players": num_players, "database": game.resource_database, }).value) # Hints hints = {} for asset_id, hint in game_modifications["hints"].items(): hints[LogbookAsset(int(asset_id))] = Hint.from_json(hint) return GamePatches( player_index=player_index, pickup_assignment=pickup_assignment, # PickupAssignment elevator_connection=elevator_connection, # Dict[int, AreaLocation] dock_connection={}, # Dict[Tuple[int, int], DockConnection] dock_weakness={}, # Dict[Tuple[int, int], DockWeakness] translator_gates=translator_gates, starting_items=starting_items, # ResourceGainTuple starting_location=starting_location, # AreaLocation hints=hints, game_specific=game_specific, )
def _replace(self, **kwargs): return replace(self, **kwargs)
def replace(self, **kw): return dataclasses.replace(self, **kw)
from dataclasses import dataclass, field, InitVar, replace @dataclass class A: a: int b: str = "str" replace(A(1)) replace(A(1), a=1, b="abc") replace(A(1), <warning descr="Expected type 'int', got 'str' instead">a="str"</warning>, <warning descr="Expected type 'str', got 'int' instead">b=1</warning>) @dataclass class B: a: int b: str = field(default="str", init=False) replace(B(1)) replace(B(1), a=1) replace(B(1), <warning descr="Expected type 'int', got 'str' instead">a="str"</warning>) @dataclass class C: a: int b: InitVar[str] = "str"
def mangle_column_value(exp: Expression) -> Expression: if not isinstance(exp, Column): return exp return replace(exp, column_name=f"{alias_prefix}{exp.column_name}")
def mangle_aliases(exp: Expression) -> Expression: alias = exp.alias if alias is not None: return replace(exp, alias=f"{alias_prefix}{alias}") return exp
def command_args(self, args: ty.Dict): self.inputs = dc.replace(self.inputs, **args)
def main(): parser = HfArgumentParser((ModelArguments, DynamicDataTrainingArguments, DynamicTrainingArguments)) if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): # If we pass only one argument to the script and it's the path to a json file, # let's parse it to get our arguments. model_args, data_args, training_args = parser.parse_json_file( json_file=os.path.abspath(sys.argv[1])) else: model_args, data_args, training_args = parser.parse_args_into_dataclasses( ) if 'prompt' in model_args.few_shot_type: data_args.prompt = True if training_args.no_train: training_args.do_train = False if training_args.no_predict: training_args.do_predict = False # Setup logging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN, ) # Load prompt/template/mapping file if data_args.prompt: if data_args.prompt_path is not None: assert data_args.prompt_id is not None prompt_list = [] with open(data_args.prompt_path) as f: for line in f: line = line.strip() template, mapping = line.split('\t') prompt_list.append((template, mapping)) data_args.template, data_args.mapping = prompt_list[ data_args.prompt_id] logger.info( "Specify load the %d-th prompt: %s | %s" % (data_args.prompt_id, data_args.template, data_args.mapping)) else: if data_args.template_path is not None: with open(data_args.template_path) as f: data_args.template_list = [] for line in f: line = line.strip() if len(line) > 0: data_args.template_list.append(line) # Load top-n templates if data_args.top_n_template is not None: data_args.template_list = data_args.template_list[: data_args . top_n_template] logger.info( "Load top-%d templates from %s" % (len(data_args.template_list), data_args.template_path)) # ... or load i-th template if data_args.template_id is not None: data_args.template = data_args.template_list[ data_args.template_id] data_args.template_list = None logger.info("Specify load the %d-th template: %s" % (data_args.template_id, data_args.template)) if data_args.mapping_path is not None: assert data_args.mapping_id is not None # Only can use one label word mapping with open(data_args.mapping_path) as f: mapping_list = [] for line in f: line = line.strip() mapping_list.append(line) data_args.mapping = mapping_list[data_args.mapping_id] logger.info("Specify using the %d-th mapping: %s" % (data_args.mapping_id, data_args.mapping)) # Check save path if (os.path.exists(training_args.output_dir) and os.listdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir): raise ValueError( f"Output directory ({training_args.output_dir}) already exists.") logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", training_args.local_rank, training_args.device, training_args.n_gpu, bool(training_args.local_rank != -1), training_args.fp16, ) logger.info("Training/evaluation parameters %s", training_args) # Set seed set_seed(training_args.seed) try: num_labels = num_labels_mapping[data_args.task_name] output_mode = output_modes_mapping[data_args.task_name] logger.info( "Task name: {}, number of labels: {}, output mode: {}".format( data_args.task_name, num_labels, output_mode)) except KeyError: raise ValueError("Task not found: %s" % (data_args.task_name)) # Automatically generate template for using demonstrations if data_args.auto_demo and model_args.few_shot_type == 'prompt-demo': # GPT-3's in-context learning if data_args.gpt3_in_context_head or data_args.gpt3_in_context_tail: logger.info( "Automatically convert the template to GPT-3's in-context learning." ) assert data_args.template_list is None old_template = data_args.template new_template = old_template + '' old_template = old_template.replace('*cls*', '') # Single sentence or sentence pair? sent_num = 1 if "_1" in old_template: sent_num = 2 for instance_id in range(data_args.gpt3_in_context_num): sub_template = old_template + '' # Replace sent_id for sent_id in range(sent_num): sub_template = sub_template.replace( "_{}*".format(sent_id), "_{}*".format(sent_num + sent_num * instance_id + sent_id)) # Replace mask sub_template = sub_template.replace( "*mask*", "*labelx_{}*".format(instance_id)) if data_args.gpt3_in_context_tail: new_template = new_template + sub_template # Put context at the end else: new_template = sub_template + new_template # Put context at the beginning logger.info("| {} => {}".format(data_args.template, new_template)) data_args.template = new_template else: logger.info( "Automatically convert the template to using demonstrations.") if data_args.template_list is not None: for i in range(len(data_args.template_list)): old_template = data_args.template_list[i] new_template = old_template + '' old_template = old_template.replace('*cls*', '') # Single sentence or sentence pair? sent_num = 1 if "_1" in old_template: sent_num = 2 for label_id in range(num_labels): sub_template = old_template + '' # Replace sent id for sent_id in range(sent_num): sub_template = sub_template.replace( "_{}*".format(sent_id), "_{}*".format(sent_num + sent_num * label_id + sent_id)) # Replace mask sub_template = sub_template.replace( "*mask*", "*label_{}*".format(label_id)) new_template = new_template + sub_template logger.info("| {} => {}".format(data_args.template_list[i], new_template)) data_args.template_list[i] = new_template else: old_template = data_args.template new_template = old_template + '' old_template = old_template.replace('*cls*', '') # Single sentence or sentence pair? sent_num = 1 if "_1" in old_template: sent_num = 2 for label_id in range(num_labels): sub_template = old_template + '' # Replace sent id for sent_id in range(sent_num): sub_template = sub_template.replace( "_{}".format(sent_id), "_{}".format(sent_num + sent_num * label_id + sent_id)) # Replace mask sub_template = sub_template.replace( "*mask*", "*label_{}*".format(label_id)) new_template = new_template + sub_template logger.info("| {} => {}".format(data_args.template, new_template)) data_args.template = new_template # Create config config = AutoConfig.from_pretrained( model_args.config_name if model_args.config_name else model_args.model_name_or_path, num_labels=num_labels, finetuning_task=data_args.task_name, cache_dir=model_args.cache_dir, ) if 'prompt' in model_args.few_shot_type: if config.model_type == 'roberta': model_fn = RobertaForPromptFinetuning elif config.model_type == 'bert': model_fn = BertForPromptFinetuning else: raise NotImplementedError elif model_args.few_shot_type == 'finetune': model_fn = AutoModelForSequenceClassification else: raise NotImplementedError special_tokens = [] # Create tokenizer tokenizer = AutoTokenizer.from_pretrained( model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, additional_special_tokens=special_tokens, cache_dir=model_args.cache_dir, ) # Get our special datasets. train_dataset = (FewShotDataset(data_args, tokenizer=tokenizer, mode="train", use_demo=("demo" in model_args.few_shot_type))) eval_dataset = (FewShotDataset(data_args, tokenizer=tokenizer, mode="dev", use_demo=("demo" in model_args.few_shot_type)) if training_args.do_eval else None) test_dataset = (FewShotDataset(data_args, tokenizer=tokenizer, mode="test", use_demo=("demo" in model_args.few_shot_type)) if training_args.do_predict else None) set_seed(training_args.seed) model = model_fn.from_pretrained( model_args.model_name_or_path, from_tf=bool(".ckpt" in model_args.model_name_or_path), config=config, cache_dir=model_args.cache_dir, ) # For BERT, increase the size of the segment (token type) embeddings if config.model_type == 'bert': model.resize_token_embeddings(len(tokenizer)) resize_token_type_embeddings(model, new_num_types=10, random_segment=model_args.random_segment) # Pass dataset and argument information to the model if data_args.prompt: model.label_word_list = torch.tensor( train_dataset.label_word_list).long().cuda() if output_modes_mapping[data_args.task_name] == 'regression': # lower / upper bounds model.lb, model.ub = bound_mapping[data_args.task_name] model.model_args = model_args model.data_args = data_args model.tokenizer = tokenizer # Build metric def build_compute_metrics_fn( task_name: str) -> Callable[[EvalPrediction], Dict]: def compute_metrics_fn(p: EvalPrediction): # Note: the eval dataloader is sequential, so the examples are in order. # We average the logits over each sample for using demonstrations. predictions = p.predictions num_logits = predictions.shape[-1] logits = predictions.reshape( [eval_dataset.num_sample, -1, num_logits]) logits = logits.mean(axis=0) if num_logits == 1: preds = np.squeeze(logits) else: preds = np.argmax(logits, axis=1) # Just for sanity, assert label ids are the same. label_ids = p.label_ids.reshape([eval_dataset.num_sample, -1]) label_ids_avg = label_ids.mean(axis=0) label_ids_avg = label_ids_avg.astype(p.label_ids.dtype) assert (label_ids_avg - label_ids[0]).mean() < 1e-2 label_ids = label_ids[0] return compute_metrics_mapping[task_name](task_name, preds, label_ids) return compute_metrics_fn # Initialize our Trainer trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=build_compute_metrics_fn( data_args.task_name)) # Training if training_args.do_train: trainer.train(model_path=model_args.model_name_or_path if os.path. isdir(model_args.model_name_or_path) else None) # Use the early stop, so do not save the model in the end (unless specify save_at_last) if training_args.save_at_last: trainer.save_model(training_args.output_dir) if trainer.is_world_master(): tokenizer.save_pretrained(training_args.output_dir) torch.save( model_args, os.path.join(training_args.output_dir, "model_args.bin")) torch.save(data_args, os.path.join(training_args.output_dir, "data_args.bin")) # Reload the best checkpoint (for eval) model = model_fn.from_pretrained(training_args.output_dir) model = model.to(training_args.device) trainer.model = model if data_args.prompt: model.label_word_list = torch.tensor( train_dataset.label_word_list).long().cuda() if output_modes_mapping[data_args.task_name] == 'regression': # lower / upper bounds model.lb, model.ub = bound_mapping[data_args.task_name] model.model_args = model_args model.data_args = data_args model.tokenizer = tokenizer # Evaluation final_result = { 'time': str(datetime.today()), } eval_results = {} if training_args.do_eval: logger.info("*** Validate ***") eval_datasets = [eval_dataset] for eval_dataset in eval_datasets: trainer.compute_metrics = build_compute_metrics_fn( eval_dataset.args.task_name) output = trainer.evaluate(eval_dataset=eval_dataset) eval_result = output.metrics output_eval_file = os.path.join( training_args.output_dir, f"eval_results_{eval_dataset.args.task_name}.txt") if trainer.is_world_master(): with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format( eval_dataset.args.task_name)) for key, value in eval_result.items(): logger.info(" %s = %s", key, value) writer.write("%s = %s\n" % (key, value)) final_result[eval_dataset.args.task_name + '_dev_' + key] = value eval_results.update(eval_result) test_results = {} if training_args.do_predict: logging.info("*** Test ***") test_datasets = [test_dataset] if data_args.task_name == "mnli": mnli_mm_data_args = dataclasses.replace(data_args, task_name="mnli-mm") test_datasets.append( FewShotDataset(mnli_mm_data_args, tokenizer=tokenizer, mode="test", use_demo=('demo' in model_args.few_shot_type))) for test_dataset in test_datasets: trainer.compute_metrics = build_compute_metrics_fn( test_dataset.args.task_name) output = trainer.evaluate(eval_dataset=test_dataset) test_result = output.metrics output_test_file = os.path.join( training_args.output_dir, f"test_results_{test_dataset.args.task_name}.txt") if trainer.is_world_master(): with open(output_test_file, "w") as writer: logger.info("***** Test results {} *****".format( test_dataset.args.task_name)) for key, value in test_result.items(): logger.info(" %s = %s", key, value) writer.write("%s = %s\n" % (key, value)) final_result[test_dataset.args.task_name + '_test_' + key] = value if training_args.save_logit: predictions = output.predictions num_logits = predictions.shape[-1] logits = predictions.reshape( [test_dataset.num_sample, -1, num_logits]).mean(axis=0) np.save( os.path.join( training_args.save_logit_dir, "{}-{}-{}.npy".format(test_dataset.task_name, training_args.model_id, training_args.array_id)), logits) test_results.update(test_result) with FileLock('log.lock'): with open('log', 'a') as f: final_result.update(vars(model_args)) final_result.update(vars(training_args)) final_result.update(vars(data_args)) if 'evaluation_strategy' in final_result: final_result.pop('evaluation_strategy') f.write(str(final_result) + '\n') return eval_results
def analyze_calltree(fn: Callable, options: AnalysisOptions, conditions: Conditions) -> CallTreeAnalysis: debug('Begin analyze calltree ', fn.__name__) all_messages = MessageCollector() search_root = SinglePathNode(True) space_exhausted = False failing_precondition: Optional[ ConditionExpr] = conditions.pre[0] if conditions.pre else None failing_precondition_reason: str = '' num_confirmed_paths = 0 cur_space: List[StateSpace] = [cast(StateSpace, None)] short_circuit = ShortCircuitingContext(lambda: cur_space[0]) _ = get_subclass_map() # ensure loaded top_analysis: Optional[CallAnalysis] = None enforced_conditions = EnforcedConditions( fn_globals(fn), builtin_patches(), interceptor=short_circuit.make_interceptor) def in_symbolic_mode(): return (cur_space[0] is not None and not cur_space[0].running_framework_code) patched_builtins = Patched(_PATCH_REGISTRATIONS, in_symbolic_mode) with enforced_conditions, patched_builtins, enforced_conditions.disabled_enforcement( ): for i in itertools.count(1): start = time.time() if start > options.deadline: debug('Exceeded condition timeout, stopping') break options.incr('num_paths') debug('Iteration ', i) space = TrackingStateSpace( execution_deadline=start + options.per_path_timeout, model_check_timeout=options.per_path_timeout / 2, search_root=search_root) cur_space[0] = space try: # The real work happens here!: call_analysis = attempt_call(conditions, space, fn, short_circuit, enforced_conditions) if failing_precondition is not None: cur_precondition = call_analysis.failing_precondition if cur_precondition is None: if call_analysis.verification_status is not None: # We escaped the all the pre conditions on this try: failing_precondition = None elif (cur_precondition.line == failing_precondition.line and call_analysis.failing_precondition_reason): failing_precondition_reason = call_analysis.failing_precondition_reason elif cur_precondition.line > failing_precondition.line: failing_precondition = cur_precondition failing_precondition_reason = call_analysis.failing_precondition_reason except UnexploredPath: call_analysis = CallAnalysis(VerificationStatus.UNKNOWN) except IgnoreAttempt: call_analysis = CallAnalysis() status = call_analysis.verification_status if status == VerificationStatus.CONFIRMED: num_confirmed_paths += 1 top_analysis, space_exhausted = space.bubble_status(call_analysis) overall_status = top_analysis.verification_status if top_analysis else None debug('Iter complete. Worst status found so far:', overall_status.name if overall_status else 'None') if space_exhausted or top_analysis == VerificationStatus.REFUTED: break top_analysis = search_root.child.get_result() if top_analysis.messages: #log = space.execution_log() all_messages.extend( replace( m, #execution_log=log, test_fn=fn.__qualname__, condition_src=conditions.post[0].expr_source) for m in top_analysis.messages) if top_analysis.verification_status is None: top_analysis.verification_status = VerificationStatus.UNKNOWN if failing_precondition: assert num_confirmed_paths == 0 addl_ctx = ' ' + failing_precondition.addl_context if failing_precondition.addl_context else '' message = f'Unable to meet precondition {addl_ctx}' if failing_precondition_reason: message += f' (possibly because {failing_precondition_reason}?)' all_messages.extend([ AnalysisMessage(MessageType.PRE_UNSAT, message, failing_precondition.filename, failing_precondition.line, 0, '') ]) top_analysis = CallAnalysis(VerificationStatus.REFUTED) assert top_analysis.verification_status is not None debug( ('Exhausted' if space_exhausted else 'Aborted'), ' calltree search with', top_analysis.verification_status.name, 'and', len(all_messages.get()), 'messages.', 'Number of iterations: ', i) return CallTreeAnalysis( messages=all_messages.get(), verification_status=top_analysis.verification_status, num_confirmed_paths=num_confirmed_paths)
def visit_lambda(self, exp: Lambda) -> Expression: self.__level += 1 res = replace(exp, transformation=exp.transformation.accept(self)) self.__level -= 1 return res
def transform_state( cls, state: MaskProjectTuple, new_roi_info: ROIInfo, new_roi_extraction_parameters: typing.Dict[ int, typing.Optional[ROIExtractionProfile]], list_of_components: typing.List[int], save_chosen: bool = True, ) -> MaskProjectTuple: """ :param MaskProjectTuple state: state to be transformed :param ROIInfo new_roi_info: roi description :param typing.Dict[int, typing.Optional[ROIExtractionProfile]] new_roi_extraction_parameters: Parameters used to extract roi :param typing.List[int] list_of_components: list of components from new_roi which should be selected :param bool save_chosen: if save currently selected components :return: new state """ # TODO Refactor if not save_chosen or state.roi_info.roi is None or len( state.selected_components) == 0: return dataclasses.replace( state, roi_info=new_roi_info, selected_components=list_of_components, roi_extraction_parameters={ i: new_roi_extraction_parameters[i] for i in new_roi_info.bound_info }, ) if list_of_components is None: list_of_components = [] if new_roi_extraction_parameters is None: new_roi_extraction_parameters = defaultdict(lambda: None) segmentation_count = len(state.roi_info.bound_info) new_segmentation_count = len(new_roi_info.bound_info) segmentation_dtype = minimal_dtype(segmentation_count + new_segmentation_count) roi_base = reduce_array(state.roi_info.roi, state.selected_components, dtype=segmentation_dtype) annotation_base = { i: state.roi_info.annotations.get(x) for i, x in enumerate(state.selected_components, start=1) } alternative_base = { name: cls._clip_data_array(roi_base, array) for name, array in state.roi_info.alternative.items() } components_parameters_dict = { i: state.roi_extraction_parameters[val] for i, val in enumerate(sorted(state.selected_components), 1) } base_chose = list(annotation_base.keys()) if new_segmentation_count == 0: return dataclasses.replace( state, roi_info=ROIInfo(roi=roi_base, annotations=annotation_base, alternative=alternative_base), selected_components=base_chose, roi_extraction_parameters=components_parameters_dict, ) new_segmentation = np.copy(new_roi_info.roi) new_segmentation[roi_base > 0] = 0 left_component_list = np.unique(new_segmentation.flat) if left_component_list[0] == 0: left_component_list = left_component_list[1:] new_segmentation = reduce_array(new_segmentation, dtype=segmentation_dtype) roi_base[new_segmentation > 0] = new_segmentation[new_segmentation > 0] + len(base_chose) for name, array in new_roi_info.alternative.items(): if name in alternative_base: alternative_base[name][new_segmentation > 0] = array[ new_segmentation > 0] else: alternative_base[name] = cls._clip_data_array( new_segmentation, array) for i, el in enumerate(left_component_list, start=len(base_chose) + 1): annotation_base[i] = new_roi_info.annotations.get(el) if el in list_of_components: base_chose.append(i) components_parameters_dict[i] = new_roi_extraction_parameters[el] roi_info = ROIInfo(roi=roi_base, annotations=annotation_base, alternative=alternative_base) return dataclasses.replace( state, roi_info=roi_info, selected_components=base_chose, roi_extraction_parameters=components_parameters_dict, )
def fit(self, X: torch.Tensor, Y: torch.Tensor, Xts: Optional[torch.Tensor] = None, Yts: Optional[torch.Tensor] = None): """Fits the Falkon KRR model. Parameters ----------- X : torch.Tensor The tensor of training data, of shape [num_samples, num_dimensions]. If X is in Fortran order (i.e. column-contiguous) then we can avoid an extra copy of the data. Y : torch.Tensor The tensor of training targets, of shape [num_samples, num_outputs]. If X and Y represent a classification problem, Y can be encoded as a one-hot vector. If Y is in Fortran order (i.e. column-contiguous) then we can avoid an extra copy of the data. Xts : torch.Tensor or None Tensor of validation data, of shape [num_test_samples, num_dimensions]. If validation data is provided and `error_fn` was specified when creating the model, they will be used to print the validation error during the optimization iterations. If Xts is in Fortran order (i.e. column-contiguous) then we can avoid an extra copy of the data. Yts : torch.Tensor or None Tensor of validation targets, of shape [num_test_samples, num_outputs]. If validation data is provided and `error_fn` was specified when creating the model, they will be used to print the validation error during the optimization iterations. If Yts is in Fortran order (i.e. column-contiguous) then we can avoid an extra copy of the data. Returns -------- model: Falkon The fitted model """ X, Y, Xts, Yts = self._check_fit_inputs(X, Y, Xts, Yts) dtype = X.dtype # Decide whether to use CUDA for preconditioning based on M _use_cuda_preconditioner = ( self.use_cuda_ and (not self.options.cpu_preconditioner) and self.M >= get_min_cuda_preconditioner_size(dtype, self.options) ) _use_cuda_mmv = ( self.use_cuda_ and X.shape[0] * X.shape[1] * self.M / self.num_gpus >= get_min_cuda_mmv_size(dtype, self.options) ) self.fit_times_ = [] self.ny_points_ = None self.alpha_ = None t_s = time.time() # noinspection PyTypeChecker ny_points: Union[torch.Tensor, falkon.sparse.SparseTensor] = self.center_selection.select(X, None, self.M) if self.use_cuda_: ny_points = ny_points.pin_memory() with TicToc("Calcuating Preconditioner of size %d" % (self.M), debug=self.options.debug): pc_opt: FalkonOptions = dataclasses.replace(self.options, use_cpu=not _use_cuda_preconditioner) if pc_opt.debug: print("Preconditioner will run on %s" % ("CPU" if pc_opt.use_cpu else ("%d GPUs" % self.num_gpus))) precond = falkon.preconditioner.FalkonPreconditioner(self.penalty, self.kernel, pc_opt) precond.init(ny_points) if _use_cuda_mmv: # Cache must be emptied to ensure enough memory is visible to the optimizer torch.cuda.empty_cache() X = X.pin_memory() # K_NM storage decision k_opt = dataclasses.replace(self.options, use_cpu=True) cpu_info = get_device_info(k_opt) available_ram = min(k_opt.max_cpu_mem, cpu_info[-1].free_memory) * 0.9 if self._can_store_knm(X, ny_points, available_ram): Knm = self.kernel(X, ny_points, opt=self.options) else: Knm = None self.fit_times_.append(time.time() - t_s) # Preparation time # Here we define the callback function which will run at the end # of conjugate gradient iterations. This function computes and # displays the validation error. validation_cback = None if self.error_fn is not None and self.error_every is not None: validation_cback = self._get_callback_fn(X, Y, Xts, Yts, ny_points, precond) # Start with the falkon algorithm with TicToc('Computing Falkon iterations', debug=self.options.debug): o_opt: FalkonOptions = dataclasses.replace(self.options, use_cpu=not _use_cuda_mmv) if o_opt.debug: print("Optimizer will run on %s" % ("CPU" if o_opt.use_cpu else ("%d GPUs" % self.num_gpus)), flush=True) optim = falkon.optim.FalkonConjugateGradient(self.kernel, precond, o_opt) if Knm is not None: beta = optim.solve( Knm, None, Y, self.penalty, initial_solution=None, max_iter=self.maxiter, callback=validation_cback) else: beta = optim.solve( X, ny_points, Y, self.penalty, initial_solution=None, max_iter=self.maxiter, callback=validation_cback) self.alpha_ = precond.apply(beta) self.ny_points_ = ny_points return self
def update_cmd(cmd): cmd = replace_command_sender(cmd, address=new_sender_address) return dataclasses.replace(cmd, my_actor_address=new_sender_address)
def process_msg_and_check(self, message: Message) -> bool: """ Returns True if message can be processed successfully, false if a rate limit is passed. """ current_minute = int(time.time() // self.reset_seconds) if current_minute != self.current_minute: self.current_minute = current_minute self.message_counts = Counter() self.message_cumulative_sizes = Counter() self.non_tx_message_counts = 0 self.non_tx_cumulative_size = 0 try: message_type = ProtocolMessageTypes(message.type) except Exception as e: log.warning(f"Invalid message: {message.type}, {e}") return True new_message_counts: int = self.message_counts[message_type] + 1 new_cumulative_size: int = self.message_cumulative_sizes[message_type] + len(message.data) new_non_tx_count: int = self.non_tx_message_counts new_non_tx_size: int = self.non_tx_cumulative_size proportion_of_limit: float = self.percentage_of_limit / 100 ret: bool = False try: limits = DEFAULT_SETTINGS if message_type in rate_limits_tx: limits = rate_limits_tx[message_type] elif message_type in rate_limits_other: limits = rate_limits_other[message_type] new_non_tx_count = self.non_tx_message_counts + 1 new_non_tx_size = self.non_tx_cumulative_size + len(message.data) if new_non_tx_count > NON_TX_FREQ * proportion_of_limit: return False if new_non_tx_size > NON_TX_MAX_TOTAL_SIZE * proportion_of_limit: return False else: log.warning(f"Message type {message_type} not found in rate limits") if limits.max_total_size is None: limits = dataclasses.replace(limits, max_total_size=limits.frequency * limits.max_size) assert limits.max_total_size is not None if new_message_counts > limits.frequency * proportion_of_limit: return False if len(message.data) > limits.max_size: return False if new_cumulative_size > limits.max_total_size * proportion_of_limit: return False ret = True return True finally: if self.incoming or ret: # now that we determined that it's OK to send the message, commit the # updates to the counters. Alternatively, if this was an # incoming message, we already received it and it should # increment the counters unconditionally self.message_counts[message_type] = new_message_counts self.message_cumulative_sizes[message_type] = new_cumulative_size self.non_tx_message_counts = new_non_tx_count self.non_tx_cumulative_size = new_non_tx_size
def replace_actor(cmd, name, actor, **changes): new_actor = dataclasses.replace(actor, **changes) return replace_command_payment(cmd, **{name: new_actor})
def register_spec_with_docutils(spec: specparser.Spec, default_domain: Optional[str]) -> Registry: """Register all of the definitions in the spec with docutils, overwriting the previous call to this function. This function should only be called once in the process lifecycle.""" builder = Registry.Builder() directives = list(spec.directive.items()) roles = list(spec.role.items()) # Define rstobjects for name, rst_object in spec.rstobject.items(): directive = rst_object.create_directive() directives.append((name, directive)) role = rst_object.create_role() roles.append((name, role)) for name, directive in directives: # Skip abstract base directives if name.startswith("_"): continue options: Dict[str, object] = { option_name: spec.get_validator(option) for option_name, option in directive.options.items() } base_class: Any = BaseDocutilsDirective # Tabs have special handling because of the need to support legacy syntax if name == "tabs": base_class = BaseTabsDirective elif name in SPECIAL_DIRECTIVE_HANDLERS: base_class = SPECIAL_DIRECTIVE_HANDLERS[name] DocutilsDirective = make_docutils_directive_handler( directive, base_class, name, options) builder.add_directive(name, DocutilsDirective) # reference tabs directive declaration as first step in registering tabs-* with docutils tabs_directive = spec.directive["tabs"] # Define tabsets for name in spec.tabs: tabs_base_class: Any = BaseTabsDirective tabs_name = "tabs-" + name # copy and modify the tabs directive to update its name to match the deprecated tabs-* naming convention modified_tabs_directive = dataclasses.replace(tabs_directive, name=tabs_name) tabs_options: Dict[str, object] = { option_name: spec.get_validator(option) for option_name, option in tabs_directive.options.items() } DocutilsDirective = make_docutils_directive_handler( modified_tabs_directive, tabs_base_class, "tabs", tabs_options) builder.add_directive(tabs_name, DocutilsDirective) # Docutils builtins builder.add_directive("unicode", docutils.parsers.rst.directives.misc.Unicode) builder.add_directive("replace", docutils.parsers.rst.directives.misc.Replace) # Define roles builder.add_role("", handle_role_null) for name, role_spec in roles: handler: Optional[RoleHandlerType] = None domain = role_spec.domain or "" if not role_spec.type or role_spec.type == specparser.PrimitiveRoleType.text: handler = TextRoleHandler(domain) elif isinstance(role_spec.type, specparser.LinkRoleType): handler = LinkRoleHandler( role_spec.type.link, role_spec.type.ensure_trailing_slash == True, role_spec.type.format, ) elif isinstance(role_spec.type, specparser.RefRoleType): handler = RefRoleHandler( role_spec.type.domain or domain, role_spec.type.name, role_spec.type.tag, role_spec.rstobject.type if role_spec.rstobject else specparser.TargetType.plain, role_spec.type.format, ) elif role_spec.type == specparser.PrimitiveRoleType.explicit_title: handler = ExplicitTitleRoleHandler(domain) if not handler: raise ValueError('Unknown role type "{}"'.format(role_spec.type)) builder.add_role(name, handler) return builder.build(default_domain)
def replace_command_payment(cmd, **changes): new_payment = dataclasses.replace(cmd.payment, **changes) return dataclasses.replace(cmd, payment=new_payment)
import dataclasses from typing import Type, Union class A: pass dataclasses.fields(<warning descr="'dataclasses.fields' method should be called on dataclass instances or types">A</warning>) dataclasses.fields(<warning descr="'dataclasses.fields' method should be called on dataclass instances or types">A()</warning>) dataclasses.asdict(<warning descr="'dataclasses.asdict' method should be called on dataclass instances">A()</warning>) dataclasses.astuple(<warning descr="'dataclasses.astuple' method should be called on dataclass instances">A()</warning>) dataclasses.replace(<warning descr="'dataclasses.replace' method should be called on dataclass instances">A()</warning>) @dataclasses.dataclass class B: pass dataclasses.fields(B) dataclasses.fields(B()) dataclasses.asdict(B()) dataclasses.astuple(B()) dataclasses.replace(B()) dataclasses.asdict(<warning descr="'dataclasses.asdict' method should be called on dataclass instances">B</warning>) dataclasses.astuple(<warning descr="'dataclasses.astuple' method should be called on dataclass instances">B</warning>) dataclasses.replace(<warning descr="'dataclasses.replace' method should be called on dataclass instances">B</warning>)
def union1(p: Union[A, B]): dataclasses.fields(p) dataclasses.asdict(p) dataclasses.astuple(p) dataclasses.replace(p)
def merge_schema(default: Schema, override: Schema) -> Schema: if override.override: return override return replace(override, child=merge_schema(default, override.child))
from dataclasses import dataclass, field, InitVar, replace @dataclass class A: a: int b: str = "str" replace(A(1), <arg1>) @dataclass class B: a: int b: str = field(default="str", init=False) replace(B(1), <arg2>) @dataclass class C: a: int b: InitVar[str] = "str" replace(C(1), <arg3>) class D:
def main(): # See all possible arguments in src/transformers/training_args.py # or by passing the --help flag to this script. # We now keep distinct sets of args, for a cleaner separation of concerns. parser = HfArgumentParser( (ModelArguments, DataTrainingArguments, TrainingArguments)) if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): # If we pass only one argument to the script and it's the path to a json file, # let's parse it to get our arguments. model_args, data_args, training_args = parser.parse_json_file( json_file=os.path.abspath(sys.argv[1])) else: model_args, data_args, training_args = parser.parse_args_into_dataclasses( ) if (os.path.exists(training_args.output_dir) and os.listdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir): raise ValueError( f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome." ) # Setup logging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN, ) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", training_args.local_rank, training_args.device, training_args.n_gpu, bool(training_args.local_rank != -1), training_args.fp16, ) logger.info("Training/evaluation parameters %s", training_args) # Set seed set_seed(training_args.seed) try: num_labels = glue_tasks_num_labels[data_args.task_name] output_mode = glue_output_modes[data_args.task_name] except KeyError: raise ValueError("Task not found: %s" % (data_args.task_name)) # Load pretrained model and tokenizer # # Distributed training: # The .from_pretrained methods guarantee that only one local process can concurrently # download model & vocab. config = AutoConfig.from_pretrained( model_args.config_name if model_args.config_name else model_args.model_name_or_path, num_labels=num_labels, finetuning_task=data_args.task_name, cache_dir=model_args.cache_dir, ) tokenizer = AutoTokenizer.from_pretrained( model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, ) model = AutoModelForSequenceClassification.from_pretrained( model_args.model_name_or_path, from_tf=bool(".ckpt" in model_args.model_name_or_path), config=config, cache_dir=model_args.cache_dir, ) # Get datasets train_dataset = GlueDataset( data_args, tokenizer=tokenizer) if training_args.do_train else None eval_dataset = GlueDataset( data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None def compute_metrics(p: EvalPrediction) -> Dict: if output_mode == "classification": preds = np.argmax(p.predictions, axis=1) elif output_mode == "regression": preds = np.squeeze(p.predictions) return glue_compute_metrics(data_args.task_name, preds, p.label_ids) # Initialize our Trainer trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=compute_metrics, ) # Training if training_args.do_train: trainer.train(model_path=model_args.model_name_or_path if os.path. isdir(model_args.model_name_or_path) else None) trainer.save_model() # For convenience, we also re-save the tokenizer to the same directory, # so that you can share your model easily on huggingface.co/models =) if trainer.is_world_master(): tokenizer.save_pretrained(training_args.output_dir) # Evaluation results = {} if training_args.do_eval and training_args.local_rank in [-1, 0]: logger.info("*** Evaluate ***") # Loop to handle MNLI double evaluation (matched, mis-matched) eval_datasets = [eval_dataset] if data_args.task_name == "mnli": mnli_mm_data_args = dataclasses.replace(data_args, task_name="mnli-mm") eval_datasets.append( GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, evaluate=True)) for eval_dataset in eval_datasets: #result = trainer.evaluate(eval_dataset=eval_dataset) predictions = trainer.predict( test_dataset=eval_dataset).predictions fil = open('/dccstor/tuhinstor/tuhin/output1.txt', 'w') for p in predictions: fil.write(str(p) + '\n') output_eval_file = os.path.join( training_args.output_dir, f"eval_results_{eval_dataset.args.task_name}.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format( eval_dataset.args.task_name)) for key, value in result.items(): logger.info(" %s = %s", key, value) writer.write("%s = %s\n" % (key, value)) results.update(result) return results
def transform(self, ray: Ray) -> Ray: new_ray = replace(ray, position=self.context.end_path) return new_ray
def make_scalar(self, new_value): return dc.replace(self, value=new_value)
def set_serialized_creds(self, serialized_creds): return dataclasses.replace(self, serialized_creds=serialized_creds)
def run( self, universal_newlines: Optional[bool] = None, *, capture_output: bool = False, check: bool = False, encoding: Optional[str] = None, errors: Optional[str] = None, input: Optional[Union[str, bytes]] = None, text: Optional[bool] = None, timeout: Optional[float] = None, **kwargs: Any, ) -> subprocess.CompletedProcess[Any]: r"""Run command in :func:`subprocess.run`, optionally overrides via kwargs. Parameters ---------- input : Union[bytes, str], optional pass string to subprocess's stdin. Bytes by default, str in text mode. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. check : bool If True and the exit code was non-zero, it raises a :exc:`subprocess.CalledProcessError`. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. timeout : int If given, and the process takes too long, a :exc:`subprocess.TimeoutExpired` **kwargs : dict, optional Overrides existing attributes for :func:`subprocess.run` Examples -------- >>> import subprocess >>> cmd = SubprocessCommand( ... ["/bin/sh", "-c", "ls -l non_existent_file ; exit 0"]) >>> cmd.run() CompletedProcess(args=['/bin/sh', '-c', 'ls -l non_existent_file ; exit 0'], returncode=0) >>> import subprocess >>> cmd = SubprocessCommand( ... ["/bin/sh", "-c", "ls -l non_existent_file ; exit 0"]) >>> cmd.run(check=True) CompletedProcess(args=['/bin/sh', '-c', 'ls -l non_existent_file ; exit 0'], returncode=0) >>> cmd = SubprocessCommand(["sed", "-e", "s/foo/bar/"]) >>> completed = cmd.run(input=b"when in the course of fooman events\n") >>> completed CompletedProcess(args=['sed', '-e', 's/foo/bar/'], returncode=0) >>> completed.stderr >>> cmd = SubprocessCommand(["sed", "-e", "s/foo/bar/"]) >>> completed = cmd.run(input=b"when in the course of fooman events\n", ... capture_output=True) >>> completed CompletedProcess(args=['sed', '-e', 's/foo/bar/'], returncode=0, stdout=b'when in the course of barman events\n', stderr=b'') >>> completed.stdout b'when in the course of barman events\n' >>> completed.stderr b'' """ return subprocess.run( **dataclasses.replace( self, universal_newlines=universal_newlines, errors=errors, text=text, **kwargs, ).__dict__, check=check, capture_output=capture_output, input=input, timeout=timeout, )
from dataclasses import dataclass, replace @dataclass class A1: a: int class B1(A1): b: str replace(B1(1), <arg1>) class A2: a: int @dataclass class B2(A2): b: str replace(B2("1"), <arg2>) @dataclass class A3: a: int class B3(A3): def __init__(self, b: str): self.a = 10 replace(B3("1"), <arg3>)
def merge_framework_metadata( exception_info: datamodel.ExceptionData, framework_infos: Sequence[datamodel.FrameworkMetadata], ) -> datamodel.FrameworkMetadata: """ Return the merger between the *frameworks* infos and *exception_info*. This will raise an exception when the information cannot be merged. """ result = datamodel.FrameworkMetadata() for info in framework_infos: result.architectures.update(info.architectures) # enum_type result = replace( result, enum_type=merge_enum_type( exception_info.enum_type, [(next(iter(info.architectures)), info.enum_type) for info in framework_infos], ), ) # enum result = replace( result, enum=merge_enum( exception_info.enum, [(next(iter(info.architectures)), info.enum) for info in framework_infos], ), ) # structs ... # externs result = replace( result, externs=merge_externs( exception_info.externs, [(next(iter(info.architectures)), info.externs) for info in framework_infos], ), ) # cftypes ... # literals result = replace( result, literals=merge_literals( exception_info.literals, [(next(iter(info.architectures)), info.literals) for info in framework_infos], ), ) # formal_protocols ... # informal_protocols ... # classes ... # aliases result = replace( result, aliases=merge_aliases( exception_info.aliases, [(next(iter(info.architectures)), info.aliases) for info in framework_infos], ), ) # expressions result = replace( result, aliases=merge_expressions( exception_info.expressions, [(next(iter(info.architectures)), info.expressions) for info in framework_infos], ), ) # func_macros result = replace( result, aliases=merge_func_macros( exception_info.func_macros, [(next(iter(info.architectures)), info.func_macros) for info in framework_infos], ), ) # functions ... return result
from dataclasses import dataclass, field, InitVar, replace @dataclass class A: a: int b: str = "str" replace(A(1)) replace(A(1), a=1) replace(A(1), a=1, b="abc") replace(A(1), a=1, b="abc", <warning descr="Unexpected argument">c=2</warning>) @dataclass class B: a: int b: str = field(default="str", init=False) replace(B(1)) replace(B(1), a=1) replace(B(1), a=1, <warning descr="Unexpected argument">b="abc"</warning>) replace(B(1), a=1, <warning descr="Unexpected argument">b="abc"</warning>, <warning descr="Unexpected argument">c=2</warning>) @dataclass class C: a: int b: InitVar[str] = "str"
@dataclasses.dataclass class Base: pass class A(Base): pass dataclasses.fields(A) dataclasses.fields(A()) dataclasses.asdict(<warning descr="'dataclasses.asdict' method should be called on dataclass instances">A()</warning>) dataclasses.astuple(A()) dataclasses.replace(A()) @dataclasses.dataclass class B(Base): pass dataclasses.fields(B) dataclasses.fields(B()) dataclasses.asdict(B()) dataclasses.astuple(B()) dataclasses.replace(B()) dataclasses.asdict(<warning descr="'dataclasses.asdict' method should be called on dataclass instances">B</warning>)
def copy(self): return replace(self)
async def set_name(self, new_name: str): new_info = replace(self.wallet_info, name=new_name) self.wallet_info = new_info await self.wallet_state_manager.user_store.update_wallet( self.wallet_info)