def get(self, idx=-1, suffix=tuple()): """ get obj from other parties. Args: idx: id of party to get from. The default is -1, which means get values from parties regardless their party id suffix: additional tag suffix, the default is tuple() Returns: object or list of object """ src_parties = get_latest_opened().parties.roles_to_parties( roles=self._src, strict=False) if isinstance(idx, list): rtn = self.get_parties(parties=[src_parties[i] for i in idx], suffix=suffix) elif isinstance(idx, int): rtn = self.get_parties(parties=src_parties, suffix=suffix) if idx < 0 else \ self.get_parties(parties=src_parties[idx], suffix=suffix)[0] else: raise ValueError( f"illegal idx type: {type(idx)}, supported types: int or list of int" ) return rtn
def remote(self, obj, role=None, idx=-1, suffix=tuple()): """ send obj to other parties. Args: obj: object to be sent role: role of parties to sent to, use one of ['Host', 'Guest', 'Arbiter', None]. The default is None, means sent values to parties regardless their party role idx: id of party to sent to. The default is -1, which means sent values to parties regardless their party id suffix: additional tag suffix, the default is tuple() """ party_info = get_latest_opened().parties if idx >= 0 and role is None: raise ValueError("role cannot be None if idx specified") # get subset of dst roles in runtime conf if role is None: parties = party_info.roles_to_parties(self._dst, strict=False) else: if isinstance(role, str): role = [role] parties = party_info.roles_to_parties(role) if idx >= 0: parties = parties[idx] return self.remote_parties(obj=obj, parties=parties, suffix=suffix)
def local_party(): """ indicate local party Returns ------- Party party this program running on """ return get_latest_opened().parties.local_party
def all_parties(): """ get all parties Returns ------- list list of parties """ return get_latest_opened().parties.all_parties
def _set_parties(self): # since multi-host not supported yet, we assume parties are one from guest and one from host parties = [] guest_parties = session.get_latest_opened().parties.roles_to_parties( ["guest"]) host_parties = session.get_latest_opened().parties.roles_to_parties( ["host"]) if len(guest_parties) != 1 or len(host_parties) != 1: raise ValueError( f"one guest and one host required, " f"while {len(guest_parties)} guest and {len(host_parties)} host provided" ) parties.extend(guest_parties) parties.extend(host_parties) local_party = session.get_latest_opened().parties.local_party other_party = parties[0] if parties[0] != local_party else parties[1] self.parties = parties self.local_party = local_party self.other_party = other_party
def remote_parties(self, obj, parties: Union[typing.List[Party], Party], suffix: Union[typing.Any, typing.Tuple] = tuple()): """ remote object to specified parties Parameters ---------- obj: object or table object or table to remote parties: typing.List[Party] parties to remote object/table to suffix: str or tuple of str suffix used to distinguish federation with in variable Returns ------- None """ session = get_latest_opened() if isinstance(parties, Party): parties = [parties] if not isinstance(suffix, tuple): suffix = (suffix, ) tag = FederationTagNamespace.generate_tag(*suffix) for party in parties: if party.role not in self._dst: raise RuntimeError( f"not allowed to remote object to {party} using {self._name}" ) local = session.parties.local_party.role if local not in self._src: raise RuntimeError( f"not allowed to remote object from {local} using {self._name}" ) name = self._short_name if self._use_short_name else self._name timer = profile.federation_remote_timer(name, self._name, tag, local, parties) session.federation.remote(v=obj, name=name, tag=tag, parties=parties, gc=self._remote_gc) timer.done(session.federation) self._remote_gc.gc()
def get_parties(self, parties: Union[typing.List[Party], Party], suffix: Union[typing.Any, typing.Tuple] = tuple()): """ get objects/tables from specified parties Parameters ---------- parties: typing.List[Party] parties to remote object/table to suffix: str or tuple of str suffix used to distinguish federation with in variable Returns ------- list a list of objects/tables get from parties with same order of ``parties`` """ session = get_latest_opened() if not isinstance(parties, list): parties = [parties] if not isinstance(suffix, tuple): suffix = (suffix, ) tag = FederationTagNamespace.generate_tag(*suffix) for party in parties: if party.role not in self._src: raise RuntimeError( f"not allowed to get object from {party} using {self._name}" ) local = session.parties.local_party.role if local not in self._dst: raise RuntimeError( f"not allowed to get object to {local} using {self._name}") name = self._short_name if self._use_short_name else self._name timer = profile.federation_get_timer(name, self._name, tag, local, parties) rtn = session.federation.get(name=name, tag=tag, parties=parties, gc=self._get_gc) timer.done(session.federation) self._get_gc.gc() return rtn
def kill(): session.get_latest_opened().computing.kill()
def stop(): session.get_latest_opened().computing.stop()
def clean_tables(namespace, regex_string='*'): session.get_latest_opened().computing.cleanup(namespace=namespace, name=regex_string)
def get_session_id(): return session.get_latest_opened().computing.session_id
def cleanup(name, namespace, *args, **kwargs): if len(args) > 0 or len(kwargs) > 0: LOGGER.warning(f"some args removed, please check! {args}, {kwargs}") return session.get_latest_opened().computing.cleanup(name=name, namespace=namespace)
def parallelize(data: Iterable, partition, include_key=False, **kwargs) -> CTableABC: return session.get_latest_opened().computing.parallelize( data=data, partition=partition, include_key=include_key, **kwargs)
def get_parties(roles): return session.get_latest_opened().parties.roles_to_parties(roles=roles)
def roles_to_parties(roles): party_info = get_latest_opened().parties return party_info.roles_to_parties(roles)
def get_task_run_args(cls, job_id, role, party_id, task_id, task_version, job_args, job_parameters: RunParameters, task_parameters: RunParameters, input_dsl, filter_type=None, filter_attr=None, get_input_table=False): task_run_args = {} input_table = {} if 'idmapping' in role: return {} for input_type, input_detail in input_dsl.items(): if filter_type and input_type not in filter_type: continue if input_type == 'data': this_type_args = task_run_args[input_type] = task_run_args.get( input_type, {}) for data_type, data_list in input_detail.items(): data_dict = {} for data_key in data_list: data_key_item = data_key.split('.') data_dict[data_key_item[0]] = {data_type: []} for data_key in data_list: data_key_item = data_key.split('.') search_component_name, search_data_name = data_key_item[ 0], data_key_item[1] storage_table_meta = None if search_component_name == 'args': if job_args.get( 'data', {}).get(search_data_name).get( 'namespace', '') and job_args.get( 'data', {}).get(search_data_name).get( 'name', ''): storage_table_meta = storage.StorageTableMeta( name=job_args['data'][search_data_name] ['name'], namespace=job_args['data'] [search_data_name]['namespace']) else: tracker_client = TrackerClient( job_id=job_id, role=role, party_id=party_id, component_name=search_component_name) upstream_output_table_infos_json = tracker_client.get_output_data_info( data_name=search_data_name) if upstream_output_table_infos_json: tracker = Tracker( job_id=job_id, role=role, party_id=party_id, component_name=search_component_name) upstream_output_table_infos = [] for _ in upstream_output_table_infos_json: upstream_output_table_infos.append( fill_db_model_object( Tracker.get_dynamic_db_model( TrackingOutputDataInfo, job_id)(), _)) output_tables_meta = tracker.get_output_data_table( output_data_infos= upstream_output_table_infos) if output_tables_meta: storage_table_meta = output_tables_meta.get( search_data_name, None) args_from_component = this_type_args[ search_component_name] = this_type_args.get( search_component_name, {}) if get_input_table and storage_table_meta: input_table[data_key] = { 'namespace': storage_table_meta.get_namespace(), 'name': storage_table_meta.get_name() } computing_table = None elif storage_table_meta: LOGGER.info( f"load computing table use {task_parameters.computing_partitions}" ) computing_table = session.get_latest_opened( ).computing.load( storage_table_meta.get_address(), schema=storage_table_meta.get_schema(), partitions=task_parameters.computing_partitions ) else: computing_table = None if not computing_table or not filter_attr or not filter_attr.get( "data", None): data_dict[search_component_name][data_type].append( computing_table) args_from_component[data_type] = data_dict[ search_component_name][data_type] else: args_from_component[data_type] = dict([ (a, getattr(computing_table, "get_{}".format(a))()) for a in filter_attr["data"] ]) elif input_type in ['model', 'isometric_model']: this_type_args = task_run_args[input_type] = task_run_args.get( input_type, {}) for dsl_model_key in input_detail: dsl_model_key_items = dsl_model_key.split('.') if len(dsl_model_key_items) == 2: search_component_name, search_model_alias = dsl_model_key_items[ 0], dsl_model_key_items[1] elif len(dsl_model_key_items ) == 3 and dsl_model_key_items[0] == 'pipeline': search_component_name, search_model_alias = dsl_model_key_items[ 1], dsl_model_key_items[2] else: raise Exception( 'get input {} failed'.format(input_type)) models = Tracker( job_id=job_id, role=role, party_id=party_id, component_name=search_component_name, model_id=job_parameters.model_id, model_version=job_parameters.model_version ).get_output_model(model_alias=search_model_alias) this_type_args[search_component_name] = models if get_input_table: return input_table return task_run_args
def table(name, namespace, **kwargs) -> CTableABC: return session.get_latest_opened().computing.load(name=name, namespace=namespace, **kwargs)