def get(self, idx=-1, suffix=tuple()):
"""
get obj from other parties.
Args:
idx: id of party to get from.
The default is -1, which means get values from parties regardless their party id
suffix: additional tag suffix, the default is tuple()
Returns:
object or list of object
"""
src_parties = get_latest_opened().parties.roles_to_parties(
roles=self._src, strict=False)
if isinstance(idx, list):
rtn = self.get_parties(parties=[src_parties[i] for i in idx],
suffix=suffix)
elif isinstance(idx, int):
rtn = self.get_parties(parties=src_parties, suffix=suffix) if idx < 0 else \
self.get_parties(parties=src_parties[idx], suffix=suffix)[0]
else:
raise ValueError(
f"illegal idx type: {type(idx)}, supported types: int or list of int"
)
return rtn
def remote(self, obj, role=None, idx=-1, suffix=tuple()):
"""
send obj to other parties.
Args:
obj: object to be sent
role: role of parties to sent to, use one of ['Host', 'Guest', 'Arbiter', None].
The default is None, means sent values to parties regardless their party role
idx: id of party to sent to.
The default is -1, which means sent values to parties regardless their party id
suffix: additional tag suffix, the default is tuple()
"""
party_info = get_latest_opened().parties
if idx >= 0 and role is None:
raise ValueError("role cannot be None if idx specified")
# get subset of dst roles in runtime conf
if role is None:
parties = party_info.roles_to_parties(self._dst, strict=False)
else:
if isinstance(role, str):
role = [role]
parties = party_info.roles_to_parties(role)
if idx >= 0:
parties = parties[idx]
return self.remote_parties(obj=obj, parties=parties, suffix=suffix)
def local_party():
"""
indicate local party
Returns
-------
Party
party this program running on
"""
return get_latest_opened().parties.local_party
def all_parties():
"""
get all parties
Returns
-------
list
list of parties
"""
return get_latest_opened().parties.all_parties
def _set_parties(self):
# since multi-host not supported yet, we assume parties are one from guest and one from host
parties = []
guest_parties = session.get_latest_opened().parties.roles_to_parties(
["guest"])
host_parties = session.get_latest_opened().parties.roles_to_parties(
["host"])
if len(guest_parties) != 1 or len(host_parties) != 1:
raise ValueError(
f"one guest and one host required, "
f"while {len(guest_parties)} guest and {len(host_parties)} host provided"
)
parties.extend(guest_parties)
parties.extend(host_parties)
local_party = session.get_latest_opened().parties.local_party
other_party = parties[0] if parties[0] != local_party else parties[1]
self.parties = parties
self.local_party = local_party
self.other_party = other_party
def remote_parties(self,
obj,
parties: Union[typing.List[Party], Party],
suffix: Union[typing.Any, typing.Tuple] = tuple()):
"""
remote object to specified parties
Parameters
----------
obj: object or table
object or table to remote
parties: typing.List[Party]
parties to remote object/table to
suffix: str or tuple of str
suffix used to distinguish federation with in variable
Returns
-------
None
"""
session = get_latest_opened()
if isinstance(parties, Party):
parties = [parties]
if not isinstance(suffix, tuple):
suffix = (suffix, )
tag = FederationTagNamespace.generate_tag(*suffix)
for party in parties:
if party.role not in self._dst:
raise RuntimeError(
f"not allowed to remote object to {party} using {self._name}"
)
local = session.parties.local_party.role
if local not in self._src:
raise RuntimeError(
f"not allowed to remote object from {local} using {self._name}"
)
name = self._short_name if self._use_short_name else self._name
timer = profile.federation_remote_timer(name, self._name, tag, local,
parties)
session.federation.remote(v=obj,
name=name,
tag=tag,
parties=parties,
gc=self._remote_gc)
timer.done(session.federation)
self._remote_gc.gc()
def get_parties(self,
parties: Union[typing.List[Party], Party],
suffix: Union[typing.Any, typing.Tuple] = tuple()):
"""
get objects/tables from specified parties
Parameters
----------
parties: typing.List[Party]
parties to remote object/table to
suffix: str or tuple of str
suffix used to distinguish federation with in variable
Returns
-------
list
a list of objects/tables get from parties with same order of ``parties``
"""
session = get_latest_opened()
if not isinstance(parties, list):
parties = [parties]
if not isinstance(suffix, tuple):
suffix = (suffix, )
tag = FederationTagNamespace.generate_tag(*suffix)
for party in parties:
if party.role not in self._src:
raise RuntimeError(
f"not allowed to get object from {party} using {self._name}"
)
local = session.parties.local_party.role
if local not in self._dst:
raise RuntimeError(
f"not allowed to get object to {local} using {self._name}")
name = self._short_name if self._use_short_name else self._name
timer = profile.federation_get_timer(name, self._name, tag, local,
parties)
rtn = session.federation.get(name=name,
tag=tag,
parties=parties,
gc=self._get_gc)
timer.done(session.federation)
self._get_gc.gc()
return rtn
def kill():
session.get_latest_opened().computing.kill()
def stop():
session.get_latest_opened().computing.stop()
def clean_tables(namespace, regex_string='*'):
session.get_latest_opened().computing.cleanup(namespace=namespace,
name=regex_string)
def get_session_id():
return session.get_latest_opened().computing.session_id
def cleanup(name, namespace, *args, **kwargs):
if len(args) > 0 or len(kwargs) > 0:
LOGGER.warning(f"some args removed, please check! {args}, {kwargs}")
return session.get_latest_opened().computing.cleanup(name=name,
namespace=namespace)
def parallelize(data: Iterable,
partition,
include_key=False,
**kwargs) -> CTableABC:
return session.get_latest_opened().computing.parallelize(
data=data, partition=partition, include_key=include_key, **kwargs)
def get_parties(roles):
return session.get_latest_opened().parties.roles_to_parties(roles=roles)
def roles_to_parties(roles):
party_info = get_latest_opened().parties
return party_info.roles_to_parties(roles)
def get_task_run_args(cls,
job_id,
role,
party_id,
task_id,
task_version,
job_args,
job_parameters: RunParameters,
task_parameters: RunParameters,
input_dsl,
filter_type=None,
filter_attr=None,
get_input_table=False):
task_run_args = {}
input_table = {}
if 'idmapping' in role:
return {}
for input_type, input_detail in input_dsl.items():
if filter_type and input_type not in filter_type:
continue
if input_type == 'data':
this_type_args = task_run_args[input_type] = task_run_args.get(
input_type, {})
for data_type, data_list in input_detail.items():
data_dict = {}
for data_key in data_list:
data_key_item = data_key.split('.')
data_dict[data_key_item[0]] = {data_type: []}
for data_key in data_list:
data_key_item = data_key.split('.')
search_component_name, search_data_name = data_key_item[
0], data_key_item[1]
storage_table_meta = None
if search_component_name == 'args':
if job_args.get(
'data', {}).get(search_data_name).get(
'namespace', '') and job_args.get(
'data',
{}).get(search_data_name).get(
'name', ''):
storage_table_meta = storage.StorageTableMeta(
name=job_args['data'][search_data_name]
['name'],
namespace=job_args['data']
[search_data_name]['namespace'])
else:
tracker_client = TrackerClient(
job_id=job_id,
role=role,
party_id=party_id,
component_name=search_component_name)
upstream_output_table_infos_json = tracker_client.get_output_data_info(
data_name=search_data_name)
if upstream_output_table_infos_json:
tracker = Tracker(
job_id=job_id,
role=role,
party_id=party_id,
component_name=search_component_name)
upstream_output_table_infos = []
for _ in upstream_output_table_infos_json:
upstream_output_table_infos.append(
fill_db_model_object(
Tracker.get_dynamic_db_model(
TrackingOutputDataInfo,
job_id)(), _))
output_tables_meta = tracker.get_output_data_table(
output_data_infos=
upstream_output_table_infos)
if output_tables_meta:
storage_table_meta = output_tables_meta.get(
search_data_name, None)
args_from_component = this_type_args[
search_component_name] = this_type_args.get(
search_component_name, {})
if get_input_table and storage_table_meta:
input_table[data_key] = {
'namespace':
storage_table_meta.get_namespace(),
'name': storage_table_meta.get_name()
}
computing_table = None
elif storage_table_meta:
LOGGER.info(
f"load computing table use {task_parameters.computing_partitions}"
)
computing_table = session.get_latest_opened(
).computing.load(
storage_table_meta.get_address(),
schema=storage_table_meta.get_schema(),
partitions=task_parameters.computing_partitions
)
else:
computing_table = None
if not computing_table or not filter_attr or not filter_attr.get(
"data", None):
data_dict[search_component_name][data_type].append(
computing_table)
args_from_component[data_type] = data_dict[
search_component_name][data_type]
else:
args_from_component[data_type] = dict([
(a, getattr(computing_table,
"get_{}".format(a))())
for a in filter_attr["data"]
])
elif input_type in ['model', 'isometric_model']:
this_type_args = task_run_args[input_type] = task_run_args.get(
input_type, {})
for dsl_model_key in input_detail:
dsl_model_key_items = dsl_model_key.split('.')
if len(dsl_model_key_items) == 2:
search_component_name, search_model_alias = dsl_model_key_items[
0], dsl_model_key_items[1]
elif len(dsl_model_key_items
) == 3 and dsl_model_key_items[0] == 'pipeline':
search_component_name, search_model_alias = dsl_model_key_items[
1], dsl_model_key_items[2]
else:
raise Exception(
'get input {} failed'.format(input_type))
models = Tracker(
job_id=job_id,
role=role,
party_id=party_id,
component_name=search_component_name,
model_id=job_parameters.model_id,
model_version=job_parameters.model_version
).get_output_model(model_alias=search_model_alias)
this_type_args[search_component_name] = models
if get_input_table:
return input_table
return task_run_args
def table(name, namespace, **kwargs) -> CTableABC:
return session.get_latest_opened().computing.load(name=name,
namespace=namespace,
**kwargs)