def load(cls):
conf = file_utils.load_yaml_conf(FATE_FLOW_JOB_DEFAULT_CONFIG_PATH)
if not isinstance(conf, dict):
raise ValueError("invalid config file")
for k, v in conf.items():
if hasattr(cls, k):
setattr(cls, k, v)
else:
stat_logger.warning(f"job default config not supported {k}")
return cls.get_all()
def component_output_data_download():
request_data = request.json
output_data_table = get_component_output_data_table(task_data=request_data)
limit = request_data.get('limit', -1)
if not output_data_table:
return error_response(response_code=500, retmsg='no data')
if limit == 0:
return error_response(response_code=500, retmsg='limit is 0')
output_data_count = 0
have_data_label = False
output_tmp_dir = os.path.join(os.getcwd(),
'tmp/{}'.format(get_fate_uuid()))
output_file_path = '{}/output_%s'.format(output_tmp_dir)
output_data_file_path = output_file_path % 'data.csv'
os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True)
with open(output_data_file_path, 'w') as fw:
for k, v in output_data_table.collect():
data_line, have_data_label = get_component_output_data_line(
src_key=k, src_value=v)
fw.write('{}\n'.format(','.join(map(lambda x: str(x), data_line))))
output_data_count += 1
if output_data_count == limit:
break
if output_data_count:
# get meta
header = get_component_output_data_meta(
output_data_table=output_data_table,
have_data_label=have_data_label)
output_data_meta_file_path = output_file_path % 'data_meta.json'
with open(output_data_meta_file_path, 'w') as fw:
json.dump({'header': header}, fw, indent=4)
# tar
memory_file = io.BytesIO()
tar = tarfile.open(fileobj=memory_file, mode='w:gz')
tar.add(output_data_file_path,
os.path.relpath(output_data_file_path, output_tmp_dir))
tar.add(output_data_meta_file_path,
os.path.relpath(output_data_meta_file_path, output_tmp_dir))
tar.close()
memory_file.seek(0)
try:
shutil.rmtree(os.path.dirname(output_data_file_path))
except Exception as e:
# warning
stat_logger.warning(e)
tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(
request_data['job_id'], request_data['component_name'],
request_data['role'], request_data['party_id'])
return send_file(memory_file,
attachment_filename=tar_file_name,
as_attachment=True)
def get_proto_buffer_class(buffer_name):
module_path, base_import_path = provider_utils.get_provider_model_paths(RuntimeConfig.COMPONENT_PROVIDER)
exception = ModuleNotFoundError(f'no module named {buffer_name}')
for f in module_path.glob('*.py'):
try:
proto_module = importlib.import_module('.'.join([*base_import_path, f.stem]))
for name, obj in inspect.getmembers(proto_module):
if inspect.isclass(obj) and name == buffer_name:
return obj
except Exception as e:
exception = e
stat_logger.warning(e)
raise exception
def wait_child_process(signum, frame):
try:
while True:
child_pid, status = os.waitpid(-1, os.WNOHANG)
if child_pid == 0:
stat_logger.info('no child process was immediately available')
break
exitcode = status >> 8
stat_logger.info('child process %s exit with exitcode %s', child_pid, exitcode)
except OSError as e:
if e.errno == errno.ECHILD:
stat_logger.warning('current process has no existing unwaited-for child processes.')
else:
raise
def kill_process(process: psutil.Process = None,
pid: int = None,
expected_cmdline: list = None):
process = process if process is not None else get_process_instance(pid)
if process is None:
return
for child in process.children(recursive=True):
try:
if check_process(pid=child.pid):
child.kill()
except Exception as e:
stat_logger.warning(f"kill {child.pid} process failed",
exc_info=True)
if check_process(pid=process.pid, expected_cmdline=expected_cmdline):
process.kill()
def get_proto_buffer_class(buffer_name):
package_path = os.path.join(file_utils.get_project_base_directory(), 'federatedml', 'protobuf', 'generated')
package_python_path = 'federatedml.protobuf.generated'
for f in os.listdir(package_path):
if f.startswith('.'):
continue
try:
proto_module = importlib.import_module(package_python_path + '.' + f.rstrip('.py'))
for name, obj in inspect.getmembers(proto_module):
if inspect.isclass(obj) and name == buffer_name:
return obj
except Exception as e:
stat_logger.warning(e)
else:
return None
def register_engine(cls, engine_type, engine_info):
nodes = engine_info.get("nodes", 1)
cores = engine_info.get("cores_per_node", 0) * nodes
memory = engine_info.get("memory_per_node", 0) * nodes
engine_name = engine_info.get("engine", "UNKNOWN")
engine_address = engine_info.get("address", {})
filters = [EngineRegistry.f_engine_name == engine_name, EngineRegistry.f_engine_type == engine_type]
resources = EngineRegistry.select().where(*filters)
if resources:
resource = resources[0]
update_fields = {}
update_fields[EngineRegistry.f_engine_address] = engine_address
update_fields[EngineRegistry.f_cores] = cores
update_fields[EngineRegistry.f_memory] = memory
update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + (
cores - resource.f_cores)
update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + (
memory - resource.f_memory)
update_fields[EngineRegistry.f_nodes] = nodes
operate = EngineRegistry.update(update_fields).where(*filters)
update_status = operate.execute() > 0
if update_status:
stat_logger.info(f"update {engine_type} engine {engine_name} registration information")
else:
stat_logger.info(f"update {engine_type} engine {engine_name} registration information takes no effect")
else:
resource = EngineRegistry()
resource.f_create_time = base_utils.current_timestamp()
resource.f_engine_name = engine_name
resource.f_engine_type = engine_type
resource.f_engine_address = engine_address
resource.f_cores = cores
resource.f_memory = memory
resource.f_remaining_cores = cores
resource.f_remaining_memory = memory
resource.f_nodes = nodes
try:
resource.save(force_insert=True)
except Exception as e:
stat_logger.warning(e)
stat_logger.info(f"create {engine_type} engine {engine_name} registration information")
def register_engine(cls, engine_type, engine_name, engine_entrance, engine_config):
nodes = engine_config.get("nodes", 1)
cores = engine_config.get("cores_per_node", 0) * nodes * TOTAL_CORES_OVERWEIGHT_PERCENT
memory = engine_config.get("memory_per_node", 0) * nodes * TOTAL_MEMORY_OVERWEIGHT_PERCENT
filters = [EngineRegistry.f_engine_type == engine_type, EngineRegistry.f_engine_name == engine_name]
resources = EngineRegistry.select().where(*filters)
if resources:
resource = resources[0]
update_fields = {}
update_fields[EngineRegistry.f_engine_config] = engine_config
update_fields[EngineRegistry.f_cores] = cores
update_fields[EngineRegistry.f_memory] = memory
update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + (
cores - resource.f_cores)
update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + (
memory - resource.f_memory)
update_fields[EngineRegistry.f_nodes] = nodes
operate = EngineRegistry.update(update_fields).where(*filters)
update_status = operate.execute() > 0
if update_status:
stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information")
else:
stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information takes no effect")
else:
resource = EngineRegistry()
resource.f_create_time = base_utils.current_timestamp()
resource.f_engine_type = engine_type
resource.f_engine_name = engine_name
resource.f_engine_entrance = engine_entrance
resource.f_engine_config = engine_config
resource.f_cores = cores
resource.f_memory = memory
resource.f_remaining_cores = cores
resource.f_remaining_memory = memory
resource.f_nodes = nodes
try:
resource.save(force_insert=True)
except Exception as e:
stat_logger.warning(e)
stat_logger.info(f"create {engine_type} engine {engine_name} {engine_entrance} registration information")
def component_output_data_download():
request_data = request.json
try:
output_tables_meta = get_component_output_tables_meta(task_data=request_data)
except Exception as e:
stat_logger.exception(e)
return error_response(210, str(e))
limit = request_data.get('limit', -1)
if not output_tables_meta:
return error_response(response_code=210, retmsg='no data')
if limit == 0:
return error_response(response_code=210, retmsg='limit is 0')
have_data_label = False
output_data_file_list = []
output_data_meta_file_list = []
output_tmp_dir = os.path.join(os.getcwd(), 'tmp/{}'.format(fate_uuid()))
for output_name, output_table_meta in output_tables_meta.items():
output_data_count = 0
is_str = False
output_data_file_path = "{}/{}.csv".format(output_tmp_dir, output_name)
os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True)
with open(output_data_file_path, 'w') as fw:
with storage.Session.build(name=output_table_meta.get_name(), namespace=output_table_meta.get_namespace()) as storage_session:
output_table = storage_session.get_table()
for k, v in output_table.collect():
data_line, have_data_label, is_str = get_component_output_data_line(src_key=k, src_value=v)
fw.write('{}\n'.format(','.join(map(lambda x: str(x), data_line))))
output_data_count += 1
if output_data_count == limit:
break
if output_data_count:
# get meta
output_data_file_list.append(output_data_file_path)
header = get_component_output_data_schema(output_table_meta=output_table_meta, have_data_label=have_data_label, is_str=is_str)
output_data_meta_file_path = "{}/{}.meta".format(output_tmp_dir, output_name)
output_data_meta_file_list.append(output_data_meta_file_path)
with open(output_data_meta_file_path, 'w') as fw:
json.dump({'header': header}, fw, indent=4)
if request_data.get('head', True) and header:
with open(output_data_file_path, 'r+') as f:
content = f.read()
f.seek(0, 0)
f.write('{}\n'.format(','.join(header)) + content)
# tar
memory_file = io.BytesIO()
tar = tarfile.open(fileobj=memory_file, mode='w:gz')
for index in range(0, len(output_data_file_list)):
tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir))
tar.add(output_data_meta_file_list[index], os.path.relpath(output_data_meta_file_list[index], output_tmp_dir))
tar.close()
memory_file.seek(0)
output_data_file_list.extend(output_data_meta_file_list)
for path in output_data_file_list:
try:
shutil.rmtree(os.path.dirname(path))
except Exception as e:
# warning
stat_logger.warning(e)
tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(request_data['job_id'],
request_data['component_name'],
request_data['role'], request_data['party_id'])
return send_file(memory_file, attachment_filename=tar_file_name, as_attachment=True)
def _find_dependencies(self, mode="train", version=1):
self.component_downstream = [[] for i in range(len(self.components))]
self.component_upstream = [[] for i in range(len(self.components))]
components_details = self.dsl.get("components")
components_output = self._find_outputs()
for name in self.component_name_index.keys():
idx = self.component_name_index.get(name)
upstream_input = components_details.get(name).get("input")
downstream_output = components_details.get(name).get("output", {})
self.components[idx].set_output(downstream_output)
if upstream_input is None:
continue
elif not isinstance(upstream_input, dict):
raise ComponentInputTypeError(component=name)
else:
self.components[idx].set_input(upstream_input)
input_model_keyword = ["model", "isometric_model"]
if mode == "train":
for model_key in input_model_keyword:
if model_key in upstream_input:
model_list = upstream_input.get(model_key)
if not isinstance(model_list, list):
raise ComponentInputModelValueTypeError(component=name, other_info=model_list)
for model in model_list:
module_name = model.split(".", -1)[0]
input_model_name = module_name.split(".")[-1][0]
if module_name in ["args", "pipeline"]:
module_name = model.split(".", -1)[1]
if module_name not in self.component_name_index:
raise ModelInputComponentNotExistError(component=name, input_model=module_name)
else:
if module_name not in components_output or "model" not in components_output[
module_name]:
raise ModelInputNameNotExistError(component=name, input_model=module_name,
other_info=input_model_name)
idx_dependendy = self.component_name_index.get(module_name)
self.component_downstream[idx_dependendy].append(name)
self.component_upstream[idx].append(module_name)
# self.component_upstream_model_relation_set.add((name, module_name))
if model_key == "model":
self.train_input_model[name] = module_name
if "data" in upstream_input:
data_dict = upstream_input.get("data")
if not isinstance(data_dict, dict):
raise ComponentInputDataTypeError(component=name)
for data_set in data_dict:
if not isinstance(data_dict.get(data_set), list):
raise ComponentInputDataValueTypeError(component=name, other_info=data_dict.get(data_set))
if version == 2 and data_set not in ["data", "train_data", "validate_data", "test_data",
"eval_data"]:
stat_logger.warning(
"DSLParser Warning: make sure that input data's data key should be in {}, but {} found".format(
["data", "train_data", "validate_data", "test_data", "eval_data"], data_set))
for data_key in data_dict.get(data_set):
module_name = data_key.split(".", -1)[0]
input_data_name = data_key.split(".", -1)[-1]
if module_name in ["args", "pipeline"]:
self.args_input_to_check.add(input_data_name)
continue
if module_name not in self.component_name_index:
raise DataInputComponentNotExistError(component=name, input_data=module_name)
else:
if module_name not in components_output \
or "data" not in components_output[module_name] \
or input_data_name not in components_output[module_name]["data"]:
raise DataInputNameNotExistError(component=name, input_data=module_name,
other_info=input_data_name)
idx_dependendy = self.component_name_index.get(module_name)
self.component_downstream[idx_dependendy].append(name)
self.component_upstream[idx].append(module_name)
# self.component_upstream_data_relation_set.add((name, module_name))
self.in_degree = [0 for i in range(len(self.components))]
for i in range(len(self.components)):
if self.component_downstream[i]:
self.component_downstream[i] = list(set(self.component_downstream[i]))
if self.component_upstream[i]:
self.component_upstream[i] = list(set(self.component_upstream[i]))
self.in_degree[self.component_name_index.get(self.components[i].get_name())] = len(
self.component_upstream[i])
self._check_dag_dependencies()
for i in range(len(self.components)):
self.components[i].set_upstream(self.component_upstream[i])
self.components[i].set_downstream(self.component_downstream[i])
def send_table(output_tables_meta,
tar_file_name,
limit=-1,
need_head=True):
output_data_file_list = []
output_data_meta_file_list = []
output_tmp_dir = os.path.join(
get_fate_flow_directory(),
'tmp/{}/{}'.format(datetime.datetime.now().strftime("%Y%m%d"),
fate_uuid()))
for output_name, output_table_meta in output_tables_meta.items():
output_data_count = 0
output_data_file_path = "{}/{}.csv".format(output_tmp_dir,
output_name)
output_data_meta_file_path = "{}/{}.meta".format(
output_tmp_dir, output_name)
os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True)
with open(output_data_file_path, 'w') as fw:
with Session() as sess:
output_table = sess.get_table(
name=output_table_meta.get_name(),
namespace=output_table_meta.get_namespace())
if output_table:
for k, v in output_table.collect():
data_line, is_str, extend_header = feature_utils.get_component_output_data_line(
src_key=k,
src_value=v,
schema=output_table_meta.get_schema())
# save meta
if output_data_count == 0:
output_data_file_list.append(
output_data_file_path)
header = get_component_output_data_schema(
output_table_meta=output_table_meta,
is_str=is_str,
extend_header=extend_header)
output_data_meta_file_list.append(
output_data_meta_file_path)
with open(output_data_meta_file_path,
'w') as f:
json.dump({'header': header}, f, indent=4)
if need_head and header and output_table_meta.get_have_head(
):
fw.write('{}\n'.format(','.join(header)))
fw.write('{}\n'.format(','.join(
map(lambda x: str(x), data_line))))
output_data_count += 1
if output_data_count == limit:
break
# tar
output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name)
tar = tarfile.open(output_data_tarfile, mode='w:gz')
for index in range(0, len(output_data_file_list)):
tar.add(
output_data_file_list[index],
os.path.relpath(output_data_file_list[index], output_tmp_dir))
tar.add(
output_data_meta_file_list[index],
os.path.relpath(output_data_meta_file_list[index],
output_tmp_dir))
tar.close()
for key, path in enumerate(output_data_file_list):
try:
os.remove(path)
os.remove(output_data_meta_file_list[key])
except Exception as e:
# warning
stat_logger.warning(e)
return send_file(output_data_tarfile,
attachment_filename=tar_file_name)
