def load(cls): conf = file_utils.load_yaml_conf(FATE_FLOW_JOB_DEFAULT_CONFIG_PATH) if not isinstance(conf, dict): raise ValueError("invalid config file") for k, v in conf.items(): if hasattr(cls, k): setattr(cls, k, v) else: stat_logger.warning(f"job default config not supported {k}") return cls.get_all()
def component_output_data_download(): request_data = request.json output_data_table = get_component_output_data_table(task_data=request_data) limit = request_data.get('limit', -1) if not output_data_table: return error_response(response_code=500, retmsg='no data') if limit == 0: return error_response(response_code=500, retmsg='limit is 0') output_data_count = 0 have_data_label = False output_tmp_dir = os.path.join(os.getcwd(), 'tmp/{}'.format(get_fate_uuid())) output_file_path = '{}/output_%s'.format(output_tmp_dir) output_data_file_path = output_file_path % 'data.csv' os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) with open(output_data_file_path, 'w') as fw: for k, v in output_data_table.collect(): data_line, have_data_label = get_component_output_data_line( src_key=k, src_value=v) fw.write('{}\n'.format(','.join(map(lambda x: str(x), data_line)))) output_data_count += 1 if output_data_count == limit: break if output_data_count: # get meta header = get_component_output_data_meta( output_data_table=output_data_table, have_data_label=have_data_label) output_data_meta_file_path = output_file_path % 'data_meta.json' with open(output_data_meta_file_path, 'w') as fw: json.dump({'header': header}, fw, indent=4) # tar memory_file = io.BytesIO() tar = tarfile.open(fileobj=memory_file, mode='w:gz') tar.add(output_data_file_path, os.path.relpath(output_data_file_path, output_tmp_dir)) tar.add(output_data_meta_file_path, os.path.relpath(output_data_meta_file_path, output_tmp_dir)) tar.close() memory_file.seek(0) try: shutil.rmtree(os.path.dirname(output_data_file_path)) except Exception as e: # warning stat_logger.warning(e) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format( request_data['job_id'], request_data['component_name'], request_data['role'], request_data['party_id']) return send_file(memory_file, attachment_filename=tar_file_name, as_attachment=True)
def get_proto_buffer_class(buffer_name): module_path, base_import_path = provider_utils.get_provider_model_paths(RuntimeConfig.COMPONENT_PROVIDER) exception = ModuleNotFoundError(f'no module named {buffer_name}') for f in module_path.glob('*.py'): try: proto_module = importlib.import_module('.'.join([*base_import_path, f.stem])) for name, obj in inspect.getmembers(proto_module): if inspect.isclass(obj) and name == buffer_name: return obj except Exception as e: exception = e stat_logger.warning(e) raise exception
def wait_child_process(signum, frame): try: while True: child_pid, status = os.waitpid(-1, os.WNOHANG) if child_pid == 0: stat_logger.info('no child process was immediately available') break exitcode = status >> 8 stat_logger.info('child process %s exit with exitcode %s', child_pid, exitcode) except OSError as e: if e.errno == errno.ECHILD: stat_logger.warning('current process has no existing unwaited-for child processes.') else: raise
def kill_process(process: psutil.Process = None, pid: int = None, expected_cmdline: list = None): process = process if process is not None else get_process_instance(pid) if process is None: return for child in process.children(recursive=True): try: if check_process(pid=child.pid): child.kill() except Exception as e: stat_logger.warning(f"kill {child.pid} process failed", exc_info=True) if check_process(pid=process.pid, expected_cmdline=expected_cmdline): process.kill()
def get_proto_buffer_class(buffer_name): package_path = os.path.join(file_utils.get_project_base_directory(), 'federatedml', 'protobuf', 'generated') package_python_path = 'federatedml.protobuf.generated' for f in os.listdir(package_path): if f.startswith('.'): continue try: proto_module = importlib.import_module(package_python_path + '.' + f.rstrip('.py')) for name, obj in inspect.getmembers(proto_module): if inspect.isclass(obj) and name == buffer_name: return obj except Exception as e: stat_logger.warning(e) else: return None
def register_engine(cls, engine_type, engine_info): nodes = engine_info.get("nodes", 1) cores = engine_info.get("cores_per_node", 0) * nodes memory = engine_info.get("memory_per_node", 0) * nodes engine_name = engine_info.get("engine", "UNKNOWN") engine_address = engine_info.get("address", {}) filters = [EngineRegistry.f_engine_name == engine_name, EngineRegistry.f_engine_type == engine_type] resources = EngineRegistry.select().where(*filters) if resources: resource = resources[0] update_fields = {} update_fields[EngineRegistry.f_engine_address] = engine_address update_fields[EngineRegistry.f_cores] = cores update_fields[EngineRegistry.f_memory] = memory update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + ( cores - resource.f_cores) update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + ( memory - resource.f_memory) update_fields[EngineRegistry.f_nodes] = nodes operate = EngineRegistry.update(update_fields).where(*filters) update_status = operate.execute() > 0 if update_status: stat_logger.info(f"update {engine_type} engine {engine_name} registration information") else: stat_logger.info(f"update {engine_type} engine {engine_name} registration information takes no effect") else: resource = EngineRegistry() resource.f_create_time = base_utils.current_timestamp() resource.f_engine_name = engine_name resource.f_engine_type = engine_type resource.f_engine_address = engine_address resource.f_cores = cores resource.f_memory = memory resource.f_remaining_cores = cores resource.f_remaining_memory = memory resource.f_nodes = nodes try: resource.save(force_insert=True) except Exception as e: stat_logger.warning(e) stat_logger.info(f"create {engine_type} engine {engine_name} registration information")
def register_engine(cls, engine_type, engine_name, engine_entrance, engine_config): nodes = engine_config.get("nodes", 1) cores = engine_config.get("cores_per_node", 0) * nodes * TOTAL_CORES_OVERWEIGHT_PERCENT memory = engine_config.get("memory_per_node", 0) * nodes * TOTAL_MEMORY_OVERWEIGHT_PERCENT filters = [EngineRegistry.f_engine_type == engine_type, EngineRegistry.f_engine_name == engine_name] resources = EngineRegistry.select().where(*filters) if resources: resource = resources[0] update_fields = {} update_fields[EngineRegistry.f_engine_config] = engine_config update_fields[EngineRegistry.f_cores] = cores update_fields[EngineRegistry.f_memory] = memory update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + ( cores - resource.f_cores) update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + ( memory - resource.f_memory) update_fields[EngineRegistry.f_nodes] = nodes operate = EngineRegistry.update(update_fields).where(*filters) update_status = operate.execute() > 0 if update_status: stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information") else: stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information takes no effect") else: resource = EngineRegistry() resource.f_create_time = base_utils.current_timestamp() resource.f_engine_type = engine_type resource.f_engine_name = engine_name resource.f_engine_entrance = engine_entrance resource.f_engine_config = engine_config resource.f_cores = cores resource.f_memory = memory resource.f_remaining_cores = cores resource.f_remaining_memory = memory resource.f_nodes = nodes try: resource.save(force_insert=True) except Exception as e: stat_logger.warning(e) stat_logger.info(f"create {engine_type} engine {engine_name} {engine_entrance} registration information")
def component_output_data_download(): request_data = request.json try: output_tables_meta = get_component_output_tables_meta(task_data=request_data) except Exception as e: stat_logger.exception(e) return error_response(210, str(e)) limit = request_data.get('limit', -1) if not output_tables_meta: return error_response(response_code=210, retmsg='no data') if limit == 0: return error_response(response_code=210, retmsg='limit is 0') have_data_label = False output_data_file_list = [] output_data_meta_file_list = [] output_tmp_dir = os.path.join(os.getcwd(), 'tmp/{}'.format(fate_uuid())) for output_name, output_table_meta in output_tables_meta.items(): output_data_count = 0 is_str = False output_data_file_path = "{}/{}.csv".format(output_tmp_dir, output_name) os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) with open(output_data_file_path, 'w') as fw: with storage.Session.build(name=output_table_meta.get_name(), namespace=output_table_meta.get_namespace()) as storage_session: output_table = storage_session.get_table() for k, v in output_table.collect(): data_line, have_data_label, is_str = get_component_output_data_line(src_key=k, src_value=v) fw.write('{}\n'.format(','.join(map(lambda x: str(x), data_line)))) output_data_count += 1 if output_data_count == limit: break if output_data_count: # get meta output_data_file_list.append(output_data_file_path) header = get_component_output_data_schema(output_table_meta=output_table_meta, have_data_label=have_data_label, is_str=is_str) output_data_meta_file_path = "{}/{}.meta".format(output_tmp_dir, output_name) output_data_meta_file_list.append(output_data_meta_file_path) with open(output_data_meta_file_path, 'w') as fw: json.dump({'header': header}, fw, indent=4) if request_data.get('head', True) and header: with open(output_data_file_path, 'r+') as f: content = f.read() f.seek(0, 0) f.write('{}\n'.format(','.join(header)) + content) # tar memory_file = io.BytesIO() tar = tarfile.open(fileobj=memory_file, mode='w:gz') for index in range(0, len(output_data_file_list)): tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) tar.add(output_data_meta_file_list[index], os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) tar.close() memory_file.seek(0) output_data_file_list.extend(output_data_meta_file_list) for path in output_data_file_list: try: shutil.rmtree(os.path.dirname(path)) except Exception as e: # warning stat_logger.warning(e) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(request_data['job_id'], request_data['component_name'], request_data['role'], request_data['party_id']) return send_file(memory_file, attachment_filename=tar_file_name, as_attachment=True)
def _find_dependencies(self, mode="train", version=1): self.component_downstream = [[] for i in range(len(self.components))] self.component_upstream = [[] for i in range(len(self.components))] components_details = self.dsl.get("components") components_output = self._find_outputs() for name in self.component_name_index.keys(): idx = self.component_name_index.get(name) upstream_input = components_details.get(name).get("input") downstream_output = components_details.get(name).get("output", {}) self.components[idx].set_output(downstream_output) if upstream_input is None: continue elif not isinstance(upstream_input, dict): raise ComponentInputTypeError(component=name) else: self.components[idx].set_input(upstream_input) input_model_keyword = ["model", "isometric_model"] if mode == "train": for model_key in input_model_keyword: if model_key in upstream_input: model_list = upstream_input.get(model_key) if not isinstance(model_list, list): raise ComponentInputModelValueTypeError(component=name, other_info=model_list) for model in model_list: module_name = model.split(".", -1)[0] input_model_name = module_name.split(".")[-1][0] if module_name in ["args", "pipeline"]: module_name = model.split(".", -1)[1] if module_name not in self.component_name_index: raise ModelInputComponentNotExistError(component=name, input_model=module_name) else: if module_name not in components_output or "model" not in components_output[ module_name]: raise ModelInputNameNotExistError(component=name, input_model=module_name, other_info=input_model_name) idx_dependendy = self.component_name_index.get(module_name) self.component_downstream[idx_dependendy].append(name) self.component_upstream[idx].append(module_name) # self.component_upstream_model_relation_set.add((name, module_name)) if model_key == "model": self.train_input_model[name] = module_name if "data" in upstream_input: data_dict = upstream_input.get("data") if not isinstance(data_dict, dict): raise ComponentInputDataTypeError(component=name) for data_set in data_dict: if not isinstance(data_dict.get(data_set), list): raise ComponentInputDataValueTypeError(component=name, other_info=data_dict.get(data_set)) if version == 2 and data_set not in ["data", "train_data", "validate_data", "test_data", "eval_data"]: stat_logger.warning( "DSLParser Warning: make sure that input data's data key should be in {}, but {} found".format( ["data", "train_data", "validate_data", "test_data", "eval_data"], data_set)) for data_key in data_dict.get(data_set): module_name = data_key.split(".", -1)[0] input_data_name = data_key.split(".", -1)[-1] if module_name in ["args", "pipeline"]: self.args_input_to_check.add(input_data_name) continue if module_name not in self.component_name_index: raise DataInputComponentNotExistError(component=name, input_data=module_name) else: if module_name not in components_output \ or "data" not in components_output[module_name] \ or input_data_name not in components_output[module_name]["data"]: raise DataInputNameNotExistError(component=name, input_data=module_name, other_info=input_data_name) idx_dependendy = self.component_name_index.get(module_name) self.component_downstream[idx_dependendy].append(name) self.component_upstream[idx].append(module_name) # self.component_upstream_data_relation_set.add((name, module_name)) self.in_degree = [0 for i in range(len(self.components))] for i in range(len(self.components)): if self.component_downstream[i]: self.component_downstream[i] = list(set(self.component_downstream[i])) if self.component_upstream[i]: self.component_upstream[i] = list(set(self.component_upstream[i])) self.in_degree[self.component_name_index.get(self.components[i].get_name())] = len( self.component_upstream[i]) self._check_dag_dependencies() for i in range(len(self.components)): self.components[i].set_upstream(self.component_upstream[i]) self.components[i].set_downstream(self.component_downstream[i])
def send_table(output_tables_meta, tar_file_name, limit=-1, need_head=True): output_data_file_list = [] output_data_meta_file_list = [] output_tmp_dir = os.path.join( get_fate_flow_directory(), 'tmp/{}/{}'.format(datetime.datetime.now().strftime("%Y%m%d"), fate_uuid())) for output_name, output_table_meta in output_tables_meta.items(): output_data_count = 0 output_data_file_path = "{}/{}.csv".format(output_tmp_dir, output_name) output_data_meta_file_path = "{}/{}.meta".format( output_tmp_dir, output_name) os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) with open(output_data_file_path, 'w') as fw: with Session() as sess: output_table = sess.get_table( name=output_table_meta.get_name(), namespace=output_table_meta.get_namespace()) if output_table: for k, v in output_table.collect(): data_line, is_str, extend_header = feature_utils.get_component_output_data_line( src_key=k, src_value=v, schema=output_table_meta.get_schema()) # save meta if output_data_count == 0: output_data_file_list.append( output_data_file_path) header = get_component_output_data_schema( output_table_meta=output_table_meta, is_str=is_str, extend_header=extend_header) output_data_meta_file_list.append( output_data_meta_file_path) with open(output_data_meta_file_path, 'w') as f: json.dump({'header': header}, f, indent=4) if need_head and header and output_table_meta.get_have_head( ): fw.write('{}\n'.format(','.join(header))) fw.write('{}\n'.format(','.join( map(lambda x: str(x), data_line)))) output_data_count += 1 if output_data_count == limit: break # tar output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name) tar = tarfile.open(output_data_tarfile, mode='w:gz') for index in range(0, len(output_data_file_list)): tar.add( output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) tar.add( output_data_meta_file_list[index], os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) tar.close() for key, path in enumerate(output_data_file_list): try: os.remove(path) os.remove(output_data_meta_file_list[key]) except Exception as e: # warning stat_logger.warning(e) return send_file(output_data_tarfile, attachment_filename=tar_file_name)