def init(): if USE_AUTHENTICATION: # init local storage stat_logger.info('init local authorization library') file_dir = os.path.join(file_utils.get_project_base_directory(), 'fate_flow') os.makedirs(file_dir, exist_ok=True) PrivilegeAuth.local_storage_file = os.path.join( file_dir, 'authorization_config.json') if not os.path.exists(PrivilegeAuth.local_storage_file): with open(PrivilegeAuth.local_storage_file, 'w') as fp: fp.write(json.dumps({})) # init whitelist PrivilegeAuth.command_whitelist = PRIVILEGE_COMMAND_WHITELIST # init ALL_PERMISSION component_path = os.path.join( file_utils.get_project_base_directory(), 'federatedml', 'conf', 'setting_conf') command_file_path = os.path.join( file_utils.get_project_base_directory(), 'fate_flow', 'apps', 'schedule_app.py') stat_logger.info( 'search commands from {}'.format(command_file_path)) search_command(command_file_path) stat_logger.info( 'search components from {}'.format(component_path)) search_component(component_path)
def get_schedule_logger(job_id='', log_type='schedule'): fate_flow_log_dir = os.path.join( file_utils.get_project_base_directory(), 'logs', 'fate_flow') job_log_dir = os.path.join(file_utils.get_project_base_directory(), 'logs', job_id) if not job_id: log_dirs = [fate_flow_log_dir] else: if log_type == 'audit': log_dirs = [job_log_dir, fate_flow_log_dir] else: log_dirs = [job_log_dir] os.makedirs(job_log_dir, exist_ok=True) os.makedirs(fate_flow_log_dir, exist_ok=True) logger = logging.getLogger('{}_{}'.format(job_id, log_type)) logger.setLevel(LoggerFactory.LEVEL) for job_log_dir in log_dirs: handler = LoggerFactory.get_handler(class_name=None, level=LoggerFactory.LEVEL, log_dir=job_log_dir, log_type=log_type) error_handler = LoggerFactory.get_handler(class_name=None, level=logging.ERROR, log_dir=job_log_dir, log_type=log_type) logger.addHandler(handler) logger.addHandler(error_handler) if job_id: with LoggerFactory.lock: LoggerFactory.schedule_logger_dict[job_id + log_type] = logger return logger
def download_data(data_func): _data = request.json _job_id = generate_job_id() logger.info('generated job_id {}, body {}'.format(_job_id, _data)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) _download_module = os.path.join(file_utils.get_project_base_directory(), "arch/api/utils/download.py") _upload_module = os.path.join(file_utils.get_project_base_directory(), "arch/api/utils/upload.py") if data_func == "download": _module = _download_module else: _module = _upload_module try: if data_func == "download": progs = [ "python3", _module, "-j", _job_id, "-c", os.path.abspath(_data.get("config_path")) ] else: progs = [ "python3", _module, "-c", os.path.abspath(_data.get("config_path")) ] logger.info('Starting progs: {}'.format(progs)) std_log = open(os.path.join(_job_dir, 'std.log'), 'w') task_pid_path = os.path.join(_job_dir, 'pids') if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE else: startupinfo = None p = subprocess.Popen(progs, stdout=std_log, stderr=std_log, startupinfo=startupinfo) os.makedirs(task_pid_path, exist_ok=True) with open(os.path.join(task_pid_path, data_func + ".pid"), 'w') as f: f.truncate() f.write(str(p.pid) + "\n") f.flush() return get_json_result(0, "success, job_id {}".format(_job_id)) except Exception as e: print(e) return get_json_result(-104, "failed, job_id {}".format(_job_id))
def test_job_operation(self): # submit with open(os.path.join(file_utils.get_project_base_directory(), self.dsl_path), 'r') as f: dsl_data = json.load(f) with open(os.path.join(file_utils.get_project_base_directory(), self.config_path), 'r') as f: config_data = json.load(f) config_data['job_parameters']['work_mode'] = WORK_MODE response = requests.post("/".join([self.server_url, 'job', 'submit']), json={'job_dsl': dsl_data, 'job_runtime_conf': config_data}) self.assertTrue(response.status_code in [200, 201]) self.assertTrue(int(response.json()['retcode']) == 0) job_id = response.json()['jobId'] # query response = requests.post("/".join([self.server_url, 'job', 'query']), json={'job_id': job_id, 'role': 'guest'}) self.assertTrue(int(response.json()['retcode']) == 0) job_info = response.json()['data'][0] # note response = requests.post("/".join([self.server_url, 'job', 'update']), json={'job_id': job_id, 'role': job_info['f_role'], 'party_id': job_info['f_party_id'], 'notes': 'unittest'}) self.assertTrue(int(response.json()['retcode']) == 0) # config response = requests.post("/".join([self.server_url, 'job', 'config']), json={'job_id': job_id, 'role': job_info['f_role'], 'party_id': job_info['f_party_id']}) self.assertTrue(int(response.json()['retcode']) == 0) print('waiting 10s ...') time.sleep(10) # stop response = requests.post("/".join([self.server_url, 'job', 'stop']), json={'job_id': job_id}) self.assertTrue(int(response.json()['retcode']) == 0) # logs with closing(requests.get("/".join([self.server_url, 'job', 'log']), json={'job_id': job_id}, stream=True)) as response: self.assertTrue(response.status_code in [200, 201]) # query task response = requests.post("/".join([self.server_url, 'job', '/task/query']), json={'job_id': job_id}) self.assertTrue(int(response.json()['retcode']) == 0) # query data viw response = requests.post("/".join([self.server_url, 'job', '/data/view/query']), json={'party_id': 0}) self.assertTrue(int(response.json()['retcode']) == 0) print('waiting 10s...') time.sleep(10) # clean job response = requests.post("/".join([self.server_url, 'job', '/clean']), json={'job_id': job_id}) self.assertTrue(int(response.json()['retcode']) == 0)
def run(self, component_parameters=None, args=None): self.parameters = component_parameters["UploadParam"] self.parameters["role"] = component_parameters["role"] self.parameters["local"] = component_parameters["local"] job_id = self.taskid.split("_")[0] if not os.path.isabs(self.parameters.get("file", "")): self.parameters["file"] = os.path.join(file_utils.get_project_base_directory(), self.parameters["file"]) if not os.path.exists(self.parameters["file"]): raise Exception("%s is not exist, please check the configure" % (self.parameters["file"])) table_name, namespace = dtable_utils.get_table_info(config=self.parameters, create=True) _namespace, _table_name = self.generate_table_name(self.parameters["file"]) if namespace is None: namespace = _namespace if table_name is None: table_name = _table_name read_head = self.parameters['head'] if read_head == 0: head = False elif read_head == 1: head = True else: raise Exception("'head' in conf.json should be 0 or 1") partition = self.parameters["partition"] if partition <= 0 or partition >= self.MAX_PARTITION_NUM: raise Exception("Error number of partition, it should between %d and %d" % (0, self.MAX_PARTITION_NUM)) session.init(mode=self.parameters['work_mode']) data_table_count = self.save_data_table(table_name, namespace, head, self.parameters.get('in_version', False)) LOGGER.info("------------load data finish!-----------------") LOGGER.info("file: {}".format(self.parameters["file"])) LOGGER.info("total data_count: {}".format(data_table_count)) LOGGER.info("table name: {}, table namespace: {}".format(table_name, namespace))
def init_roll_site_context(runtime_conf, session_id): from eggroll.roll_site.roll_site import RollSiteContext from eggroll.roll_pair.roll_pair import RollPairContext LOGGER.info("init_roll_site_context runtime_conf: {}".format(runtime_conf)) session_instance = FateSession.get_instance()._eggroll.get_session() rp_context = RollPairContext(session_instance) role = runtime_conf.get("local").get("role") party_id = str(runtime_conf.get("local").get("party_id")) _path = file_utils.get_project_base_directory( ) + "/arch/conf/server_conf.json" server_conf = file_utils.load_json_conf(_path) host = server_conf.get('servers').get('proxy').get("host") port = server_conf.get('servers').get('proxy').get("port") options = { 'self_role': role, 'self_party_id': party_id, 'proxy_endpoint': ErEndpoint(host, int(port)) } rs_context = RollSiteContext(session_id, rp_ctx=rp_context, options=options) LOGGER.info("init_roll_site_context done: {}".format(rs_context.__dict__)) return rp_context, rs_context
def generate_transfer_var_class(self, transfer_var_conf_path, out_path): base_dir = file_utils.get_project_base_directory() merge_conf_path = os.path.join( base_dir, "federatedml/transfer_variable_conf/transfer_conf.json") merge_dict = {} if os.path.isfile(merge_conf_path): with open(merge_conf_path, "r") as fin: merge_dict = json.loads(fin.read()) var_dict = {} with open(transfer_var_conf_path) as fin: var_dict = json.loads(fin.read()) merge_dict.update(var_dict) with open(merge_conf_path, "w") as fout: json_dumps_indent_str = json.dumps(merge_dict, indent=1) buffers = json_dumps_indent_str.split("\n", -1) for buf in buffers: fout.write(buf + "\n") fout = open(out_path, "w") with_header = True for class_name in var_dict: transfer_var_dict = var_dict[class_name] self.write_out_class(fout, class_name, transfer_var_dict, with_header) with_header = False fout.flush() fout.close()
def set_directory(directory=None, parent_log_dir=None, append_to_parent_log=None, force=False): if parent_log_dir: LoggerFactory.PARENT_LOG_DIR = parent_log_dir if append_to_parent_log: LoggerFactory.append_to_parent_log = append_to_parent_log with LoggerFactory.lock: if not directory: directory = os.path.join( file_utils.get_project_base_directory(), 'logs') if not LoggerFactory.LOG_DIR or force: LoggerFactory.LOG_DIR = directory os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True) for loggerName, ghandler in LoggerFactory.global_handler_dict.items( ): for className, (logger, handler) in LoggerFactory.logger_dict.items(): logger.removeHandler(ghandler) ghandler.close() LoggerFactory.global_handler_dict = {} for className, (logger, handler) in LoggerFactory.logger_dict.items(): logger.removeHandler(handler) _hanlder = None if handler: handler.close() if className != "default": _hanlder = LoggerFactory.get_handler(className) logger.addHandler(_hanlder) LoggerFactory.assemble_global_handler(logger) LoggerFactory.logger_dict[className] = logger, _hanlder
def list_files(namespace): _type = StoreType.LMDB.value _base_dir = os.sep.join([os.path.join(file_utils.get_project_base_directory(), 'data'), _type]) _namespace_dir = os.sep.join([_base_dir, namespace]) if not os.path.isdir(_base_dir): raise EnvironmentError("illegal datadir") return [file for file in os.listdir(_namespace_dir) if not file.endswith('.meta')]
def init(job_id=None, mode: typing.Union[int, WorkMode] = WorkMode.STANDALONE, backend: typing.Union[int, Backend] = Backend.EGGROLL, persistent_engine: StoreType = StoreType.LMDB, set_log_dir=True): if isinstance(mode, int): mode = WorkMode(mode) if isinstance(backend, int): backend = Backend(backend) if RuntimeInstance.SESSION: return if job_id is None: job_id = str(uuid.uuid1()) if set_log_dir: LoggerFactory.set_directory() else: if set_log_dir: LoggerFactory.set_directory( os.path.join(file_utils.get_project_base_directory(), 'logs', job_id)) RuntimeInstance.MODE = mode RuntimeInstance.Backend = backend from arch.api.table.session import build_session session = build_session(job_id=job_id, work_mode=mode, backend=backend) RuntimeInstance.SESSION = session
def init(job_id=None, mode: WorkMode = WorkMode.STANDALONE, naming_policy: NamingPolicy = NamingPolicy.DEFAULT): if RuntimeInstance.EGGROLL: return if job_id is None: job_id = str(uuid.uuid1()) LoggerFactory.set_directory() else: LoggerFactory.set_directory( os.path.join(file_utils.get_project_base_directory(), 'logs', job_id)) RuntimeInstance.MODE = mode eggroll_context = EggRollContext(naming_policy=naming_policy) if mode == WorkMode.STANDALONE: from arch.api.standalone.eggroll import Standalone RuntimeInstance.EGGROLL = Standalone(job_id=job_id, eggroll_context=eggroll_context) elif mode == WorkMode.CLUSTER: from arch.api.cluster.eggroll import _EggRoll from arch.api.cluster.eggroll import init as c_init c_init(job_id, eggroll_context=eggroll_context) RuntimeInstance.EGGROLL = _EggRoll.get_instance() else: from arch.api.cluster import simple_roll simple_roll.init(job_id) RuntimeInstance.EGGROLL = simple_roll.EggRoll.get_instance() RuntimeInstance.EGGROLL.table("__federation__", job_id, partition=10)
def runtime_json_generator(site_flow, runtime_path): if site_flow is None: raise Exception("site_flow is None") with open(runtime_path) as runtime_conf: runtime_json = json.load(runtime_conf) with open(file_utils.get_project_base_directory() + "/contrib/fate_script/conf/route.json") as route_conf: route_json = json.load(route_conf) site_set = set() inner_dict = defaultdict() for i in range(len(site_flow)): if site_flow[i][2] is None or site_flow[i][3] is None: continue site_set.add(site_flow[i][2]) site_set.add(site_flow[i][3]) for i in range(len(site_set)): if list(site_set)[i] in route_json["cluster_a"]["role"]: cluster_type = "cluster_a" elif list(site_set)[i] in route_json["cluster_b"]["role"]: cluster_type = "cluster_b" else: raise Exception("role:{} is not in route.json".format( list(site_set)[i])) inner_dict[list(site_set)[i]] = [route_json[cluster_type]["party_id"]] for i in range(len(site_set)): if list(site_set)[i] in route_json["cluster_a"]["role"]: cluster_type = "cluster_a" elif list(site_set)[i] in route_json["cluster_b"]["role"]: cluster_type = "cluster_b" else: raise Exception("role:{} is not in route.json".format( list(site_set)[i])) runtime_json["local"]["role"] = list(site_set)[i] runtime_json["local"]["party_id"] = route_json[cluster_type][ "party_id"] runtime_json["role"] = inner_dict json_path = file_utils.get_project_base_directory( ) + "/contrib/fate_script/conf/" + list( site_set)[i] + "_runtime_conf.json" with open(json_path, "wt") as fout: fout.write(json.dumps(runtime_json, indent=4))
def monkey_patch(): package_name = "monkey_patch" package_path = os.path.join(file_utils.get_project_base_directory(), "fate_flow", package_name) for f in os.listdir(package_path): if not os.path.isdir(os.path.join(package_path, f)) or f == "__pycache__": continue patch_module = importlib.import_module("fate_flow." + package_name + '.' + f + '.monkey_patch') patch_module.patch_all()
def __init__(self, job_id=None): self.data_dir = os.path.join(file_utils.get_project_base_directory(), 'data') self.job_id = str( uuid.uuid1()) if job_id is None else "{}".format(job_id) self.meta_table = _DTable('__META__', '__META__', 'fragments', 10) self.pool = Executor() Standalone.__instance = self
def get_role_list(): with open(file_utils.get_project_base_directory() + "/contrib/fate_script/conf/route.json") as fp: route_json = json.load(fp) print("keys:", route_json.keys()) role_list = [] for key in route_json.keys(): role_list.append(route_json[key]['role']) return reduce(operator.add, role_list)
def run_fate_script_standalone(job_id, work_mode): role_list = get_role_list() for role in role_list: runtime_conf_path = file_utils.get_project_base_directory( ) + "/contrib/fate_script/conf/" + str(role) + "_runtime_conf.json" run_cmd = "python fateScript.py " + str(role) + " " + str( job_id) + " " + str(runtime_conf_path) + " " + str(work_mode) + "&" print("run cmd:", format(run_cmd)) os.system(run_cmd)
def get_job_dsl_parser(dsl=None, runtime_conf=None, pipeline_dsl=None, train_runtime_conf=None): dsl_parser = DSLParser() default_runtime_conf_path = os.path.join( file_utils.get_project_base_directory(), *['federatedml', 'conf', 'default_runtime_conf']) setting_conf_path = os.path.join(file_utils.get_project_base_directory(), *['federatedml', 'conf', 'setting_conf']) job_type = runtime_conf.get('job_parameters', {}).get('job_type', 'train') dsl_parser.run(dsl=dsl, runtime_conf=runtime_conf, pipeline_dsl=pipeline_dsl, pipeline_runtime_conf=train_runtime_conf, default_runtime_conf_prefix=default_runtime_conf_path, setting_conf_prefix=setting_conf_path, mode=job_type) return dsl_parser
def setDirectory(directory=None): with LoggerFactory.lock: if not directory: directory = os.path.join(file_utils.get_project_base_directory(), 'logs') LoggerFactory.LOG_DIR = directory os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True) for className, (logger, handler) in LoggerFactory.loggerDict.items(): logger.removeHandler(handler) handler.close() _hanlder = LoggerFactory.get_hanlder(className) logger.addHandler(_hanlder) LoggerFactory.loggerDict[className] = logger, _hanlder
def create_pipelined_model(self): if os.path.exists(self.model_path): raise Exception("Model creation failed because it has already been created, model cache path is {}".format( self.model_path )) else: os.makedirs(self.model_path, exist_ok=False) for path in [self.variables_index_path, self.variables_data_path]: os.makedirs(path, exist_ok=False) shutil.copytree(os.path.join(file_utils.get_project_base_directory(), "federatedml", "protobuf", "proto"), self.define_proto_path) with open(self.define_meta_path, "w", encoding="utf-8") as fw: yaml.dump({"describe": "This is the model definition meta"}, fw, Dumper=yaml.RoundTripDumper)
def download_upload(data_func): request_config = request.json _job_id = generate_job_id() stat_logger.info('generated job_id {}, body {}'.format(_job_id, request_config)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) module = data_func required_arguments = ['work_mode', 'namespace', 'table_name'] if module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif module == 'download': required_arguments.extend(['output_path']) else: raise Exception('can not support this operating: {}'.format(module)) detect_utils.check_config(request_config, required_arguments=required_arguments) if module == "upload": if not os.path.isabs(request_config['file']): request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"]) try: conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module, role=request_config.get('local', {}).get("role"), party_id=request_config.get('local', {}).get("party_id", '')) file_utils.dump_json_conf(request_config, conf_file_path) progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-j", _job_id, "-c", conf_file_path ] try: p = run_subprocess(config_dir=_job_dir, process_cmd=progs) except Exception as e: stat_logger.exception(e) p = None return get_json_result(retcode=(0 if p else 101), job_id=_job_id, data={'table_name': request_config['table_name'], 'namespace': request_config['namespace'], 'pid': p.pid if p else ''}) except Exception as e: stat_logger.exception(e) return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)
def __init__(self, transfer_conf_path): self.transfer_auth = {} for path, _, file_names in os.walk( os.path.join(file_utils.get_project_base_directory(), transfer_conf_path)): for name in file_names: transfer_conf = os.path.join(path, name) if transfer_conf.endswith(".json"): self.transfer_auth.update( file_utils.load_json_conf(transfer_conf)) # cache self._authorized_src = {} self._authorized_dst = {}
def main(): parser = argparse.ArgumentParser( description="generate conf from transfer variable class") parser.add_argument( "-s", "--src", type=str, default=DEFAULT_SRC, help=f"dir to search transfer classes, defaults to {DEFAULT_SRC}") parser.add_argument( "-d", "--dst", type=str, default=DEFAULT_DST, help=f"dir to save generated json files, defaults to {DEFAULT_DST}") arg = parser.parse_args() dst = arg.dst if not os.path.exists(dst): os.mkdir(dst) if not os.path.isdir(dst): raise ValueError(f"{dst} should be a directory") src = arg.src if not os.path.exists(src): raise ValueError(f"{src} not exists") BaseTransferVariables._disable__singleton() base = file_utils.get_project_base_directory() for path, _, files in os.walk(src): path = os.path.relpath(path, base) for file_name in files: if file_name.endswith(".py"): name = f"{path.replace('/', '.')}.{file_name[:-3]}" for cls in search_subclasses(name, BaseTransferVariables): class_name = cls.__name__ file_name = _Camel2Snake.to_snake(class_name) variable_dict = {} for k, v in search_transfer_variable(cls()).items(): variable_dict[k] = {"src": v._src, "dst": v._dst} if len(variable_dict) < 1: continue saving_path = os.path.join(dst, f"{file_name}.json") print( f"found transfer_class: {class_name} in {name}, saving to {saving_path}" ) with open(saving_path, "w") as g: json.dump({class_name: variable_dict}, g, indent=2)
def download_upload(data_func): request_config = request.json _job_id = generate_job_id() logger.info('generated job_id {}, body {}'.format(_job_id, request_config)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) module = data_func if module == "upload": if not os.path.isabs(request_config.get("file", "")): request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"]) try: request_config["work_mode"] = request_config.get('work_mode', WORK_MODE) table_name, namespace = dtable_utils.get_table_info(config=request_config, create=(True if module == 'upload' else False)) if not table_name or not namespace: return get_json_result(status=102, msg='no table name and namespace') request_config['table_name'] = table_name request_config['namespace'] = namespace conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module, role=request_config.get('local', {}).get("role"), party_id=request_config.get('local', {}).get("party_id", PARTY_ID)) file_utils.dump_json_conf(request_config, conf_file_path) if module == "download": progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-j", _job_id, "-c", conf_file_path ] else: progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-c", conf_file_path ] p = run_subprocess(job_dir=_job_dir, job_role=data_func, progs=progs) return get_json_result(job_id=_job_id, data={'pid': p.pid, 'table_name': request_config['table_name'], 'namespace': request_config['namespace']}) except Exception as e: logger.exception(e) return get_json_result(status=-104, msg="failed", job_id=_job_id)
def start_workflow(job_id, module, role): _data = request.json _job_dir = get_job_directory(job_id) _party_id = str(_data['local']['party_id']) _method = _data['WorkFlowParam']['method'] conf_path_dir = os.path.join(_job_dir, _method, module, role, _party_id) os.makedirs(conf_path_dir, exist_ok=True) conf_file_path = os.path.join(conf_path_dir, 'runtime_conf.json') with open(conf_file_path, 'w+') as f: f.truncate() f.write(json.dumps(_data, indent=4)) f.flush() if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE else: startupinfo = None task_pid_path = os.path.join(_job_dir, 'pids') std_log = open(os.path.join(_job_dir, role + '.std.log'), 'w') progs = [ "python3", os.path.join(file_utils.get_project_base_directory(), _data['CodePath']), "-j", job_id, "-c", os.path.abspath(conf_file_path) ] logger.info('Starting progs: {}'.format(" ".join(progs))) p = subprocess.Popen(progs, stdout=std_log, stderr=std_log, startupinfo=startupinfo) os.makedirs(task_pid_path, exist_ok=True) with open(os.path.join(task_pid_path, role + ".pid"), 'w') as f: f.truncate() f.write(str(p.pid) + "\n") f.flush() job_data = dict() job_data["begin_date"] = datetime.datetime.now() job_data["status"] = "ready" with open(conf_file_path) as fr: config = json.load(fr) job_data.update(config) job_data["my_role"] = config.get("local", {}).get("role") save_job_info(job_id=job_id, **job_data) update_job_queue(job_id=job_id, update_data={"status": "ready"}) return get_json_result(msg="success, pid is %s" % p.pid)
def setUp(self): self.data_dir = os.path.join(file_utils.get_project_base_directory(), "examples", "data") self.upload_guest_config = { "file": os.path.join(self.data_dir, "breast_hetero_guest.csv"), "head": 1, "partition": 10, "work_mode": WORK_MODE, "namespace": "fate_flow_test_breast_hetero", "table_name": "breast_hetero_guest", "use_local_data": 0, 'drop': 0 } self.upload_host_config = { "file": os.path.join(self.data_dir, "breast_hetero_host.csv"), "head": 1, "partition": 10, "work_mode": WORK_MODE, "namespace": "fate_flow_test_breast_hetero", "table_name": "breast_hetero_host", "use_local_data": 0, 'drop': 0 } self.download_config = { "output_path": os.path.join(file_utils.get_project_base_directory(), "fate_flow/fate_flow_unittest_breast_b.csv"), "work_mode": WORK_MODE, "namespace": "fate_flow_test_breast_hetero", "table_name": "breast_hetero_guest" } self.server_url = "http://{}:{}/{}".format('127.0.0.1', HTTP_PORT, API_VERSION)
def get_proto_buffer_class(buffer_name): package_path = os.path.join(file_utils.get_project_base_directory(), 'federatedml', 'protobuf', 'generated') package_python_path = 'federatedml.protobuf.generated' for f in os.listdir(package_path): if f.startswith('.'): continue try: proto_module = importlib.import_module(package_python_path + '.' + f.rstrip('.py')) for name, obj in inspect.getmembers(proto_module): if inspect.isclass(obj) and name == buffer_name: return obj except Exception as e: stat_logger.warning(e) else: return None
def __init__(self, model_id, model_version): """ Support operations on FATE PipelinedModels TODO: add lock :param model_id: the model id stored at the local party. :param model_version: the model version. """ self.model_id = model_id self.model_version = model_version self.model_path = os.path.join(file_utils.get_project_base_directory(), "model_local_cache", model_id, model_version) self.define_proto_path = os.path.join(self.model_path, "define", "proto") self.define_meta_path = os.path.join(self.model_path, "define", "define_meta.yaml") self.variables_index_path = os.path.join(self.model_path, "variables", "index") self.variables_data_path = os.path.join(self.model_path, "variables", "data") self.default_archive_format = "zip"
def start_workflow(job_id, module, role): _config = request.json _job_dir = get_job_directory(job_id) _party_id = str(_config['local']['party_id']) _method = _config['WorkFlowParam']['method'] default_runtime_dict = file_utils.load_json_conf( 'workflow/conf/default_runtime_conf.json') fill_runtime_conf_table_info(runtime_conf=_config, default_runtime_conf=default_runtime_dict) conf_file_path = new_runtime_conf(job_dir=_job_dir, method=_method, module=module, role=role, party_id=_party_id) with open(conf_file_path, 'w+') as f: f.truncate() f.write(json.dumps(_config, indent=4)) f.flush() progs = [ "python3", os.path.join(file_utils.get_project_base_directory(), _config['CodePath']), "-j", job_id, "-c", os.path.abspath(conf_file_path) ] p = run_subprocess(job_dir=_job_dir, job_role=role, progs=progs) job_status = "start" job_data = dict() job_data["begin_date"] = datetime.datetime.now() job_data["status"] = job_status job_data.update(_config) job_data["pid"] = p.pid job_data["all_party"] = json.dumps(_config.get("role", {})) job_data["initiator"] = _config.get("JobParam", {}).get("initiator") save_job_info(job_id=job_id, role=_config.get("local", {}).get("role"), party_id=_config.get("local", {}).get("party_id"), save_info=job_data, create=True) update_job_queue(job_id=job_id, role=role, party_id=_party_id, save_data={ "status": job_status, "pid": p.pid }) return get_json_result(data={'pid': p.pid}, job_id=job_id)
def get_schedule_logger(job_id): job_log_dir = os.path.join(file_utils.get_project_base_directory(), 'logs', job_id) os.makedirs(job_log_dir, exist_ok=True) logger = logging.getLogger(job_id) logger.setLevel(LoggerFactory.LEVEL) handler = LoggerFactory.get_handler(class_name=None, level=LoggerFactory.LEVEL, log_dir=job_log_dir, is_schedule=True) error_handler = LoggerFactory.get_handler(class_name=None, level=logging.ERROR, log_dir=job_log_dir, is_schedule=True) logger.addHandler(handler) logger.addHandler(error_handler) with LoggerFactory.lock: LoggerFactory.schedule_logger_dict[job_id] = logger return logger
def __init__(self, job_id=None, eggroll_context=None): self.data_dir = os.path.join(file_utils.get_project_base_directory(), 'data') self.job_id = str( uuid.uuid1()) if job_id is None else "{}".format(job_id) self.meta_table = _DTable('__META__', '__META__', 'fragments', 10) self.pool = Executor() Standalone.__instance = self if not eggroll_context: eggroll_context = EggRollContext() self.eggroll_context = eggroll_context self.unique_id_template = '_EggRoll_%s_%s_%s_%.20f_%d' # todo: move to EggRollContext try: self.host_name = socket.gethostname() self.host_ip = socket.gethostbyname(self.host_name) except socket.gaierror as e: self.host_name = 'unknown' self.host_ip = 'unknown'