def get_schedule_logger(job_id='', log_type='schedule'): fate_flow_log_dir = os.path.join( file_utils.get_project_base_directory(), 'logs', 'fate_flow') job_log_dir = os.path.join(file_utils.get_project_base_directory(), 'logs', job_id) if not job_id: log_dirs = [fate_flow_log_dir] else: if log_type == 'audit': log_dirs = [job_log_dir, fate_flow_log_dir] else: log_dirs = [job_log_dir] os.makedirs(job_log_dir, exist_ok=True) os.makedirs(fate_flow_log_dir, exist_ok=True) logger = logging.getLogger('{}_{}'.format(job_id, log_type)) logger.setLevel(LoggerFactory.LEVEL) for job_log_dir in log_dirs: handler = LoggerFactory.get_handler(class_name=None, level=LoggerFactory.LEVEL, log_dir=job_log_dir, log_type=log_type) error_handler = LoggerFactory.get_handler(class_name=None, level=logging.ERROR, log_dir=job_log_dir, log_type=log_type) logger.addHandler(handler) logger.addHandler(error_handler) with LoggerFactory.lock: LoggerFactory.schedule_logger_dict[job_id + log_type] = logger return logger
def do_load_model(): request_data = request.json adapter_servings_config(request_data) retcode, retmsg = publish_model.load_model(config_data=request_data) try: if not retcode: with DB.connection_context(): model = MLModel.get_or_none(MLModel.f_role == request_data.get("local").get("role"), MLModel.f_party_id == request_data.get("local").get("party_id"), MLModel.f_model_id == request_data.get("job_parameters").get("model_id"), MLModel.f_model_version == request_data.get("job_parameters").get("model_version")) if model: count = model.f_loaded_times model.f_loaded_times = count + 1 model.save() except Exception as modify_err: stat_logger.exception(modify_err) try: party_model_id = gen_party_model_id(role=request_data.get("local").get("role"), party_id=request_data.get("local").get("party_id"), model_id=request_data.get("job_parameters").get("model_id")) src_model_path = os.path.join(file_utils.get_project_base_directory(), 'model_local_cache', party_model_id, request_data.get("job_parameters").get("model_version")) dst_model_path = os.path.join(file_utils.get_project_base_directory(), 'loaded_model_backup', party_model_id, request_data.get("job_parameters").get("model_version")) if not os.path.exists(dst_model_path): shutil.copytree(src=src_model_path, dst=dst_model_path) except Exception as copy_err: stat_logger.exception(copy_err) operation_record(request_data, "load", "success" if not retcode else "failed") return get_json_result(retcode=retcode, retmsg=retmsg)
def set_directory(directory=None, parent_log_dir=None, append_to_parent_log=None, force=False): if parent_log_dir: LoggerFactory.PARENT_LOG_DIR = parent_log_dir if append_to_parent_log: LoggerFactory.append_to_parent_log = append_to_parent_log with LoggerFactory.lock: if not directory: directory = os.path.join( file_utils.get_project_base_directory(), 'logs') if not LoggerFactory.LOG_DIR or force: LoggerFactory.LOG_DIR = directory os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True) for loggerName, ghandler in LoggerFactory.global_handler_dict.items( ): for className, (logger, handler) in LoggerFactory.logger_dict.items(): logger.removeHandler(ghandler) ghandler.close() LoggerFactory.global_handler_dict = {} for className, (logger, handler) in LoggerFactory.logger_dict.items(): logger.removeHandler(handler) _hanlder = None if handler: handler.close() if className != "default": _hanlder = LoggerFactory.get_handler(className) logger.addHandler(_hanlder) LoggerFactory.assemble_global_handler(logger) LoggerFactory.logger_dict[className] = logger, _hanlder
def update_config(key, value, conf_name=SERVICE_CONF): conf_path = conf_realpath(conf_name=conf_name) if not os.path.isabs(conf_path): conf_path = os.path.join(file_utils.get_project_base_directory(), conf_path) with filelock.FileLock(os.path.join(os.path.dirname(conf_path), ".lock")): config = file_utils.load_yaml_conf(conf_path=conf_path) or dict() config[key] = value file_utils.rewrite_yaml_conf(conf_path=conf_path, config=config)
def __init__(self): database_config = DATABASE.copy() db_name = database_config.pop("name") if is_standalone: from playhouse.apsw_ext import APSWDatabase self.database_connection = APSWDatabase( file_utils.get_project_base_directory("fate_sqlite.db")) else: from playhouse.pool import PooledMySQLDatabase self.database_connection = PooledMySQLDatabase( db_name, **database_config)
def query_model_info_from_file(model_id=None, model_version=None, role=None, party_id=None, query_filters=None, to_dict=False, **kwargs): res = {} if to_dict else [] model_dir = os.path.join(get_project_base_directory(), 'model_local_cache') glob_dir = f"{model_dir}{os.sep}{role if role else '*'}#{party_id if party_id else '*'}#{model_id if model_id else '*'}{os.sep}{model_version if model_version else '*'}" stat_logger.info(f'glob model dir: {glob_dir}') model_fp_list = glob.glob(glob_dir) if model_fp_list: for fp in model_fp_list: pipeline_model = PipelinedModel(model_id=fp.split('/')[-2], model_version=fp.split('/')[-1]) model_info = gather_model_info_data(pipeline_model, query_filters=query_filters) if model_info: if isinstance(res, dict): res[fp] = model_info else: res.append(model_info) if kwargs.get('save'): try: insert_info = gather_model_info_data( pipeline_model).copy() insert_info['role'] = fp.split('/')[-2].split('#')[0] insert_info['party_id'] = fp.split('/')[-2].split( '#')[1] insert_info['job_id'] = insert_info.get( 'f_model_version') insert_info[ 'size'] = pipeline_model.calculate_model_file_size( ) if compare_version(insert_info['f_fate_version'], '1.5.1') == 'lt': insert_info['roles'] = insert_info.get( 'f_train_runtime_conf', {}).get('role', {}) insert_info['initiator_role'] = insert_info.get( 'f_train_runtime_conf', {}).get('initiator', {}).get('role') insert_info[ 'initiator_party_id'] = insert_info.get( 'f_train_runtime_conf', {}).get('initiator', {}).get('party_id') save_model_info(insert_info) except Exception as e: stat_logger.exception(e) if res: return 0, 'Query model info from local model success.', res return 100, 'Query model info failed, cannot find model from local model files.', res
def setUp(self): self.data_dir = os.path.join(file_utils.get_project_base_directory(), "examples", "data") self.upload_guest_config = { "file": os.path.join(self.data_dir, "breast_hetero_guest.csv"), "head": 1, "partition": 10, "work_mode": WORK_MODE, "namespace": "experiment", "table_name": "breast_hetero_guest", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ',', } self.upload_host_config = { "file": os.path.join(self.data_dir, "breast_hetero_host.csv"), "head": 1, "partition": 10, "work_mode": WORK_MODE, "namespace": "experiment", "table_name": "breast_hetero_host", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ',', } self.download_config = { "output_path": os.path.join(file_utils.get_project_base_directory(), "fate_flow/fate_flow_unittest_breast_b.csv"), "work_mode": WORK_MODE, "namespace": "experiment", "table_name": "breast_hetero_guest" } ip = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get("host") self.server_url = "http://{}:{}/{}".format(ip, HTTP_PORT, API_VERSION)
def __init__(self): database_config = DATABASE.copy() db_name = database_config.pop("name") if WORK_MODE == WorkMode.STANDALONE: self.database_connection = APSWDatabase( os.path.join(file_utils.get_project_base_directory(), 'fate_flow_sqlite.db')) elif WORK_MODE == WorkMode.CLUSTER: self.database_connection = PooledMySQLDatabase( db_name, **database_config) else: raise Exception('can not init database')
def setUp(self): self.data_dir = os.path.join(file_utils.get_project_base_directory(), "examples", "data") self.upload_config = { "file": os.path.join(self.data_dir, "breast_hetero_guest.csv"), "head": 1, "partition": 10, "work_mode": WORK_MODE, "namespace": "fate_flow_test_table_breast_hetero", "table_name": "breast_hetero_guest", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ',' }
def __init__(self): database_config = DATABASE.copy() db_name = database_config.pop("name") if IS_STANDALONE: from playhouse.apsw_ext import APSWDatabase self.database_connection = APSWDatabase( file_utils.get_project_base_directory("fate_sqlite.db")) RuntimeConfig.init_config(USE_LOCAL_DATABASE=True) stat_logger.info( 'init sqlite database on standalone mode successfully') else: self.database_connection = PooledMySQLDatabase( db_name, **database_config) stat_logger.info( 'init mysql database on cluster mode successfully')
def write_component_model(self, component_model): for storage_path, buffer_object_serialized_string in component_model.get( "buffer").items(): storage_path = file_utils.get_project_base_directory( ) + storage_path os.makedirs(os.path.dirname(storage_path), exist_ok=True) with open(storage_path, "wb") as fw: fw.write( base64.b64decode(buffer_object_serialized_string.encode())) self.update_component_meta( component_name=component_model["component_name"], component_module_name=component_model["component_module_name"], model_alias=component_model["model_alias"], model_proto_index=component_model["model_proto_index"]) stat_logger.info("Save {} {} successfully".format( component_model["component_name"], component_model["model_alias"]))
def save_component_model(self, component_name, component_module_name, model_alias, model_buffers, tracker_client=None): model_proto_index = {} component_model = {"buffer": {}} component_model_storage_path = os.path.join(self.variables_data_path, component_name, model_alias) if not tracker_client: os.makedirs(component_model_storage_path, exist_ok=True) for model_name, buffer_object in model_buffers.items(): storage_path = os.path.join(component_model_storage_path, model_name) buffer_object_serialized_string = buffer_object.SerializeToString() if not buffer_object_serialized_string: fill_message = default_empty_fill_pb2.DefaultEmptyFillMessage() fill_message.flag = 'set' buffer_object_serialized_string = fill_message.SerializeToString( ) if not tracker_client: with self.lock, open(storage_path, "wb") as fw: fw.write(buffer_object_serialized_string) else: component_model["buffer"][storage_path.replace(file_utils.get_project_base_directory(), "")] = \ base64.b64encode(buffer_object_serialized_string).decode() model_proto_index[model_name] = type( buffer_object ).__name__ # index of model name and proto buffer class name stat_logger.info("Save {} {} {} buffer".format( component_name, model_alias, model_name)) if not tracker_client: self.update_component_meta( component_name=component_name, component_module_name=component_module_name, model_alias=model_alias, model_proto_index=model_proto_index) stat_logger.info("Save {} {} successfully".format( component_name, model_alias)) else: component_model["component_name"] = component_name component_model["component_module_name"] = component_module_name component_model["model_alias"] = model_alias component_model["model_proto_index"] = model_proto_index tracker_client.save_component_output_model(component_model)
def _put_all(self, kv_list, **kwargs): id_name, feature_name_list, id_delimiter = self.get_id_feature_name() create_table = "create table if not exists {}(k varchar(128) NOT NULL, v string) row format delimited fields terminated by" \ " '{}'".format(self._address.name, id_delimiter) self._cur.execute(create_table) # load local file or hdfs file temp_path = os.path.join(get_project_base_directory(), 'temp_data', uuid.uuid1().hex) os.makedirs(os.path.dirname(temp_path), exist_ok=True) with open(temp_path, 'w') as f: for k, v in kv_list: f.write(hive_utils.serialize_line(k, v)) sql = "load data local inpath '{}' into table {}".format( temp_path, self._address.name) self._cur.execute(sql) self._con.commit() os.remove(temp_path)
def __init__(self): database_config = DATABASE.copy() db_name = database_config.pop("name") if WORK_MODE == WorkMode.STANDALONE: self.database_connection = APSWDatabase( os.path.join(file_utils.get_project_base_directory(), 'fate_flow_sqlite.db')) RuntimeConfig.init_config(USE_LOCAL_DATABASE=True) stat_logger.info( 'init sqlite database on standalone mode successfully') elif WORK_MODE == WorkMode.CLUSTER: self.database_connection = PooledMySQLDatabase( db_name, **database_config) stat_logger.info( 'init mysql database on cluster mode successfully') RuntimeConfig.init_config(USE_LOCAL_DATABASE=False) else: raise Exception('can not init database')
def get_predict_conf(): request_data = request.json required_parameters = ['model_id', 'model_version'] check_config(request_data, required_parameters) model_dir = os.path.join(get_project_base_directory(), 'model_local_cache') model_fp_list = glob.glob( model_dir + f"/guest#*#{request_data['model_id']}/{request_data['model_version']}") if model_fp_list: fp = model_fp_list[0] pipeline_model = PipelinedModel(model_id=fp.split('/')[-2], model_version=fp.split('/')[-1]) pipeline = pipeline_model.read_component_model('pipeline', 'pipeline')['Pipeline'] predict_dsl = json_loads(pipeline.inference_dsl) train_runtime_conf = json_loads(pipeline.train_runtime_conf) parser = schedule_utils.get_dsl_parser_by_version( train_runtime_conf.get('dsl_version', '1')) predict_conf = parser.generate_predict_conf_template( predict_dsl=predict_dsl, train_conf=train_runtime_conf, model_id=request_data['model_id'], model_version=request_data['model_version']) else: predict_conf = '' if predict_conf: if request_data.get("filename"): os.makedirs(TEMP_DIRECTORY, exist_ok=True) temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json_dumps(predict_conf, indent=4)) return send_file(open(temp_filepath, "rb"), as_attachment=True, attachment_filename=request_data.get("filename")) else: return get_json_result(data=predict_conf) return error_response( 210, "No model found, please check if arguments are specified correctly.")
def load(cls): path = Path( file_utils.get_project_base_directory()) / 'conf' / SERVICE_CONF conf = file_utils.load_yaml_conf(path) if not isinstance(conf, dict): raise ValueError('invalid config file') local_path = path.with_name(f'local.{SERVICE_CONF}') if local_path.exists(): local_conf = file_utils.load_yaml_conf(local_path) if not isinstance(local_conf, dict): raise ValueError('invalid local config file') conf.update(local_conf) cls.LINKIS_SPARK_CONFIG = conf.get('fate_on_spark', {}).get('linkis_spark') for k, v in conf.items(): if isinstance(v, dict): setattr(cls, k.upper(), v)
def __init__(self, model_id, model_version): """ Support operations on FATE PipelinedModels TODO: add lock :param model_id: the model id stored at the local party. :param model_version: the model version. """ self.model_id = model_id self.model_version = model_version self.model_path = os.path.join(file_utils.get_project_base_directory(), "model_local_cache", model_id, model_version) self.define_proto_path = os.path.join(self.model_path, "define", "proto") self.define_meta_path = os.path.join(self.model_path, "define", "define_meta.yaml") self.variables_index_path = os.path.join(self.model_path, "variables", "index") self.variables_data_path = os.path.join(self.model_path, "variables", "data") self.default_archive_format = "zip"
def run(self, component_parameters=None, args=None): self.parameters = component_parameters["UploadParam"] LOGGER.info(self.parameters) LOGGER.info(args) self.parameters["role"] = component_parameters["role"] self.parameters["local"] = component_parameters["local"] storage_engine = self.parameters["storage_engine"] storage_address = self.parameters["storage_address"] # if not set storage, use job storage as default if not storage_engine: storage_engine = args["job_parameters"].storage_engine if not storage_address: storage_address = args["job_parameters"].engines_address[ EngineType.STORAGE] job_id = self.task_version_id.split("_")[0] if not os.path.isabs(self.parameters.get("file", "")): self.parameters["file"] = os.path.join( file_utils.get_project_base_directory(), self.parameters["file"]) if not os.path.exists(self.parameters["file"]): raise Exception("%s is not exist, please check the configure" % (self.parameters["file"])) if not os.path.getsize(self.parameters["file"]): raise Exception("%s is an empty file" % (self.parameters["file"])) name, namespace = self.parameters.get("name"), self.parameters.get( "namespace") _namespace, _table_name = self.generate_table_name( self.parameters["file"]) if namespace is None: namespace = _namespace if name is None: name = _table_name read_head = self.parameters['head'] if read_head == 0: head = False elif read_head == 1: head = True else: raise Exception("'head' in conf.json should be 0 or 1") partitions = self.parameters["partition"] if partitions <= 0 or partitions >= self.MAX_PARTITIONS: raise Exception( "Error number of partition, it should between %d and %d" % (0, self.MAX_PARTITIONS)) with storage.Session.build(session_id=job_utils.generate_session_id( self.tracker.task_id, self.tracker.task_version, self.tracker.role, self.tracker.party_id, suffix="storage", random_end=True), namespace=namespace, name=name) as storage_session: if self.parameters.get("destroy", False): table = storage_session.get_table() if table: LOGGER.info( f"destroy table name: {name} namespace: {namespace} engine: {table.get_engine()}" ) table.destroy() else: LOGGER.info( f"can not found table name: {name} namespace: {namespace}, pass destroy" ) address_dict = storage_address.copy() with storage.Session.build( session_id=job_utils.generate_session_id( self.tracker.task_id, self.tracker.task_version, self.tracker.role, self.tracker.party_id, suffix="storage", random_end=True), storage_engine=storage_engine, options=self.parameters.get("options")) as storage_session: if storage_engine in { StorageEngine.EGGROLL, StorageEngine.STANDALONE }: upload_address = { "name": name, "namespace": namespace, "storage_type": EggRollStorageType.ROLLPAIR_LMDB } elif storage_engine in {StorageEngine.MYSQL}: upload_address = {"db": namespace, "name": name} elif storage_engine in {StorageEngine.HDFS}: upload_address = { "path": data_utils.default_input_fs_path( name=name, namespace=namespace, prefix=address_dict.get("path_prefix")) } else: raise RuntimeError( f"can not support this storage engine: {storage_engine}") address_dict.update(upload_address) LOGGER.info( f"upload to {storage_engine} storage, address: {address_dict}") address = storage.StorageTableMeta.create_address( storage_engine=storage_engine, address_dict=address_dict) self.parameters["partitions"] = partitions self.parameters["name"] = name self.table = storage_session.create_table(address=address, **self.parameters) data_table_count = self.save_data_table(job_id, name, namespace, head) self.table.get_meta().update_metas(in_serialized=True) LOGGER.info("------------load data finish!-----------------") # rm tmp file try: if '{}/fate_upload_tmp'.format(job_id) in self.parameters['file']: LOGGER.info("remove tmp upload file") LOGGER.info(os.path.dirname(self.parameters["file"])) shutil.rmtree(os.path.dirname(self.parameters["file"])) except: LOGGER.info("remove tmp file failed") LOGGER.info("file: {}".format(self.parameters["file"])) LOGGER.info("total data_count: {}".format(data_table_count)) LOGGER.info("table name: {}, table namespace: {}".format( name, namespace))
import os from fate_arch.computing import ComputingEngine from fate_arch.federation import FederationEngine from fate_arch.storage import StorageEngine from fate_arch.common import file_utils, log, EngineType from fate_flow.entity.runtime_config import RuntimeConfig from fate_arch.common.conf_utils import get_base_config import __main__ # Server API_VERSION = "v1" FATEFLOW_SERVICE_NAME = "fateflow" MAIN_MODULE = os.path.relpath(__main__.__file__) SERVER_MODULE = "fate_flow_server.py" TEMP_DIRECTORY = os.path.join(file_utils.get_project_base_directory(), "temp", "fate_flow") HEADERS = { "Content-Type": "application/json", "Connection": "close", "service": FATEFLOW_SERVICE_NAME } _ONE_DAY_IN_SECONDS = 60 * 60 * 24 GRPC_SERVER_MAX_WORKERS = None IP = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") HTTP_PORT = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("http_port") GRPC_PORT = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("grpc_port") WORK_MODE = get_base_config("work_mode", 0) DATABASE = get_base_config("database", {})
def default_localfs_path(name, namespace, data_type): return os.path.join(file_utils.get_project_base_directory(), 'localfs', data_type, namespace, name)
STANDALONE_BACKEND_VIRTUAL_CORES_PER_NODE = 20 _ONE_DAY_IN_SECONDS = 60 * 60 * 24 # abnormal condition parameter DEFAULT_GRPC_OVERALL_TIMEOUT = 2 * 60 * 1000 # ms DEFAULT_FEDERATED_COMMAND_TRYS = 3 JOB_DEFAULT_TIMEOUT = 3 * 24 * 60 * 60 JOB_START_TIMEOUT = 60 * 1000 # ms ''' Constants ''' API_VERSION = "v1" FATEFLOW_SERVICE_NAME = 'fateflow' MAIN_MODULE = os.path.relpath(__main__.__file__) SERVER_MODULE = 'fate_flow_server.py' TEMP_DIRECTORY = os.path.join(file_utils.get_project_base_directory(), "temp", "fate_flow") HEADERS = {'Content-Type': 'application/json', 'Connection': 'close'} # endpoint FATE_FLOW_MODEL_TRANSFER_ENDPOINT = '/v1/model/transfer' FATE_MANAGER_GET_NODE_INFO_ENDPOINT = '/fate-manager/api/site/secretinfo' FATE_MANAGER_NODE_CHECK_ENDPOINT = '/fate-manager/api/site/checksite' FATE_BOARD_DASHBOARD_ENDPOINT = '/index.html#/dashboard?job_id={}&role={}&party_id={}' # logger log.LoggerFactory.LEVEL = 10 # {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0} log.LoggerFactory.set_directory( os.path.join(file_utils.get_project_base_directory(), 'logs', 'fate_flow')) stat_logger = log.getLogger("fate_flow_stat")
def call_fun(func, config_data, dsl_path, config_path): ip = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get("host") http_port = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get( "http_port") server_url = "http://{}:{}/{}".format(ip, http_port, API_VERSION) if func in JOB_OPERATE_FUNC: if func == 'submit_job': if not config_path: raise Exception( 'the following arguments are required: {}'.format( 'runtime conf path')) dsl_data = {} if dsl_path or config_data.get('job_parameters', {}).get( 'job_type', '') == 'predict': if dsl_path: dsl_path = os.path.abspath(dsl_path) with open(dsl_path, 'r') as f: dsl_data = json.load(f) else: raise Exception( 'the following arguments are required: {}'.format( 'dsl path')) post_data = {'job_dsl': dsl_data, 'job_runtime_conf': config_data} response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) except: pass elif func == 'data_view_query' or func == 'clean_queue': response = requests.post("/".join( [server_url, "job", func.replace('_', '/')]), json=config_data) else: if func != 'query_job': detect_utils.check_config(config=config_data, required_arguments=['job_id']) post_data = config_data response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) if func == 'query_job': response = response.json() if response['retcode'] == 0: for i in range(len(response['data'])): del response['data'][i]['f_runtime_conf'] del response['data'][i]['f_dsl'] elif func in JOB_FUNC: if func == 'job_config': detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'role', 'party_id', 'output_path' ]) response = requests.post("/".join( [server_url, func.replace('_', '/')]), json=config_data) response_data = response.json() if response_data['retcode'] == 0: job_id = response_data['data']['job_id'] download_directory = os.path.join( config_data['output_path'], 'job_{}_config'.format(job_id)) os.makedirs(download_directory, exist_ok=True) for k, v in response_data['data'].items(): if k == 'job_id': continue with open('{}/{}.json'.format(download_directory, k), 'w') as fw: json.dump(v, fw, indent=4) del response_data['data']['dsl'] del response_data['data']['runtime_conf'] response_data['directory'] = download_directory response_data[ 'retmsg'] = 'download successfully, please check {} directory'.format( download_directory) response = response_data elif func == 'job_log': detect_utils.check_config( config=config_data, required_arguments=['job_id', 'output_path']) job_id = config_data['job_id'] tar_file_name = 'job_{}_log.tar.gz'.format(job_id) extract_dir = os.path.join(config_data['output_path'], 'job_{}_log'.format(job_id)) with closing( requests.get("/".join([server_url, func.replace('_', '/')]), json=config_data, stream=True)) as response: if response.status_code == 200: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'. format(extract_dir) } else: response = response.json() elif func in TASK_OPERATE_FUNC: response = requests.post("/".join( [server_url, "job", "task", func.rstrip('_task')]), json=config_data) elif func in TRACKING_FUNC: if func != 'component_metric_delete': detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'component_name', 'role', 'party_id' ]) if func == 'component_output_data': detect_utils.check_config(config=config_data, required_arguments=['output_path']) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format( config_data['job_id'], config_data['component_name'], config_data['role'], config_data['party_id']) extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', '')) with closing( requests.get("/".join([ server_url, "tracking", func.replace('_', '/'), 'download' ]), json=config_data, stream=True)) as response: if response.status_code == 200: try: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'. format(extract_dir) } except: response = { 'retcode': 100, 'retmsg': 'download failed, please check if the parameters are correct' } else: response = response.json() else: response = requests.post("/".join( [server_url, "tracking", func.replace('_', '/')]), json=config_data) elif func in DATA_FUNC: if func == 'upload' and config_data.get('use_local_data', 1) != 0: file_name = config_data.get('file') if not os.path.isabs(file_name): file_name = os.path.join( file_utils.get_project_base_directory(), file_name) if os.path.exists(file_name): with open(file_name, 'rb') as fp: data = MultipartEncoder( fields={ 'file': (os.path.basename(file_name), fp, 'application/octet-stream') }) tag = [0] def read_callback(monitor): if config_data.get('verbose') == 1: sys.stdout.write("\r UPLOADING:{0}{1}".format( "|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len))) sys.stdout.flush() if monitor.bytes_read / monitor.len == 1: tag[0] += 1 if tag[0] == 2: sys.stdout.write('\n') data = MultipartEncoderMonitor(data, read_callback) response = requests.post( "/".join([server_url, "data", func.replace('_', '/')]), data=data, params=json.dumps(config_data), headers={'Content-Type': data.content_type}) else: raise Exception( 'The file is obtained from the fate flow client machine, but it does not exist, ' 'please check the path: {}'.format(file_name)) else: response = requests.post("/".join( [server_url, "data", func.replace('_', '/')]), json=config_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join([server_url, "data", func]), json=config_data) except: pass elif func in TABLE_FUNC: if func == "table_info": detect_utils.check_config( config=config_data, required_arguments=['namespace', 'table_name']) response = requests.post("/".join([server_url, "table", func]), json=config_data) else: response = requests.post("/".join( [server_url, func.replace('_', '/')]), json=config_data) elif func in MODEL_FUNC: if func == "import": file_path = config_data["file"] if not os.path.isabs(file_path): file_path = os.path.join( file_utils.get_project_base_directory(), file_path) if os.path.exists(file_path): files = {'file': open(file_path, 'rb')} else: raise Exception( 'The file is obtained from the fate flow client machine, but it does not exist, ' 'please check the path: {}'.format(file_path)) response = requests.post("/".join([server_url, "model", func]), data=config_data, files=files) elif func == "export": with closing( requests.get("/".join([server_url, "model", func]), json=config_data, stream=True)) as response: if response.status_code == 200: archive_file_name = re.findall( "filename=(.+)", response.headers["Content-Disposition"])[0] os.makedirs(config_data["output_path"], exist_ok=True) archive_file_path = os.path.join( config_data["output_path"], archive_file_name) with open(archive_file_path, 'wb') as fw: for chunk in response.iter_content(1024): if chunk: fw.write(chunk) response = { 'retcode': 0, 'file': archive_file_path, 'retmsg': 'download successfully, please check {}'.format( archive_file_path) } else: response = response.json() else: response = requests.post("/".join([server_url, "model", func]), json=config_data) elif func in PERMISSION_FUNC: detect_utils.check_config( config=config_data, required_arguments=['src_party_id', 'src_role']) response = requests.post("/".join( [server_url, "permission", func.replace('_', '/')]), json=config_data) return response.json() if isinstance( response, requests.models.Response) else response
def conf_realpath(conf_name): conf_path = f"conf/{conf_name}" return os.path.join(file_utils.get_project_base_directory(), conf_path)
def upload_dependencies_to_hadoop(cls, provider, dependence_type, storage_engine=FateDependenceStorageEngine.HDFS.value): LOGGER.info(f'upload {dependence_type} dependencies to hadoop') LOGGER.info(f'dependencies loading ...') if dependence_type == FateDependenceName.Python_Env.value: # todo: version python env target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "python_env.zip") source_path = os.path.dirname(os.path.dirname(os.getenv("VIRTUAL_ENV"))) cls.rewrite_pyvenv_cfg(os.path.join(os.getenv("VIRTUAL_ENV"), "pyvenv.cfg"), "python_env") env_dir_list = ["python", "miniconda3"] cls.zip_dir(source_path, target_file, env_dir_list) dependencies_conf = {"executor_python": f"./{dependence_type}/python/venv/bin/python", "driver_python": f"{os.path.join(os.getenv('VIRTUAL_ENV'), 'bin', 'python')}"} else: fate_code_dependencies = { "fate_flow": get_fate_flow_python_directory("fate_flow"), "fate_arch": file_utils.get_fate_python_directory("fate_arch"), "conf": file_utils.get_project_base_directory("conf") } fate_flow_snapshot_time = DependenceRegistry.get_modify_time(fate_code_dependencies["fate_flow"]) fate_code_base_dir = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate_code", "fate") python_base_dir = os.path.join(fate_code_base_dir, "python") if os.path.exists(os.path.dirname(python_base_dir)): shutil.rmtree(os.path.dirname(python_base_dir)) for key, path in fate_code_dependencies.items(): cls.copy_dir(path, os.path.join(python_base_dir, key)) if key == "conf": cls.move_dir(os.path.join(python_base_dir, key), os.path.dirname(fate_code_base_dir)) if provider.name == ComponentProviderName.FATE.value: source_path = provider.path else: source_path = ComponentProviderInfo.get_or_none( ComponentProviderInfo.f_version == provider.version, ComponentProviderInfo.f_provider_name == ComponentProviderName.FATE.value ).f_path cls.copy_dir(source_path, os.path.join(python_base_dir, "federatedml")) target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate.zip") cls.zip_dir(os.path.dirname(fate_code_base_dir), target_file) dependencies_conf = {"executor_env_pythonpath": f"./{dependence_type}/fate/python:$PYTHONPATH"} LOGGER.info(f'dependencies loading success') LOGGER.info(f'start upload') snapshot_time = DependenceRegistry.get_modify_time(source_path) storage_dir = f"/fate_dependence/{provider.version}" os.system(f" {os.getenv('HADOOP_HOME')}/bin/hdfs dfs -mkdir -p {storage_dir}") status = os.system(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put -f {target_file} {storage_dir}") LOGGER.info(f'upload end, status is {status}') if status == 0: storage_path = os.path.join(storage_dir, os.path.basename(target_file)) storage_meta = { "f_storage_engine": storage_engine, "f_type": dependence_type, "f_version": provider.version, "f_storage_path": storage_path, "f_snapshot_time": snapshot_time, "f_fate_flow_snapshot_time": fate_flow_snapshot_time if dependence_type == FateDependenceName.Fate_Source_Code.value else None, "f_dependencies_conf": {"archives": "#".join([storage_path, dependence_type])}, "f_upload_status": False, "f_pid": 0 } storage_meta["f_dependencies_conf"].update(dependencies_conf) DependenceRegistry.save_dependencies_storage_meta(storage_meta) else: raise Exception(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put {target_file} {storage_dir} failed status: {status}") return storage_meta
def gen_model_file_path(model_id, model_version): return os.path.join(get_project_base_directory(), "model_local_cache", model_id, model_version)
def get_versions() -> typing.Mapping[str, typing.Any]: return dotenv.dotenv_values( dotenv_path=os.path.join(get_project_base_directory(), "fate.env"))
def get_job_log_directory(job_id): return os.path.join(file_utils.get_project_base_directory(), 'logs', job_id)
def set_model_path(self): self.model_path = os.path.join(file_utils.get_project_base_directory(), "model_local_cache", self.model_id, self.model_version)
) return _meta_table # noinspection PyProtectedMember def _get_from_meta_table(key): return _get_meta_table().get(key) # noinspection PyProtectedMember def _put_to_meta_table(key, value): _get_meta_table().put(key, value) _data_dir = Path( file_utils.get_project_base_directory()).joinpath("data").absolute() def _get_data_dir(): return _data_dir def _get_storage_dir(*args): return _data_dir.joinpath(*args) async def _check_status_and_get_value(get_func, key): value = get_func(key) while value is None: await asyncio.sleep(0.1) value = get_func(key)