def get_schedule_logger(job_id='', log_type='schedule'):
fate_flow_log_dir = os.path.join(
file_utils.get_project_base_directory(), 'logs', 'fate_flow')
job_log_dir = os.path.join(file_utils.get_project_base_directory(),
'logs', job_id)
if not job_id:
log_dirs = [fate_flow_log_dir]
else:
if log_type == 'audit':
log_dirs = [job_log_dir, fate_flow_log_dir]
else:
log_dirs = [job_log_dir]
os.makedirs(job_log_dir, exist_ok=True)
os.makedirs(fate_flow_log_dir, exist_ok=True)
logger = logging.getLogger('{}_{}'.format(job_id, log_type))
logger.setLevel(LoggerFactory.LEVEL)
for job_log_dir in log_dirs:
handler = LoggerFactory.get_handler(class_name=None,
level=LoggerFactory.LEVEL,
log_dir=job_log_dir,
log_type=log_type)
error_handler = LoggerFactory.get_handler(class_name=None,
level=logging.ERROR,
log_dir=job_log_dir,
log_type=log_type)
logger.addHandler(handler)
logger.addHandler(error_handler)
with LoggerFactory.lock:
LoggerFactory.schedule_logger_dict[job_id + log_type] = logger
return logger
def do_load_model():
request_data = request.json
adapter_servings_config(request_data)
retcode, retmsg = publish_model.load_model(config_data=request_data)
try:
if not retcode:
with DB.connection_context():
model = MLModel.get_or_none(MLModel.f_role == request_data.get("local").get("role"),
MLModel.f_party_id == request_data.get("local").get("party_id"),
MLModel.f_model_id == request_data.get("job_parameters").get("model_id"),
MLModel.f_model_version == request_data.get("job_parameters").get("model_version"))
if model:
count = model.f_loaded_times
model.f_loaded_times = count + 1
model.save()
except Exception as modify_err:
stat_logger.exception(modify_err)
try:
party_model_id = gen_party_model_id(role=request_data.get("local").get("role"),
party_id=request_data.get("local").get("party_id"),
model_id=request_data.get("job_parameters").get("model_id"))
src_model_path = os.path.join(file_utils.get_project_base_directory(), 'model_local_cache', party_model_id,
request_data.get("job_parameters").get("model_version"))
dst_model_path = os.path.join(file_utils.get_project_base_directory(), 'loaded_model_backup',
party_model_id, request_data.get("job_parameters").get("model_version"))
if not os.path.exists(dst_model_path):
shutil.copytree(src=src_model_path, dst=dst_model_path)
except Exception as copy_err:
stat_logger.exception(copy_err)
operation_record(request_data, "load", "success" if not retcode else "failed")
return get_json_result(retcode=retcode, retmsg=retmsg)
def set_directory(directory=None,
parent_log_dir=None,
append_to_parent_log=None,
force=False):
if parent_log_dir:
LoggerFactory.PARENT_LOG_DIR = parent_log_dir
if append_to_parent_log:
LoggerFactory.append_to_parent_log = append_to_parent_log
with LoggerFactory.lock:
if not directory:
directory = os.path.join(
file_utils.get_project_base_directory(), 'logs')
if not LoggerFactory.LOG_DIR or force:
LoggerFactory.LOG_DIR = directory
os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
for loggerName, ghandler in LoggerFactory.global_handler_dict.items(
):
for className, (logger,
handler) in LoggerFactory.logger_dict.items():
logger.removeHandler(ghandler)
ghandler.close()
LoggerFactory.global_handler_dict = {}
for className, (logger,
handler) in LoggerFactory.logger_dict.items():
logger.removeHandler(handler)
_hanlder = None
if handler:
handler.close()
if className != "default":
_hanlder = LoggerFactory.get_handler(className)
logger.addHandler(_hanlder)
LoggerFactory.assemble_global_handler(logger)
LoggerFactory.logger_dict[className] = logger, _hanlder
def update_config(key, value, conf_name=SERVICE_CONF):
conf_path = conf_realpath(conf_name=conf_name)
if not os.path.isabs(conf_path):
conf_path = os.path.join(file_utils.get_project_base_directory(), conf_path)
with filelock.FileLock(os.path.join(os.path.dirname(conf_path), ".lock")):
config = file_utils.load_yaml_conf(conf_path=conf_path) or dict()
config[key] = value
file_utils.rewrite_yaml_conf(conf_path=conf_path, config=config)
def __init__(self):
database_config = DATABASE.copy()
db_name = database_config.pop("name")
if is_standalone:
from playhouse.apsw_ext import APSWDatabase
self.database_connection = APSWDatabase(
file_utils.get_project_base_directory("fate_sqlite.db"))
else:
from playhouse.pool import PooledMySQLDatabase
self.database_connection = PooledMySQLDatabase(
db_name, **database_config)
def query_model_info_from_file(model_id=None,
model_version=None,
role=None,
party_id=None,
query_filters=None,
to_dict=False,
**kwargs):
res = {} if to_dict else []
model_dir = os.path.join(get_project_base_directory(), 'model_local_cache')
glob_dir = f"{model_dir}{os.sep}{role if role else '*'}#{party_id if party_id else '*'}#{model_id if model_id else '*'}{os.sep}{model_version if model_version else '*'}"
stat_logger.info(f'glob model dir: {glob_dir}')
model_fp_list = glob.glob(glob_dir)
if model_fp_list:
for fp in model_fp_list:
pipeline_model = PipelinedModel(model_id=fp.split('/')[-2],
model_version=fp.split('/')[-1])
model_info = gather_model_info_data(pipeline_model,
query_filters=query_filters)
if model_info:
if isinstance(res, dict):
res[fp] = model_info
else:
res.append(model_info)
if kwargs.get('save'):
try:
insert_info = gather_model_info_data(
pipeline_model).copy()
insert_info['role'] = fp.split('/')[-2].split('#')[0]
insert_info['party_id'] = fp.split('/')[-2].split(
'#')[1]
insert_info['job_id'] = insert_info.get(
'f_model_version')
insert_info[
'size'] = pipeline_model.calculate_model_file_size(
)
if compare_version(insert_info['f_fate_version'],
'1.5.1') == 'lt':
insert_info['roles'] = insert_info.get(
'f_train_runtime_conf', {}).get('role', {})
insert_info['initiator_role'] = insert_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('role')
insert_info[
'initiator_party_id'] = insert_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('party_id')
save_model_info(insert_info)
except Exception as e:
stat_logger.exception(e)
if res:
return 0, 'Query model info from local model success.', res
return 100, 'Query model info failed, cannot find model from local model files.', res
def setUp(self):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
"examples", "data")
self.upload_guest_config = {
"file": os.path.join(self.data_dir, "breast_hetero_guest.csv"),
"head": 1,
"partition": 10,
"work_mode": WORK_MODE,
"namespace": "experiment",
"table_name": "breast_hetero_guest",
"use_local_data": 0,
'drop': 1,
'backend': 0,
"id_delimiter": ',',
}
self.upload_host_config = {
"file": os.path.join(self.data_dir, "breast_hetero_host.csv"),
"head": 1,
"partition": 10,
"work_mode": WORK_MODE,
"namespace": "experiment",
"table_name": "breast_hetero_host",
"use_local_data": 0,
'drop': 1,
'backend': 0,
"id_delimiter": ',',
}
self.download_config = {
"output_path":
os.path.join(file_utils.get_project_base_directory(),
"fate_flow/fate_flow_unittest_breast_b.csv"),
"work_mode":
WORK_MODE,
"namespace":
"experiment",
"table_name":
"breast_hetero_guest"
}
ip = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get("host")
self.server_url = "http://{}:{}/{}".format(ip, HTTP_PORT, API_VERSION)
def __init__(self):
database_config = DATABASE.copy()
db_name = database_config.pop("name")
if WORK_MODE == WorkMode.STANDALONE:
self.database_connection = APSWDatabase(
os.path.join(file_utils.get_project_base_directory(),
'fate_flow_sqlite.db'))
elif WORK_MODE == WorkMode.CLUSTER:
self.database_connection = PooledMySQLDatabase(
db_name, **database_config)
else:
raise Exception('can not init database')
def setUp(self):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
"examples", "data")
self.upload_config = {
"file": os.path.join(self.data_dir, "breast_hetero_guest.csv"),
"head": 1,
"partition": 10,
"work_mode": WORK_MODE,
"namespace": "fate_flow_test_table_breast_hetero",
"table_name": "breast_hetero_guest",
"use_local_data": 0,
'drop': 1,
'backend': 0,
"id_delimiter": ','
}
def __init__(self):
database_config = DATABASE.copy()
db_name = database_config.pop("name")
if IS_STANDALONE:
from playhouse.apsw_ext import APSWDatabase
self.database_connection = APSWDatabase(
file_utils.get_project_base_directory("fate_sqlite.db"))
RuntimeConfig.init_config(USE_LOCAL_DATABASE=True)
stat_logger.info(
'init sqlite database on standalone mode successfully')
else:
self.database_connection = PooledMySQLDatabase(
db_name, **database_config)
stat_logger.info(
'init mysql database on cluster mode successfully')
def write_component_model(self, component_model):
for storage_path, buffer_object_serialized_string in component_model.get(
"buffer").items():
storage_path = file_utils.get_project_base_directory(
) + storage_path
os.makedirs(os.path.dirname(storage_path), exist_ok=True)
with open(storage_path, "wb") as fw:
fw.write(
base64.b64decode(buffer_object_serialized_string.encode()))
self.update_component_meta(
component_name=component_model["component_name"],
component_module_name=component_model["component_module_name"],
model_alias=component_model["model_alias"],
model_proto_index=component_model["model_proto_index"])
stat_logger.info("Save {} {} successfully".format(
component_model["component_name"], component_model["model_alias"]))
def save_component_model(self,
component_name,
component_module_name,
model_alias,
model_buffers,
tracker_client=None):
model_proto_index = {}
component_model = {"buffer": {}}
component_model_storage_path = os.path.join(self.variables_data_path,
component_name,
model_alias)
if not tracker_client:
os.makedirs(component_model_storage_path, exist_ok=True)
for model_name, buffer_object in model_buffers.items():
storage_path = os.path.join(component_model_storage_path,
model_name)
buffer_object_serialized_string = buffer_object.SerializeToString()
if not buffer_object_serialized_string:
fill_message = default_empty_fill_pb2.DefaultEmptyFillMessage()
fill_message.flag = 'set'
buffer_object_serialized_string = fill_message.SerializeToString(
)
if not tracker_client:
with self.lock, open(storage_path, "wb") as fw:
fw.write(buffer_object_serialized_string)
else:
component_model["buffer"][storage_path.replace(file_utils.get_project_base_directory(), "")] = \
base64.b64encode(buffer_object_serialized_string).decode()
model_proto_index[model_name] = type(
buffer_object
).__name__ # index of model name and proto buffer class name
stat_logger.info("Save {} {} {} buffer".format(
component_name, model_alias, model_name))
if not tracker_client:
self.update_component_meta(
component_name=component_name,
component_module_name=component_module_name,
model_alias=model_alias,
model_proto_index=model_proto_index)
stat_logger.info("Save {} {} successfully".format(
component_name, model_alias))
else:
component_model["component_name"] = component_name
component_model["component_module_name"] = component_module_name
component_model["model_alias"] = model_alias
component_model["model_proto_index"] = model_proto_index
tracker_client.save_component_output_model(component_model)
def _put_all(self, kv_list, **kwargs):
id_name, feature_name_list, id_delimiter = self.get_id_feature_name()
create_table = "create table if not exists {}(k varchar(128) NOT NULL, v string) row format delimited fields terminated by" \
" '{}'".format(self._address.name, id_delimiter)
self._cur.execute(create_table)
# load local file or hdfs file
temp_path = os.path.join(get_project_base_directory(), 'temp_data',
uuid.uuid1().hex)
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
with open(temp_path, 'w') as f:
for k, v in kv_list:
f.write(hive_utils.serialize_line(k, v))
sql = "load data local inpath '{}' into table {}".format(
temp_path, self._address.name)
self._cur.execute(sql)
self._con.commit()
os.remove(temp_path)
def __init__(self):
database_config = DATABASE.copy()
db_name = database_config.pop("name")
if WORK_MODE == WorkMode.STANDALONE:
self.database_connection = APSWDatabase(
os.path.join(file_utils.get_project_base_directory(),
'fate_flow_sqlite.db'))
RuntimeConfig.init_config(USE_LOCAL_DATABASE=True)
stat_logger.info(
'init sqlite database on standalone mode successfully')
elif WORK_MODE == WorkMode.CLUSTER:
self.database_connection = PooledMySQLDatabase(
db_name, **database_config)
stat_logger.info(
'init mysql database on cluster mode successfully')
RuntimeConfig.init_config(USE_LOCAL_DATABASE=False)
else:
raise Exception('can not init database')
def get_predict_conf():
request_data = request.json
required_parameters = ['model_id', 'model_version']
check_config(request_data, required_parameters)
model_dir = os.path.join(get_project_base_directory(), 'model_local_cache')
model_fp_list = glob.glob(
model_dir +
f"/guest#*#{request_data['model_id']}/{request_data['model_version']}")
if model_fp_list:
fp = model_fp_list[0]
pipeline_model = PipelinedModel(model_id=fp.split('/')[-2],
model_version=fp.split('/')[-1])
pipeline = pipeline_model.read_component_model('pipeline',
'pipeline')['Pipeline']
predict_dsl = json_loads(pipeline.inference_dsl)
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
parser = schedule_utils.get_dsl_parser_by_version(
train_runtime_conf.get('dsl_version', '1'))
predict_conf = parser.generate_predict_conf_template(
predict_dsl=predict_dsl,
train_conf=train_runtime_conf,
model_id=request_data['model_id'],
model_version=request_data['model_version'])
else:
predict_conf = ''
if predict_conf:
if request_data.get("filename"):
os.makedirs(TEMP_DIRECTORY, exist_ok=True)
temp_filepath = os.path.join(TEMP_DIRECTORY,
request_data.get("filename"))
with open(temp_filepath, "w") as fout:
fout.write(json_dumps(predict_conf, indent=4))
return send_file(open(temp_filepath, "rb"),
as_attachment=True,
attachment_filename=request_data.get("filename"))
else:
return get_json_result(data=predict_conf)
return error_response(
210,
"No model found, please check if arguments are specified correctly.")
def load(cls):
path = Path(
file_utils.get_project_base_directory()) / 'conf' / SERVICE_CONF
conf = file_utils.load_yaml_conf(path)
if not isinstance(conf, dict):
raise ValueError('invalid config file')
local_path = path.with_name(f'local.{SERVICE_CONF}')
if local_path.exists():
local_conf = file_utils.load_yaml_conf(local_path)
if not isinstance(local_conf, dict):
raise ValueError('invalid local config file')
conf.update(local_conf)
cls.LINKIS_SPARK_CONFIG = conf.get('fate_on_spark',
{}).get('linkis_spark')
for k, v in conf.items():
if isinstance(v, dict):
setattr(cls, k.upper(), v)
def __init__(self, model_id, model_version):
"""
Support operations on FATE PipelinedModels
TODO: add lock
:param model_id: the model id stored at the local party.
:param model_version: the model version.
"""
self.model_id = model_id
self.model_version = model_version
self.model_path = os.path.join(file_utils.get_project_base_directory(),
"model_local_cache", model_id,
model_version)
self.define_proto_path = os.path.join(self.model_path, "define",
"proto")
self.define_meta_path = os.path.join(self.model_path, "define",
"define_meta.yaml")
self.variables_index_path = os.path.join(self.model_path, "variables",
"index")
self.variables_data_path = os.path.join(self.model_path, "variables",
"data")
self.default_archive_format = "zip"
def run(self, component_parameters=None, args=None):
self.parameters = component_parameters["UploadParam"]
LOGGER.info(self.parameters)
LOGGER.info(args)
self.parameters["role"] = component_parameters["role"]
self.parameters["local"] = component_parameters["local"]
storage_engine = self.parameters["storage_engine"]
storage_address = self.parameters["storage_address"]
# if not set storage, use job storage as default
if not storage_engine:
storage_engine = args["job_parameters"].storage_engine
if not storage_address:
storage_address = args["job_parameters"].engines_address[
EngineType.STORAGE]
job_id = self.task_version_id.split("_")[0]
if not os.path.isabs(self.parameters.get("file", "")):
self.parameters["file"] = os.path.join(
file_utils.get_project_base_directory(),
self.parameters["file"])
if not os.path.exists(self.parameters["file"]):
raise Exception("%s is not exist, please check the configure" %
(self.parameters["file"]))
if not os.path.getsize(self.parameters["file"]):
raise Exception("%s is an empty file" % (self.parameters["file"]))
name, namespace = self.parameters.get("name"), self.parameters.get(
"namespace")
_namespace, _table_name = self.generate_table_name(
self.parameters["file"])
if namespace is None:
namespace = _namespace
if name is None:
name = _table_name
read_head = self.parameters['head']
if read_head == 0:
head = False
elif read_head == 1:
head = True
else:
raise Exception("'head' in conf.json should be 0 or 1")
partitions = self.parameters["partition"]
if partitions <= 0 or partitions >= self.MAX_PARTITIONS:
raise Exception(
"Error number of partition, it should between %d and %d" %
(0, self.MAX_PARTITIONS))
with storage.Session.build(session_id=job_utils.generate_session_id(
self.tracker.task_id,
self.tracker.task_version,
self.tracker.role,
self.tracker.party_id,
suffix="storage",
random_end=True),
namespace=namespace,
name=name) as storage_session:
if self.parameters.get("destroy", False):
table = storage_session.get_table()
if table:
LOGGER.info(
f"destroy table name: {name} namespace: {namespace} engine: {table.get_engine()}"
)
table.destroy()
else:
LOGGER.info(
f"can not found table name: {name} namespace: {namespace}, pass destroy"
)
address_dict = storage_address.copy()
with storage.Session.build(
session_id=job_utils.generate_session_id(
self.tracker.task_id,
self.tracker.task_version,
self.tracker.role,
self.tracker.party_id,
suffix="storage",
random_end=True),
storage_engine=storage_engine,
options=self.parameters.get("options")) as storage_session:
if storage_engine in {
StorageEngine.EGGROLL, StorageEngine.STANDALONE
}:
upload_address = {
"name": name,
"namespace": namespace,
"storage_type": EggRollStorageType.ROLLPAIR_LMDB
}
elif storage_engine in {StorageEngine.MYSQL}:
upload_address = {"db": namespace, "name": name}
elif storage_engine in {StorageEngine.HDFS}:
upload_address = {
"path":
data_utils.default_input_fs_path(
name=name,
namespace=namespace,
prefix=address_dict.get("path_prefix"))
}
else:
raise RuntimeError(
f"can not support this storage engine: {storage_engine}")
address_dict.update(upload_address)
LOGGER.info(
f"upload to {storage_engine} storage, address: {address_dict}")
address = storage.StorageTableMeta.create_address(
storage_engine=storage_engine, address_dict=address_dict)
self.parameters["partitions"] = partitions
self.parameters["name"] = name
self.table = storage_session.create_table(address=address,
**self.parameters)
data_table_count = self.save_data_table(job_id, name, namespace,
head)
self.table.get_meta().update_metas(in_serialized=True)
LOGGER.info("------------load data finish!-----------------")
# rm tmp file
try:
if '{}/fate_upload_tmp'.format(job_id) in self.parameters['file']:
LOGGER.info("remove tmp upload file")
LOGGER.info(os.path.dirname(self.parameters["file"]))
shutil.rmtree(os.path.dirname(self.parameters["file"]))
except:
LOGGER.info("remove tmp file failed")
LOGGER.info("file: {}".format(self.parameters["file"]))
LOGGER.info("total data_count: {}".format(data_table_count))
LOGGER.info("table name: {}, table namespace: {}".format(
name, namespace))
import os
from fate_arch.computing import ComputingEngine
from fate_arch.federation import FederationEngine
from fate_arch.storage import StorageEngine
from fate_arch.common import file_utils, log, EngineType
from fate_flow.entity.runtime_config import RuntimeConfig
from fate_arch.common.conf_utils import get_base_config
import __main__
# Server
API_VERSION = "v1"
FATEFLOW_SERVICE_NAME = "fateflow"
MAIN_MODULE = os.path.relpath(__main__.__file__)
SERVER_MODULE = "fate_flow_server.py"
TEMP_DIRECTORY = os.path.join(file_utils.get_project_base_directory(), "temp",
"fate_flow")
HEADERS = {
"Content-Type": "application/json",
"Connection": "close",
"service": FATEFLOW_SERVICE_NAME
}
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
GRPC_SERVER_MAX_WORKERS = None
IP = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("host", "127.0.0.1")
HTTP_PORT = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("http_port")
GRPC_PORT = get_base_config(FATEFLOW_SERVICE_NAME, {}).get("grpc_port")
WORK_MODE = get_base_config("work_mode", 0)
DATABASE = get_base_config("database", {})
def default_localfs_path(name, namespace, data_type):
return os.path.join(file_utils.get_project_base_directory(), 'localfs', data_type, namespace, name)
STANDALONE_BACKEND_VIRTUAL_CORES_PER_NODE = 20
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
# abnormal condition parameter
DEFAULT_GRPC_OVERALL_TIMEOUT = 2 * 60 * 1000 # ms
DEFAULT_FEDERATED_COMMAND_TRYS = 3
JOB_DEFAULT_TIMEOUT = 3 * 24 * 60 * 60
JOB_START_TIMEOUT = 60 * 1000 # ms
'''
Constants
'''
API_VERSION = "v1"
FATEFLOW_SERVICE_NAME = 'fateflow'
MAIN_MODULE = os.path.relpath(__main__.__file__)
SERVER_MODULE = 'fate_flow_server.py'
TEMP_DIRECTORY = os.path.join(file_utils.get_project_base_directory(), "temp",
"fate_flow")
HEADERS = {'Content-Type': 'application/json', 'Connection': 'close'}
# endpoint
FATE_FLOW_MODEL_TRANSFER_ENDPOINT = '/v1/model/transfer'
FATE_MANAGER_GET_NODE_INFO_ENDPOINT = '/fate-manager/api/site/secretinfo'
FATE_MANAGER_NODE_CHECK_ENDPOINT = '/fate-manager/api/site/checksite'
FATE_BOARD_DASHBOARD_ENDPOINT = '/index.html#/dashboard?job_id={}&role={}&party_id={}'
# logger
log.LoggerFactory.LEVEL = 10
# {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0}
log.LoggerFactory.set_directory(
os.path.join(file_utils.get_project_base_directory(), 'logs', 'fate_flow'))
stat_logger = log.getLogger("fate_flow_stat")
def call_fun(func, config_data, dsl_path, config_path):
ip = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get("host")
http_port = conf_utils.get_base_config(FATEFLOW_SERVICE_NAME).get(
"http_port")
server_url = "http://{}:{}/{}".format(ip, http_port, API_VERSION)
if func in JOB_OPERATE_FUNC:
if func == 'submit_job':
if not config_path:
raise Exception(
'the following arguments are required: {}'.format(
'runtime conf path'))
dsl_data = {}
if dsl_path or config_data.get('job_parameters', {}).get(
'job_type', '') == 'predict':
if dsl_path:
dsl_path = os.path.abspath(dsl_path)
with open(dsl_path, 'r') as f:
dsl_data = json.load(f)
else:
raise Exception(
'the following arguments are required: {}'.format(
'dsl path'))
post_data = {'job_dsl': dsl_data, 'job_runtime_conf': config_data}
response = requests.post("/".join(
[server_url, "job", func.rstrip('_job')]),
json=post_data)
try:
if response.json()['retcode'] == 999:
start_cluster_standalone_job_server()
response = requests.post("/".join(
[server_url, "job",
func.rstrip('_job')]),
json=post_data)
except:
pass
elif func == 'data_view_query' or func == 'clean_queue':
response = requests.post("/".join(
[server_url, "job", func.replace('_', '/')]),
json=config_data)
else:
if func != 'query_job':
detect_utils.check_config(config=config_data,
required_arguments=['job_id'])
post_data = config_data
response = requests.post("/".join(
[server_url, "job", func.rstrip('_job')]),
json=post_data)
if func == 'query_job':
response = response.json()
if response['retcode'] == 0:
for i in range(len(response['data'])):
del response['data'][i]['f_runtime_conf']
del response['data'][i]['f_dsl']
elif func in JOB_FUNC:
if func == 'job_config':
detect_utils.check_config(config=config_data,
required_arguments=[
'job_id', 'role', 'party_id',
'output_path'
])
response = requests.post("/".join(
[server_url, func.replace('_', '/')]),
json=config_data)
response_data = response.json()
if response_data['retcode'] == 0:
job_id = response_data['data']['job_id']
download_directory = os.path.join(
config_data['output_path'], 'job_{}_config'.format(job_id))
os.makedirs(download_directory, exist_ok=True)
for k, v in response_data['data'].items():
if k == 'job_id':
continue
with open('{}/{}.json'.format(download_directory, k),
'w') as fw:
json.dump(v, fw, indent=4)
del response_data['data']['dsl']
del response_data['data']['runtime_conf']
response_data['directory'] = download_directory
response_data[
'retmsg'] = 'download successfully, please check {} directory'.format(
download_directory)
response = response_data
elif func == 'job_log':
detect_utils.check_config(
config=config_data,
required_arguments=['job_id', 'output_path'])
job_id = config_data['job_id']
tar_file_name = 'job_{}_log.tar.gz'.format(job_id)
extract_dir = os.path.join(config_data['output_path'],
'job_{}_log'.format(job_id))
with closing(
requests.get("/".join([server_url,
func.replace('_', '/')]),
json=config_data,
stream=True)) as response:
if response.status_code == 200:
download_from_request(http_response=response,
tar_file_name=tar_file_name,
extract_dir=extract_dir)
response = {
'retcode':
0,
'directory':
extract_dir,
'retmsg':
'download successfully, please check {} directory'.
format(extract_dir)
}
else:
response = response.json()
elif func in TASK_OPERATE_FUNC:
response = requests.post("/".join(
[server_url, "job", "task",
func.rstrip('_task')]),
json=config_data)
elif func in TRACKING_FUNC:
if func != 'component_metric_delete':
detect_utils.check_config(config=config_data,
required_arguments=[
'job_id', 'component_name', 'role',
'party_id'
])
if func == 'component_output_data':
detect_utils.check_config(config=config_data,
required_arguments=['output_path'])
tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(
config_data['job_id'], config_data['component_name'],
config_data['role'], config_data['party_id'])
extract_dir = os.path.join(config_data['output_path'],
tar_file_name.replace('.tar.gz', ''))
with closing(
requests.get("/".join([
server_url, "tracking",
func.replace('_', '/'), 'download'
]),
json=config_data,
stream=True)) as response:
if response.status_code == 200:
try:
download_from_request(http_response=response,
tar_file_name=tar_file_name,
extract_dir=extract_dir)
response = {
'retcode':
0,
'directory':
extract_dir,
'retmsg':
'download successfully, please check {} directory'.
format(extract_dir)
}
except:
response = {
'retcode':
100,
'retmsg':
'download failed, please check if the parameters are correct'
}
else:
response = response.json()
else:
response = requests.post("/".join(
[server_url, "tracking",
func.replace('_', '/')]),
json=config_data)
elif func in DATA_FUNC:
if func == 'upload' and config_data.get('use_local_data', 1) != 0:
file_name = config_data.get('file')
if not os.path.isabs(file_name):
file_name = os.path.join(
file_utils.get_project_base_directory(), file_name)
if os.path.exists(file_name):
with open(file_name, 'rb') as fp:
data = MultipartEncoder(
fields={
'file': (os.path.basename(file_name), fp,
'application/octet-stream')
})
tag = [0]
def read_callback(monitor):
if config_data.get('verbose') == 1:
sys.stdout.write("\r UPLOADING:{0}{1}".format(
"|" *
(monitor.bytes_read * 100 // monitor.len),
'%.2f%%' %
(monitor.bytes_read * 100 // monitor.len)))
sys.stdout.flush()
if monitor.bytes_read / monitor.len == 1:
tag[0] += 1
if tag[0] == 2:
sys.stdout.write('\n')
data = MultipartEncoderMonitor(data, read_callback)
response = requests.post(
"/".join([server_url, "data",
func.replace('_', '/')]),
data=data,
params=json.dumps(config_data),
headers={'Content-Type': data.content_type})
else:
raise Exception(
'The file is obtained from the fate flow client machine, but it does not exist, '
'please check the path: {}'.format(file_name))
else:
response = requests.post("/".join(
[server_url, "data",
func.replace('_', '/')]),
json=config_data)
try:
if response.json()['retcode'] == 999:
start_cluster_standalone_job_server()
response = requests.post("/".join([server_url, "data", func]),
json=config_data)
except:
pass
elif func in TABLE_FUNC:
if func == "table_info":
detect_utils.check_config(
config=config_data,
required_arguments=['namespace', 'table_name'])
response = requests.post("/".join([server_url, "table", func]),
json=config_data)
else:
response = requests.post("/".join(
[server_url, func.replace('_', '/')]),
json=config_data)
elif func in MODEL_FUNC:
if func == "import":
file_path = config_data["file"]
if not os.path.isabs(file_path):
file_path = os.path.join(
file_utils.get_project_base_directory(), file_path)
if os.path.exists(file_path):
files = {'file': open(file_path, 'rb')}
else:
raise Exception(
'The file is obtained from the fate flow client machine, but it does not exist, '
'please check the path: {}'.format(file_path))
response = requests.post("/".join([server_url, "model", func]),
data=config_data,
files=files)
elif func == "export":
with closing(
requests.get("/".join([server_url, "model", func]),
json=config_data,
stream=True)) as response:
if response.status_code == 200:
archive_file_name = re.findall(
"filename=(.+)",
response.headers["Content-Disposition"])[0]
os.makedirs(config_data["output_path"], exist_ok=True)
archive_file_path = os.path.join(
config_data["output_path"], archive_file_name)
with open(archive_file_path, 'wb') as fw:
for chunk in response.iter_content(1024):
if chunk:
fw.write(chunk)
response = {
'retcode':
0,
'file':
archive_file_path,
'retmsg':
'download successfully, please check {}'.format(
archive_file_path)
}
else:
response = response.json()
else:
response = requests.post("/".join([server_url, "model", func]),
json=config_data)
elif func in PERMISSION_FUNC:
detect_utils.check_config(
config=config_data,
required_arguments=['src_party_id', 'src_role'])
response = requests.post("/".join(
[server_url, "permission",
func.replace('_', '/')]),
json=config_data)
return response.json() if isinstance(
response, requests.models.Response) else response
def conf_realpath(conf_name):
conf_path = f"conf/{conf_name}"
return os.path.join(file_utils.get_project_base_directory(), conf_path)
def upload_dependencies_to_hadoop(cls, provider, dependence_type, storage_engine=FateDependenceStorageEngine.HDFS.value):
LOGGER.info(f'upload {dependence_type} dependencies to hadoop')
LOGGER.info(f'dependencies loading ...')
if dependence_type == FateDependenceName.Python_Env.value:
# todo: version python env
target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "python_env.zip")
source_path = os.path.dirname(os.path.dirname(os.getenv("VIRTUAL_ENV")))
cls.rewrite_pyvenv_cfg(os.path.join(os.getenv("VIRTUAL_ENV"), "pyvenv.cfg"), "python_env")
env_dir_list = ["python", "miniconda3"]
cls.zip_dir(source_path, target_file, env_dir_list)
dependencies_conf = {"executor_python": f"./{dependence_type}/python/venv/bin/python",
"driver_python": f"{os.path.join(os.getenv('VIRTUAL_ENV'), 'bin', 'python')}"}
else:
fate_code_dependencies = {
"fate_flow": get_fate_flow_python_directory("fate_flow"),
"fate_arch": file_utils.get_fate_python_directory("fate_arch"),
"conf": file_utils.get_project_base_directory("conf")
}
fate_flow_snapshot_time = DependenceRegistry.get_modify_time(fate_code_dependencies["fate_flow"])
fate_code_base_dir = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate_code", "fate")
python_base_dir = os.path.join(fate_code_base_dir, "python")
if os.path.exists(os.path.dirname(python_base_dir)):
shutil.rmtree(os.path.dirname(python_base_dir))
for key, path in fate_code_dependencies.items():
cls.copy_dir(path, os.path.join(python_base_dir, key))
if key == "conf":
cls.move_dir(os.path.join(python_base_dir, key), os.path.dirname(fate_code_base_dir))
if provider.name == ComponentProviderName.FATE.value:
source_path = provider.path
else:
source_path = ComponentProviderInfo.get_or_none(
ComponentProviderInfo.f_version == provider.version,
ComponentProviderInfo.f_provider_name == ComponentProviderName.FATE.value
).f_path
cls.copy_dir(source_path, os.path.join(python_base_dir, "federatedml"))
target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate.zip")
cls.zip_dir(os.path.dirname(fate_code_base_dir), target_file)
dependencies_conf = {"executor_env_pythonpath": f"./{dependence_type}/fate/python:$PYTHONPATH"}
LOGGER.info(f'dependencies loading success')
LOGGER.info(f'start upload')
snapshot_time = DependenceRegistry.get_modify_time(source_path)
storage_dir = f"/fate_dependence/{provider.version}"
os.system(f" {os.getenv('HADOOP_HOME')}/bin/hdfs dfs -mkdir -p {storage_dir}")
status = os.system(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put -f {target_file} {storage_dir}")
LOGGER.info(f'upload end, status is {status}')
if status == 0:
storage_path = os.path.join(storage_dir, os.path.basename(target_file))
storage_meta = {
"f_storage_engine": storage_engine,
"f_type": dependence_type,
"f_version": provider.version,
"f_storage_path": storage_path,
"f_snapshot_time": snapshot_time,
"f_fate_flow_snapshot_time": fate_flow_snapshot_time if dependence_type == FateDependenceName.Fate_Source_Code.value else None,
"f_dependencies_conf": {"archives": "#".join([storage_path, dependence_type])},
"f_upload_status": False,
"f_pid": 0
}
storage_meta["f_dependencies_conf"].update(dependencies_conf)
DependenceRegistry.save_dependencies_storage_meta(storage_meta)
else:
raise Exception(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put {target_file} {storage_dir} failed status: {status}")
return storage_meta
def gen_model_file_path(model_id, model_version):
return os.path.join(get_project_base_directory(), "model_local_cache",
model_id, model_version)
def get_versions() -> typing.Mapping[str, typing.Any]:
return dotenv.dotenv_values(
dotenv_path=os.path.join(get_project_base_directory(), "fate.env"))
def get_job_log_directory(job_id):
return os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id)
def set_model_path(self):
self.model_path = os.path.join(file_utils.get_project_base_directory(),
"model_local_cache", self.model_id,
self.model_version)
)
return _meta_table
# noinspection PyProtectedMember
def _get_from_meta_table(key):
return _get_meta_table().get(key)
# noinspection PyProtectedMember
def _put_to_meta_table(key, value):
_get_meta_table().put(key, value)
_data_dir = Path(
file_utils.get_project_base_directory()).joinpath("data").absolute()
def _get_data_dir():
return _data_dir
def _get_storage_dir(*args):
return _data_dir.joinpath(*args)
async def _check_status_and_get_value(get_func, key):
value = get_func(key)
while value is None:
await asyncio.sleep(0.1)
value = get_func(key)
