def init():
if USE_AUTHENTICATION:
# init local storage
stat_logger.info('init local authorization library')
file_dir = os.path.join(file_utils.get_project_base_directory(),
'fate_flow')
os.makedirs(file_dir, exist_ok=True)
PrivilegeAuth.local_storage_file = os.path.join(
file_dir, 'authorization_config.json')
if not os.path.exists(PrivilegeAuth.local_storage_file):
with open(PrivilegeAuth.local_storage_file, 'w') as fp:
fp.write(json.dumps({}))
# init whitelist
PrivilegeAuth.command_whitelist = PRIVILEGE_COMMAND_WHITELIST
# init ALL_PERMISSION
component_path = os.path.join(
file_utils.get_project_base_directory(), 'federatedml', 'conf',
'setting_conf')
command_file_path = os.path.join(
file_utils.get_project_base_directory(), 'fate_flow', 'apps',
'schedule_app.py')
stat_logger.info(
'search commands from {}'.format(command_file_path))
search_command(command_file_path)
stat_logger.info(
'search components from {}'.format(component_path))
search_component(component_path)
def get_schedule_logger(job_id='', log_type='schedule'):
fate_flow_log_dir = os.path.join(
file_utils.get_project_base_directory(), 'logs', 'fate_flow')
job_log_dir = os.path.join(file_utils.get_project_base_directory(),
'logs', job_id)
if not job_id:
log_dirs = [fate_flow_log_dir]
else:
if log_type == 'audit':
log_dirs = [job_log_dir, fate_flow_log_dir]
else:
log_dirs = [job_log_dir]
os.makedirs(job_log_dir, exist_ok=True)
os.makedirs(fate_flow_log_dir, exist_ok=True)
logger = logging.getLogger('{}_{}'.format(job_id, log_type))
logger.setLevel(LoggerFactory.LEVEL)
for job_log_dir in log_dirs:
handler = LoggerFactory.get_handler(class_name=None,
level=LoggerFactory.LEVEL,
log_dir=job_log_dir,
log_type=log_type)
error_handler = LoggerFactory.get_handler(class_name=None,
level=logging.ERROR,
log_dir=job_log_dir,
log_type=log_type)
logger.addHandler(handler)
logger.addHandler(error_handler)
if job_id:
with LoggerFactory.lock:
LoggerFactory.schedule_logger_dict[job_id + log_type] = logger
return logger
def download_data(data_func):
_data = request.json
_job_id = generate_job_id()
logger.info('generated job_id {}, body {}'.format(_job_id, _data))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
_download_module = os.path.join(file_utils.get_project_base_directory(),
"arch/api/utils/download.py")
_upload_module = os.path.join(file_utils.get_project_base_directory(),
"arch/api/utils/upload.py")
if data_func == "download":
_module = _download_module
else:
_module = _upload_module
try:
if data_func == "download":
progs = [
"python3", _module, "-j", _job_id, "-c",
os.path.abspath(_data.get("config_path"))
]
else:
progs = [
"python3", _module, "-c",
os.path.abspath(_data.get("config_path"))
]
logger.info('Starting progs: {}'.format(progs))
std_log = open(os.path.join(_job_dir, 'std.log'), 'w')
task_pid_path = os.path.join(_job_dir, 'pids')
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
else:
startupinfo = None
p = subprocess.Popen(progs,
stdout=std_log,
stderr=std_log,
startupinfo=startupinfo)
os.makedirs(task_pid_path, exist_ok=True)
with open(os.path.join(task_pid_path, data_func + ".pid"), 'w') as f:
f.truncate()
f.write(str(p.pid) + "\n")
f.flush()
return get_json_result(0, "success, job_id {}".format(_job_id))
except Exception as e:
print(e)
return get_json_result(-104, "failed, job_id {}".format(_job_id))
def test_job_operation(self):
# submit
with open(os.path.join(file_utils.get_project_base_directory(), self.dsl_path), 'r') as f:
dsl_data = json.load(f)
with open(os.path.join(file_utils.get_project_base_directory(), self.config_path), 'r') as f:
config_data = json.load(f)
config_data['job_parameters']['work_mode'] = WORK_MODE
response = requests.post("/".join([self.server_url, 'job', 'submit']),
json={'job_dsl': dsl_data, 'job_runtime_conf': config_data})
self.assertTrue(response.status_code in [200, 201])
self.assertTrue(int(response.json()['retcode']) == 0)
job_id = response.json()['jobId']
# query
response = requests.post("/".join([self.server_url, 'job', 'query']), json={'job_id': job_id, 'role': 'guest'})
self.assertTrue(int(response.json()['retcode']) == 0)
job_info = response.json()['data'][0]
# note
response = requests.post("/".join([self.server_url, 'job', 'update']), json={'job_id': job_id, 'role': job_info['f_role'], 'party_id': job_info['f_party_id'], 'notes': 'unittest'})
self.assertTrue(int(response.json()['retcode']) == 0)
# config
response = requests.post("/".join([self.server_url, 'job', 'config']), json={'job_id': job_id, 'role': job_info['f_role'], 'party_id': job_info['f_party_id']})
self.assertTrue(int(response.json()['retcode']) == 0)
print('waiting 10s ...')
time.sleep(10)
# stop
response = requests.post("/".join([self.server_url, 'job', 'stop']), json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
# logs
with closing(requests.get("/".join([self.server_url, 'job', 'log']), json={'job_id': job_id}, stream=True)) as response:
self.assertTrue(response.status_code in [200, 201])
# query task
response = requests.post("/".join([self.server_url, 'job', '/task/query']), json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
# query data viw
response = requests.post("/".join([self.server_url, 'job', '/data/view/query']), json={'party_id': 0})
self.assertTrue(int(response.json()['retcode']) == 0)
print('waiting 10s...')
time.sleep(10)
# clean job
response = requests.post("/".join([self.server_url, 'job', '/clean']), json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
def run(self, component_parameters=None, args=None):
self.parameters = component_parameters["UploadParam"]
self.parameters["role"] = component_parameters["role"]
self.parameters["local"] = component_parameters["local"]
job_id = self.taskid.split("_")[0]
if not os.path.isabs(self.parameters.get("file", "")):
self.parameters["file"] = os.path.join(file_utils.get_project_base_directory(), self.parameters["file"])
if not os.path.exists(self.parameters["file"]):
raise Exception("%s is not exist, please check the configure" % (self.parameters["file"]))
table_name, namespace = dtable_utils.get_table_info(config=self.parameters,
create=True)
_namespace, _table_name = self.generate_table_name(self.parameters["file"])
if namespace is None:
namespace = _namespace
if table_name is None:
table_name = _table_name
read_head = self.parameters['head']
if read_head == 0:
head = False
elif read_head == 1:
head = True
else:
raise Exception("'head' in conf.json should be 0 or 1")
partition = self.parameters["partition"]
if partition <= 0 or partition >= self.MAX_PARTITION_NUM:
raise Exception("Error number of partition, it should between %d and %d" % (0, self.MAX_PARTITION_NUM))
session.init(mode=self.parameters['work_mode'])
data_table_count = self.save_data_table(table_name, namespace, head, self.parameters.get('in_version', False))
LOGGER.info("------------load data finish!-----------------")
LOGGER.info("file: {}".format(self.parameters["file"]))
LOGGER.info("total data_count: {}".format(data_table_count))
LOGGER.info("table name: {}, table namespace: {}".format(table_name, namespace))
def init_roll_site_context(runtime_conf, session_id):
from eggroll.roll_site.roll_site import RollSiteContext
from eggroll.roll_pair.roll_pair import RollPairContext
LOGGER.info("init_roll_site_context runtime_conf: {}".format(runtime_conf))
session_instance = FateSession.get_instance()._eggroll.get_session()
rp_context = RollPairContext(session_instance)
role = runtime_conf.get("local").get("role")
party_id = str(runtime_conf.get("local").get("party_id"))
_path = file_utils.get_project_base_directory(
) + "/arch/conf/server_conf.json"
server_conf = file_utils.load_json_conf(_path)
host = server_conf.get('servers').get('proxy').get("host")
port = server_conf.get('servers').get('proxy').get("port")
options = {
'self_role': role,
'self_party_id': party_id,
'proxy_endpoint': ErEndpoint(host, int(port))
}
rs_context = RollSiteContext(session_id,
rp_ctx=rp_context,
options=options)
LOGGER.info("init_roll_site_context done: {}".format(rs_context.__dict__))
return rp_context, rs_context
def generate_transfer_var_class(self, transfer_var_conf_path, out_path):
base_dir = file_utils.get_project_base_directory()
merge_conf_path = os.path.join(
base_dir, "federatedml/transfer_variable_conf/transfer_conf.json")
merge_dict = {}
if os.path.isfile(merge_conf_path):
with open(merge_conf_path, "r") as fin:
merge_dict = json.loads(fin.read())
var_dict = {}
with open(transfer_var_conf_path) as fin:
var_dict = json.loads(fin.read())
merge_dict.update(var_dict)
with open(merge_conf_path, "w") as fout:
json_dumps_indent_str = json.dumps(merge_dict, indent=1)
buffers = json_dumps_indent_str.split("\n", -1)
for buf in buffers:
fout.write(buf + "\n")
fout = open(out_path, "w")
with_header = True
for class_name in var_dict:
transfer_var_dict = var_dict[class_name]
self.write_out_class(fout, class_name, transfer_var_dict,
with_header)
with_header = False
fout.flush()
fout.close()
def set_directory(directory=None,
parent_log_dir=None,
append_to_parent_log=None,
force=False):
if parent_log_dir:
LoggerFactory.PARENT_LOG_DIR = parent_log_dir
if append_to_parent_log:
LoggerFactory.append_to_parent_log = append_to_parent_log
with LoggerFactory.lock:
if not directory:
directory = os.path.join(
file_utils.get_project_base_directory(), 'logs')
if not LoggerFactory.LOG_DIR or force:
LoggerFactory.LOG_DIR = directory
os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
for loggerName, ghandler in LoggerFactory.global_handler_dict.items(
):
for className, (logger,
handler) in LoggerFactory.logger_dict.items():
logger.removeHandler(ghandler)
ghandler.close()
LoggerFactory.global_handler_dict = {}
for className, (logger,
handler) in LoggerFactory.logger_dict.items():
logger.removeHandler(handler)
_hanlder = None
if handler:
handler.close()
if className != "default":
_hanlder = LoggerFactory.get_handler(className)
logger.addHandler(_hanlder)
LoggerFactory.assemble_global_handler(logger)
LoggerFactory.logger_dict[className] = logger, _hanlder
def list_files(namespace):
_type = StoreType.LMDB.value
_base_dir = os.sep.join([os.path.join(file_utils.get_project_base_directory(), 'data'), _type])
_namespace_dir = os.sep.join([_base_dir, namespace])
if not os.path.isdir(_base_dir):
raise EnvironmentError("illegal datadir")
return [file for file in os.listdir(_namespace_dir) if not file.endswith('.meta')]
def init(job_id=None,
mode: typing.Union[int, WorkMode] = WorkMode.STANDALONE,
backend: typing.Union[int, Backend] = Backend.EGGROLL,
persistent_engine: StoreType = StoreType.LMDB,
set_log_dir=True):
if isinstance(mode, int):
mode = WorkMode(mode)
if isinstance(backend, int):
backend = Backend(backend)
if RuntimeInstance.SESSION:
return
if job_id is None:
job_id = str(uuid.uuid1())
if set_log_dir:
LoggerFactory.set_directory()
else:
if set_log_dir:
LoggerFactory.set_directory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id))
RuntimeInstance.MODE = mode
RuntimeInstance.Backend = backend
from arch.api.table.session import build_session
session = build_session(job_id=job_id, work_mode=mode, backend=backend)
RuntimeInstance.SESSION = session
def init(job_id=None,
mode: WorkMode = WorkMode.STANDALONE,
naming_policy: NamingPolicy = NamingPolicy.DEFAULT):
if RuntimeInstance.EGGROLL:
return
if job_id is None:
job_id = str(uuid.uuid1())
LoggerFactory.set_directory()
else:
LoggerFactory.set_directory(
os.path.join(file_utils.get_project_base_directory(), 'logs',
job_id))
RuntimeInstance.MODE = mode
eggroll_context = EggRollContext(naming_policy=naming_policy)
if mode == WorkMode.STANDALONE:
from arch.api.standalone.eggroll import Standalone
RuntimeInstance.EGGROLL = Standalone(job_id=job_id,
eggroll_context=eggroll_context)
elif mode == WorkMode.CLUSTER:
from arch.api.cluster.eggroll import _EggRoll
from arch.api.cluster.eggroll import init as c_init
c_init(job_id, eggroll_context=eggroll_context)
RuntimeInstance.EGGROLL = _EggRoll.get_instance()
else:
from arch.api.cluster import simple_roll
simple_roll.init(job_id)
RuntimeInstance.EGGROLL = simple_roll.EggRoll.get_instance()
RuntimeInstance.EGGROLL.table("__federation__", job_id, partition=10)
def runtime_json_generator(site_flow, runtime_path):
if site_flow is None:
raise Exception("site_flow is None")
with open(runtime_path) as runtime_conf:
runtime_json = json.load(runtime_conf)
with open(file_utils.get_project_base_directory() +
"/contrib/fate_script/conf/route.json") as route_conf:
route_json = json.load(route_conf)
site_set = set()
inner_dict = defaultdict()
for i in range(len(site_flow)):
if site_flow[i][2] is None or site_flow[i][3] is None:
continue
site_set.add(site_flow[i][2])
site_set.add(site_flow[i][3])
for i in range(len(site_set)):
if list(site_set)[i] in route_json["cluster_a"]["role"]:
cluster_type = "cluster_a"
elif list(site_set)[i] in route_json["cluster_b"]["role"]:
cluster_type = "cluster_b"
else:
raise Exception("role:{} is not in route.json".format(
list(site_set)[i]))
inner_dict[list(site_set)[i]] = [route_json[cluster_type]["party_id"]]
for i in range(len(site_set)):
if list(site_set)[i] in route_json["cluster_a"]["role"]:
cluster_type = "cluster_a"
elif list(site_set)[i] in route_json["cluster_b"]["role"]:
cluster_type = "cluster_b"
else:
raise Exception("role:{} is not in route.json".format(
list(site_set)[i]))
runtime_json["local"]["role"] = list(site_set)[i]
runtime_json["local"]["party_id"] = route_json[cluster_type][
"party_id"]
runtime_json["role"] = inner_dict
json_path = file_utils.get_project_base_directory(
) + "/contrib/fate_script/conf/" + list(
site_set)[i] + "_runtime_conf.json"
with open(json_path, "wt") as fout:
fout.write(json.dumps(runtime_json, indent=4))
def monkey_patch():
package_name = "monkey_patch"
package_path = os.path.join(file_utils.get_project_base_directory(), "fate_flow", package_name)
for f in os.listdir(package_path):
if not os.path.isdir(os.path.join(package_path, f)) or f == "__pycache__":
continue
patch_module = importlib.import_module("fate_flow." + package_name + '.' + f + '.monkey_patch')
patch_module.patch_all()
def __init__(self, job_id=None):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
'data')
self.job_id = str(
uuid.uuid1()) if job_id is None else "{}".format(job_id)
self.meta_table = _DTable('__META__', '__META__', 'fragments', 10)
self.pool = Executor()
Standalone.__instance = self
def get_role_list():
with open(file_utils.get_project_base_directory() +
"/contrib/fate_script/conf/route.json") as fp:
route_json = json.load(fp)
print("keys:", route_json.keys())
role_list = []
for key in route_json.keys():
role_list.append(route_json[key]['role'])
return reduce(operator.add, role_list)
def run_fate_script_standalone(job_id, work_mode):
role_list = get_role_list()
for role in role_list:
runtime_conf_path = file_utils.get_project_base_directory(
) + "/contrib/fate_script/conf/" + str(role) + "_runtime_conf.json"
run_cmd = "python fateScript.py " + str(role) + " " + str(
job_id) + " " + str(runtime_conf_path) + " " + str(work_mode) + "&"
print("run cmd:", format(run_cmd))
os.system(run_cmd)
def get_job_dsl_parser(dsl=None,
runtime_conf=None,
pipeline_dsl=None,
train_runtime_conf=None):
dsl_parser = DSLParser()
default_runtime_conf_path = os.path.join(
file_utils.get_project_base_directory(),
*['federatedml', 'conf', 'default_runtime_conf'])
setting_conf_path = os.path.join(file_utils.get_project_base_directory(),
*['federatedml', 'conf', 'setting_conf'])
job_type = runtime_conf.get('job_parameters', {}).get('job_type', 'train')
dsl_parser.run(dsl=dsl,
runtime_conf=runtime_conf,
pipeline_dsl=pipeline_dsl,
pipeline_runtime_conf=train_runtime_conf,
default_runtime_conf_prefix=default_runtime_conf_path,
setting_conf_prefix=setting_conf_path,
mode=job_type)
return dsl_parser
def setDirectory(directory=None):
with LoggerFactory.lock:
if not directory:
directory = os.path.join(file_utils.get_project_base_directory(), 'logs')
LoggerFactory.LOG_DIR = directory
os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
for className, (logger, handler) in LoggerFactory.loggerDict.items():
logger.removeHandler(handler)
handler.close()
_hanlder = LoggerFactory.get_hanlder(className)
logger.addHandler(_hanlder)
LoggerFactory.loggerDict[className] = logger, _hanlder
def create_pipelined_model(self):
if os.path.exists(self.model_path):
raise Exception("Model creation failed because it has already been created, model cache path is {}".format(
self.model_path
))
else:
os.makedirs(self.model_path, exist_ok=False)
for path in [self.variables_index_path, self.variables_data_path]:
os.makedirs(path, exist_ok=False)
shutil.copytree(os.path.join(file_utils.get_project_base_directory(), "federatedml", "protobuf", "proto"), self.define_proto_path)
with open(self.define_meta_path, "w", encoding="utf-8") as fw:
yaml.dump({"describe": "This is the model definition meta"}, fw, Dumper=yaml.RoundTripDumper)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
stat_logger.info('generated job_id {}, body {}'.format(_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
required_arguments = ['work_mode', 'namespace', 'table_name']
if module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif module == 'download':
required_arguments.extend(['output_path'])
else:
raise Exception('can not support this operating: {}'.format(module))
detect_utils.check_config(request_config, required_arguments=required_arguments)
if module == "upload":
if not os.path.isabs(request_config['file']):
request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"])
try:
conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", ''))
file_utils.dump_json_conf(request_config, conf_file_path)
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-j", _job_id,
"-c", conf_file_path
]
try:
p = run_subprocess(config_dir=_job_dir, process_cmd=progs)
except Exception as e:
stat_logger.exception(e)
p = None
return get_json_result(retcode=(0 if p else 101), job_id=_job_id,
data={'table_name': request_config['table_name'],
'namespace': request_config['namespace'], 'pid': p.pid if p else ''})
except Exception as e:
stat_logger.exception(e)
return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)
def __init__(self, transfer_conf_path):
self.transfer_auth = {}
for path, _, file_names in os.walk(
os.path.join(file_utils.get_project_base_directory(),
transfer_conf_path)):
for name in file_names:
transfer_conf = os.path.join(path, name)
if transfer_conf.endswith(".json"):
self.transfer_auth.update(
file_utils.load_json_conf(transfer_conf))
# cache
self._authorized_src = {}
self._authorized_dst = {}
def main():
parser = argparse.ArgumentParser(
description="generate conf from transfer variable class")
parser.add_argument(
"-s",
"--src",
type=str,
default=DEFAULT_SRC,
help=f"dir to search transfer classes, defaults to {DEFAULT_SRC}")
parser.add_argument(
"-d",
"--dst",
type=str,
default=DEFAULT_DST,
help=f"dir to save generated json files, defaults to {DEFAULT_DST}")
arg = parser.parse_args()
dst = arg.dst
if not os.path.exists(dst):
os.mkdir(dst)
if not os.path.isdir(dst):
raise ValueError(f"{dst} should be a directory")
src = arg.src
if not os.path.exists(src):
raise ValueError(f"{src} not exists")
BaseTransferVariables._disable__singleton()
base = file_utils.get_project_base_directory()
for path, _, files in os.walk(src):
path = os.path.relpath(path, base)
for file_name in files:
if file_name.endswith(".py"):
name = f"{path.replace('/', '.')}.{file_name[:-3]}"
for cls in search_subclasses(name, BaseTransferVariables):
class_name = cls.__name__
file_name = _Camel2Snake.to_snake(class_name)
variable_dict = {}
for k, v in search_transfer_variable(cls()).items():
variable_dict[k] = {"src": v._src, "dst": v._dst}
if len(variable_dict) < 1:
continue
saving_path = os.path.join(dst, f"{file_name}.json")
print(
f"found transfer_class: {class_name} in {name}, saving to {saving_path}"
)
with open(saving_path, "w") as g:
json.dump({class_name: variable_dict}, g, indent=2)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
logger.info('generated job_id {}, body {}'.format(_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
if module == "upload":
if not os.path.isabs(request_config.get("file", "")):
request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"])
try:
request_config["work_mode"] = request_config.get('work_mode', WORK_MODE)
table_name, namespace = dtable_utils.get_table_info(config=request_config, create=(True if module == 'upload' else False))
if not table_name or not namespace:
return get_json_result(status=102, msg='no table name and namespace')
request_config['table_name'] = table_name
request_config['namespace'] = namespace
conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", PARTY_ID))
file_utils.dump_json_conf(request_config, conf_file_path)
if module == "download":
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-j", _job_id,
"-c", conf_file_path
]
else:
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-c", conf_file_path
]
p = run_subprocess(job_dir=_job_dir, job_role=data_func, progs=progs)
return get_json_result(job_id=_job_id, data={'pid': p.pid, 'table_name': request_config['table_name'], 'namespace': request_config['namespace']})
except Exception as e:
logger.exception(e)
return get_json_result(status=-104, msg="failed", job_id=_job_id)
def start_workflow(job_id, module, role):
_data = request.json
_job_dir = get_job_directory(job_id)
_party_id = str(_data['local']['party_id'])
_method = _data['WorkFlowParam']['method']
conf_path_dir = os.path.join(_job_dir, _method, module, role, _party_id)
os.makedirs(conf_path_dir, exist_ok=True)
conf_file_path = os.path.join(conf_path_dir, 'runtime_conf.json')
with open(conf_file_path, 'w+') as f:
f.truncate()
f.write(json.dumps(_data, indent=4))
f.flush()
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
else:
startupinfo = None
task_pid_path = os.path.join(_job_dir, 'pids')
std_log = open(os.path.join(_job_dir, role + '.std.log'), 'w')
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
_data['CodePath']), "-j", job_id, "-c",
os.path.abspath(conf_file_path)
]
logger.info('Starting progs: {}'.format(" ".join(progs)))
p = subprocess.Popen(progs,
stdout=std_log,
stderr=std_log,
startupinfo=startupinfo)
os.makedirs(task_pid_path, exist_ok=True)
with open(os.path.join(task_pid_path, role + ".pid"), 'w') as f:
f.truncate()
f.write(str(p.pid) + "\n")
f.flush()
job_data = dict()
job_data["begin_date"] = datetime.datetime.now()
job_data["status"] = "ready"
with open(conf_file_path) as fr:
config = json.load(fr)
job_data.update(config)
job_data["my_role"] = config.get("local", {}).get("role")
save_job_info(job_id=job_id, **job_data)
update_job_queue(job_id=job_id, update_data={"status": "ready"})
return get_json_result(msg="success, pid is %s" % p.pid)
def setUp(self):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
"examples", "data")
self.upload_guest_config = {
"file": os.path.join(self.data_dir, "breast_hetero_guest.csv"),
"head": 1,
"partition": 10,
"work_mode": WORK_MODE,
"namespace": "fate_flow_test_breast_hetero",
"table_name": "breast_hetero_guest",
"use_local_data": 0,
'drop': 0
}
self.upload_host_config = {
"file": os.path.join(self.data_dir, "breast_hetero_host.csv"),
"head": 1,
"partition": 10,
"work_mode": WORK_MODE,
"namespace": "fate_flow_test_breast_hetero",
"table_name": "breast_hetero_host",
"use_local_data": 0,
'drop': 0
}
self.download_config = {
"output_path":
os.path.join(file_utils.get_project_base_directory(),
"fate_flow/fate_flow_unittest_breast_b.csv"),
"work_mode":
WORK_MODE,
"namespace":
"fate_flow_test_breast_hetero",
"table_name":
"breast_hetero_guest"
}
self.server_url = "http://{}:{}/{}".format('127.0.0.1', HTTP_PORT,
API_VERSION)
def get_proto_buffer_class(buffer_name):
package_path = os.path.join(file_utils.get_project_base_directory(), 'federatedml', 'protobuf', 'generated')
package_python_path = 'federatedml.protobuf.generated'
for f in os.listdir(package_path):
if f.startswith('.'):
continue
try:
proto_module = importlib.import_module(package_python_path + '.' + f.rstrip('.py'))
for name, obj in inspect.getmembers(proto_module):
if inspect.isclass(obj) and name == buffer_name:
return obj
except Exception as e:
stat_logger.warning(e)
else:
return None
def __init__(self, model_id, model_version):
"""
Support operations on FATE PipelinedModels
TODO: add lock
:param model_id: the model id stored at the local party.
:param model_version: the model version.
"""
self.model_id = model_id
self.model_version = model_version
self.model_path = os.path.join(file_utils.get_project_base_directory(), "model_local_cache", model_id, model_version)
self.define_proto_path = os.path.join(self.model_path, "define", "proto")
self.define_meta_path = os.path.join(self.model_path, "define", "define_meta.yaml")
self.variables_index_path = os.path.join(self.model_path, "variables", "index")
self.variables_data_path = os.path.join(self.model_path, "variables", "data")
self.default_archive_format = "zip"
def start_workflow(job_id, module, role):
_config = request.json
_job_dir = get_job_directory(job_id)
_party_id = str(_config['local']['party_id'])
_method = _config['WorkFlowParam']['method']
default_runtime_dict = file_utils.load_json_conf(
'workflow/conf/default_runtime_conf.json')
fill_runtime_conf_table_info(runtime_conf=_config,
default_runtime_conf=default_runtime_dict)
conf_file_path = new_runtime_conf(job_dir=_job_dir,
method=_method,
module=module,
role=role,
party_id=_party_id)
with open(conf_file_path, 'w+') as f:
f.truncate()
f.write(json.dumps(_config, indent=4))
f.flush()
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
_config['CodePath']), "-j", job_id, "-c",
os.path.abspath(conf_file_path)
]
p = run_subprocess(job_dir=_job_dir, job_role=role, progs=progs)
job_status = "start"
job_data = dict()
job_data["begin_date"] = datetime.datetime.now()
job_data["status"] = job_status
job_data.update(_config)
job_data["pid"] = p.pid
job_data["all_party"] = json.dumps(_config.get("role", {}))
job_data["initiator"] = _config.get("JobParam", {}).get("initiator")
save_job_info(job_id=job_id,
role=_config.get("local", {}).get("role"),
party_id=_config.get("local", {}).get("party_id"),
save_info=job_data,
create=True)
update_job_queue(job_id=job_id,
role=role,
party_id=_party_id,
save_data={
"status": job_status,
"pid": p.pid
})
return get_json_result(data={'pid': p.pid}, job_id=job_id)
def get_schedule_logger(job_id):
job_log_dir = os.path.join(file_utils.get_project_base_directory(),
'logs', job_id)
os.makedirs(job_log_dir, exist_ok=True)
logger = logging.getLogger(job_id)
logger.setLevel(LoggerFactory.LEVEL)
handler = LoggerFactory.get_handler(class_name=None,
level=LoggerFactory.LEVEL,
log_dir=job_log_dir,
is_schedule=True)
error_handler = LoggerFactory.get_handler(class_name=None,
level=logging.ERROR,
log_dir=job_log_dir,
is_schedule=True)
logger.addHandler(handler)
logger.addHandler(error_handler)
with LoggerFactory.lock:
LoggerFactory.schedule_logger_dict[job_id] = logger
return logger
def __init__(self, job_id=None, eggroll_context=None):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
'data')
self.job_id = str(
uuid.uuid1()) if job_id is None else "{}".format(job_id)
self.meta_table = _DTable('__META__', '__META__', 'fragments', 10)
self.pool = Executor()
Standalone.__instance = self
if not eggroll_context:
eggroll_context = EggRollContext()
self.eggroll_context = eggroll_context
self.unique_id_template = '_EggRoll_%s_%s_%s_%.20f_%d'
# todo: move to EggRollContext
try:
self.host_name = socket.gethostname()
self.host_ip = socket.gethostbyname(self.host_name)
except socket.gaierror as e:
self.host_name = 'unknown'
self.host_ip = 'unknown'
