def run():
parser = argparse.ArgumentParser()
parser.add_argument('-j',
'--job_id',
required=True,
type=str,
help="job id")
parser.add_argument('-w',
'--work_mode',
required=True,
type=str,
help="work mode")
parser.add_argument('-b',
'--backend',
required=True,
type=str,
help="backend")
args = parser.parse_args()
job_id = args.job_id
work_mode = int(args.work_mode)
backend = int(args.backend)
session.init(job_id=job_id,
mode=work_mode,
backend=backend,
set_log_dir=False)
try:
schedule_logger(job_id.split('_')[0]).info(
'start stop session {}'.format(session.get_session_id()))
session.stop()
schedule_logger(job_id.split('_')[0]).info(
'stop session {} success'.format(session.get_session_id()))
except Exception as e:
pass
def clean_server_used_session():
used_session_id = None
try:
used_session_id = session.get_session_id()
session.stop()
except:
pass
session.exit()
stat_logger.info("clean session {} for fate flow server done".format(used_session_id))
def tearDown(self):
session.stop()
try:
session.cleanup("*", self.jobid, True)
except EnvironmentError:
pass
try:
session.cleanup("*", self.jobid, False)
except EnvironmentError:
pass
def task_killed_detector(job_id, role, party_id, component_name, pid):
kill_path = os.path.join(get_job_directory(job_id), str(role),
str(party_id), component_name, 'kill')
if os.path.exists(kill_path):
try:
session.stop()
except Exception as e:
pass
kill_process(int(pid), only_child=True)
os.kill(int(pid), 9)
kill_process(int(pid), only_child=False)
threading.Timer(0.25,
task_killed_detector,
args=[job_id, role, party_id, component_name,
pid]).start()
def run():
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--job_id', required=True, type=str, help="job id")
parser.add_argument('-w', '--work_mode', required=True, type=str, help="work mode")
parser.add_argument('-b', '--backend', required=True, type=str, help="backend")
parser.add_argument('-c', '--command', required=True, type=str, help="command")
args = parser.parse_args()
session_job_id = args.job_id
fate_job_id = session_job_id.split('_')[0]
work_mode = int(args.work_mode)
backend = int(args.backend)
command = args.command
session.init(job_id=session_job_id, mode=work_mode, backend=backend, set_log_dir=False)
try:
schedule_logger(fate_job_id).info('start {} session {}'.format(command, session.get_session_id()))
if command == 'stop':
session.stop()
elif command == 'kill':
session.kill()
else:
schedule_logger(fate_job_id).info('{} session {} failed, this command is not supported'.format(command, session.get_session_id()))
schedule_logger(fate_job_id).info('{} session {} success'.format(command, session.get_session_id()))
except Exception as e:
pass
def tearDown(self):
session.stop()
def run_task():
task = Task()
task.f_create_time = current_timestamp()
try:
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--job_id', required=True, type=str, help="job id")
parser.add_argument('-n', '--component_name', required=True, type=str,
help="component name")
parser.add_argument('-t', '--task_id', required=True, type=str, help="task id")
parser.add_argument('-r', '--role', required=True, type=str, help="role")
parser.add_argument('-p', '--party_id', required=True, type=str, help="party id")
parser.add_argument('-c', '--config', required=True, type=str, help="task config")
parser.add_argument('--job_server', help="job server", type=str)
args = parser.parse_args()
schedule_logger(args.job_id).info('enter task process')
schedule_logger(args.job_id).info(args)
# init function args
if args.job_server:
RuntimeConfig.init_config(HTTP_PORT=args.job_server.split(':')[1])
job_id = args.job_id
component_name = args.component_name
task_id = args.task_id
role = args.role
party_id = int(args.party_id)
task_config = file_utils.load_json_conf(args.config)
job_parameters = task_config['job_parameters']
job_initiator = task_config['job_initiator']
job_args = task_config['job_args']
task_input_dsl = task_config['input']
task_output_dsl = task_config['output']
parameters = TaskExecutor.get_parameters(job_id, component_name, role, party_id)
# parameters = task_config['parameters']
module_name = task_config['module_name']
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
task.f_status = TaskStatus.FAILED
return
try:
# init environment, process is shared globally
RuntimeConfig.init_config(WORK_MODE=job_parameters['work_mode'],
BACKEND=job_parameters.get('backend', 0))
session.init(job_id='{}_{}_{}'.format(task_id, role, party_id), mode=RuntimeConfig.WORK_MODE, backend=RuntimeConfig.BACKEND)
federation.init(job_id=task_id, runtime_conf=parameters)
job_log_dir = os.path.join(job_utils.get_job_log_directory(job_id=job_id), role, str(party_id))
task_log_dir = os.path.join(job_log_dir, component_name)
log_utils.LoggerFactory.set_directory(directory=task_log_dir, parent_log_dir=job_log_dir,
append_to_parent_log=True, force=True)
task.f_job_id = job_id
task.f_component_name = component_name
task.f_task_id = task_id
task.f_role = role
task.f_party_id = party_id
task.f_operator = 'python_operator'
tracker = Tracking(job_id=job_id, role=role, party_id=party_id, component_name=component_name,
task_id=task_id,
model_id=job_parameters['model_id'],
model_version=job_parameters['model_version'],
module_name=module_name)
task.f_start_time = current_timestamp()
task.f_run_ip = get_lan_ip()
task.f_run_pid = os.getpid()
run_class_paths = parameters.get('CodePath').split('/')
run_class_package = '.'.join(run_class_paths[:-2]) + '.' + run_class_paths[-2].replace('.py','')
run_class_name = run_class_paths[-1]
task_run_args = TaskExecutor.get_task_run_args(job_id=job_id, role=role, party_id=party_id,
job_parameters=job_parameters, job_args=job_args,
input_dsl=task_input_dsl)
run_object = getattr(importlib.import_module(run_class_package), run_class_name)()
run_object.set_tracker(tracker=tracker)
run_object.set_taskid(taskid=task_id)
task.f_status = TaskStatus.RUNNING
TaskExecutor.sync_task_status(job_id=job_id, component_name=component_name, task_id=task_id, role=role,
party_id=party_id, initiator_party_id=job_initiator.get('party_id', None),
initiator_role=job_initiator.get('role', None),
task_info=task.to_json())
schedule_logger().info('run {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id))
schedule_logger().info(parameters)
schedule_logger().info(task_input_dsl)
run_object.run(parameters, task_run_args)
output_data = run_object.save_data()
tracker.save_output_data_table(output_data, task_output_dsl.get('data')[0] if task_output_dsl.get('data') else 'component')
output_model = run_object.export_model()
# There is only one model output at the current dsl version.
tracker.save_output_model(output_model, task_output_dsl['model'][0] if task_output_dsl.get('model') else 'default')
task.f_status = TaskStatus.SUCCESS
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
task.f_status = TaskStatus.FAILED
finally:
sync_success = False
try:
session.stop()
task.f_end_time = current_timestamp()
task.f_elapsed = task.f_end_time - task.f_start_time
task.f_update_time = current_timestamp()
TaskExecutor.sync_task_status(job_id=job_id, component_name=component_name, task_id=task_id, role=role,
party_id=party_id,
initiator_party_id=job_initiator.get('party_id', None),
initiator_role=job_initiator.get('role', None),
task_info=task.to_json())
sync_success = True
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
schedule_logger().info(
'finish {} {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id, task.f_status if sync_success else TaskStatus.FAILED))
print('finish {} {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id, task.f_status if sync_success else TaskStatus.FAILED))
if fit_intercept:
self.assertListEqual(dense_result, self.gradient_fit_intercept)
else:
self.assertListEqual(dense_result, self.gradient)
sparse_result = hetero_linear_model_gradient.compute_gradient(
sparse_data, fore_gradient, fit_intercept)
sparse_result = [
self.paillier_encrypt.decrypt(iterator)
for iterator in sparse_result
]
self.assertListEqual(dense_result, sparse_result)
def _make_sparse_data(self):
def trans_sparse(instance):
dense_features = instance.features
indices = [i for i in range(len(dense_features))]
sparse_features = SparseVector(indices=indices,
data=dense_features,
shape=len(dense_features))
return Instance(inst_id=None,
features=sparse_features,
label=instance.label)
return self.data_inst.mapValues(trans_sparse)
if __name__ == "__main__":
session.init("1111")
unittest.main()
session.stop()
def onsignal_term(signum, frame):
try:
session.stop()
sys.exit(1)
except Exception as e:
pass
def tearDown(self):
self.table.destroy()
session.stop()
def tearDown(self):
for table in self.table_list:
table.destroy()
session.stop()
