def log(self, job_id, output_path):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=['job_id', 'output_path'])
job_id = config_data['job_id']
tar_file_name = 'job_{}_log.tar.gz'.format(job_id)
extract_dir = os.path.join(config_data['output_path'],
'job_{}_log'.format(job_id))
with closing(
self._get(url='job/log',
handle_result=False,
json=config_data,
stream=True)) as response:
if response.status_code == 200:
download_from_request(http_response=response,
tar_file_name=tar_file_name,
extract_dir=extract_dir)
response = {
'retcode':
0,
'directory':
extract_dir,
'retmsg':
'download successfully, please check {} directory'.format(
extract_dir)
}
else:
response = response.json()
return response
def config(self, job_id, role, party_id, output_path):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(
config=config_data,
required_arguments=['job_id', 'role', 'party_id', 'output_path'])
response = self._post(url='job/config', json=config_data)
if response['retcode'] == 0:
job_id = response['data']['job_id']
download_directory = os.path.join(config_data['output_path'],
'job_{}_config'.format(job_id))
os.makedirs(download_directory, exist_ok=True)
for k, v in response['data'].items():
if k == 'job_id':
continue
with open('{}/{}.json'.format(download_directory, k),
'w') as fw:
json.dump(v, fw, indent=4)
del response['data']['dsl']
del response['data']['runtime_conf']
response['directory'] = download_directory
response[
'retmsg'] = 'download successfully, please check {} directory'.format(
download_directory)
return response
def metrics(self, job_id, role, party_id, component_name):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=[
'job_id', 'component_name', 'role', 'party_id'
])
return self._post(url='tracking/component/metrics', json=config_data)
def list_checkpoints(self, **kwargs):
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=[
'role', 'party_id', 'model_id', 'model_version',
'component_name'
])
return self._post(url='checkpoint/list', json=config_data)
def clean(self,
job_id=None,
role=None,
party_id=None,
component_name=None):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data, required_arguments=['job_id'])
return self._post(url='job/clean', json=config_data)
def get_checkpoint(self, **kwargs):
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=[
'role', 'party_id', 'model_id', 'model_version',
'component_name'
])
if len(config_data.keys() & {'step_index', 'step_name'}) != 1:
raise KeyError('step_index or step_name is required')
return self._post(url='checkpoint/get', json=config_data)
def get_summary(self, job_id, role, party_id, component_name):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=[
'job_id', 'component_name', 'role', 'party_id'
])
res = self._post(url='tracking/component/summary/download',
handle_result=True,
json=config_data)
if not res.get('data'):
res['data'] = {}
return res
def output_data(self,
job_id,
role,
party_id,
component_name,
output_path,
limit=-1):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=[
'job_id', 'component_name', 'role', 'party_id',
'output_path'
])
tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(
config_data['job_id'], config_data['component_name'],
config_data['role'], config_data['party_id'])
extract_dir = os.path.join(config_data['output_path'],
tar_file_name.replace('.tar.gz', ''))
with closing(
self._get(url='tracking/component/output/data/download',
handle_result=False,
json=config_data,
stream=True)) as response:
if response.status_code == 200:
try:
download_from_request(http_response=response,
tar_file_name=tar_file_name,
extract_dir=extract_dir)
response = {
'retcode':
0,
'directory':
extract_dir,
'retmsg':
'download successfully, please check {} directory'.
format(extract_dir)
}
except BaseException:
response = {
'retcode':
100,
'retmsg':
'download failed, please check if the parameters are correct'
}
else:
response = response.json()
return response
def delete(self, src_party_id, src_role):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=['src_party_id', 'src_role'])
return self._post(url='permission/delete/privilege', json=config_data)
def stop(self, job_id):
job_id = str(job_id)
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data, required_arguments=['job_id'])
return self._post(url='job/stop', json=config_data)
def info(self, namespace, table_name):
kwargs = locals()
config_data, dsl_data = preprocess(**kwargs)
check_config(config=config_data,
required_arguments=['namespace', 'table_name'])
return self._post(url='table/table_info', json=config_data)
