def check_local_package():
request_data = request.json
required_parameters = ['dir']
check_config(request_data, required_parameters)
if os.path.exists(request_data['dir']) and os.path.exists(os.path.join(request_data['dir'], 'roles')):
# get package module version
var_fp = os.path.join(request_data['dir'], 'example', 'var_files', 'exp', 'fate_init')
versions = {}
if os.path.exists(var_fp):
with open(var_fp, 'r') as fin:
versions = yaml.safe_load(fin.read()).get('versions', {})
output = getoutput(f"du -d 1 -h {os.path.join(request_data['dir'], 'roles')}").split('\n')
result = []
for s in output:
items = s.split()
module_name = os.path.basename(items[1])
if module_name != 'roles':
result.append({
'module': module_name,
'size': items[0],
'time': datetime.fromtimestamp(os.path.getctime(items[1])).strftime('%Y%m%d'),
'description': module_name if not versions.get(module_name) else f"{module_name} {versions.get(module_name)}",
'version': versions.get(module_name, None)
})
return get_json_result(data={'version': versions.get('fate_flow'), 'list': result})
return get_json_result(retcode=100, retmsg='package dir not exists.')
def download_package():
request_data = request.json
required_parameters = ['version', 'url']
check_config(request_data, required_parameters)
if not request_data['url']:
raise Exception(f"illegal url {request_data['url']}")
if not request_data['version']:
raise Exception(f"illegal url {request_data['version']}")
os.makedirs(os.path.join(get_project_base_directory(), 'packages'), exist_ok=True)
package_dir = get_package_dir_by_version(request_data.get('version'))
if os.path.exists(package_dir):
return get_json_result(retcode=100,
retmsg=f"Downloading mirror with version {request_data.get('version')} failed, "
f"package dir {package_dir} already exists.")
package_instance = Package.get_or_none(Package.f_version == request_data['version'],
Package.f_status == 'success')
if package_instance:
return get_json_result(retcode=100,
retmsg=f"Downloading mirror with version {request_data.get('version')} failed, "
f"version info has been stored in database.")
request_data['dir'] = package_dir
p = Process(target=do_download, args=(request_data, ))
p.start()
return get_json_result(retmsg=f"Start downloading mirror from url: {request_data.get('url')}.",
data= {'version': request_data.get('version')})
def query_package():
request_data = request.json
required_paramters = ['version']
check_config(request_data, required_paramters)
p = get_package_download_record(request_data['version'])
if p:
return get_json_result(data=p.to_json())
return get_json_result(retcode=100, retmsg=f"can not found version {request_data['version']} record")
def submit_job():
config = request.json
job_id, job_conf_path = JobController.submit_job(job_data=config)
if not job_id and not job_conf_path:
return get_json_result(retcode=100, retmsg=f"Submit job failed.")
else:
data = {'job_id': job_id, 'conf_path': job_conf_path}
return get_json_result(retcode=0,
retmsg=f"Submit job success",
data=data)
def query_task():
tasks = JobSaver.query_task(**request.json)
if not tasks:
return get_json_result(retcode=101,
retmsg='Query task failed, no task found.')
task_filters = [
'play_id', 'task_id', 'task_name', 'role', 'create_time', 'start_time',
'end_time', 'status', 'elapsed'
]
return get_json_result(retmsg="Query task successfully.",
data=tasks[0].to_json(task_filters))
def stop_job():
request_data = request.json
final_status, stop_result = JobController.stop_job(
request_data.get('job_id'))
if final_status:
return get_json_result(
retmsg=f"Stop job {request_data.get('job_id')} successfully.")
return get_json_result(
retcode=100,
retmsg=f"Stop job {request_data.get('job_id')} failed.",
data=stop_result)
def get_job_log():
data = request.json
if not data.get('job_id'):
return get_json_result(retcode=100, retmsg='job id is required')
log_dir = utils.file_utils.get_job_log_directory(job_id=data.get('job_id'))
if os.path.exists(log_dir):
with open(os.path.join(log_dir, 'schedule.log')) as f:
log_data = f.read()
return get_json_result(
retcode=0,
retmsg=f"Query log of job {data.get('job_id')} successfully",
data=log_data)
def toy_test():
request_data = request.json
required_parameters = [
'host', 'guest_party_id', 'host_party_id', 'work_mode'
]
check_config(request_data, required_parameters)
retcode, retmsg = status_utils.toy_test(**request_data)
return get_json_result(retcode=retcode, retmsg=retmsg)
def query_play():
plays = JobSaver.query_play(**request.json)
if not plays:
return get_json_result(retcode=101,
retmsg='Query play failed, no play found.')
play_filters = [
'job_id', 'play_id', 'create_time', 'start_time', 'end_time', 'status',
'elapsed'
]
data = plays[0].to_json(filters=play_filters)
tasks = JobSaver.query_task(reverse=False, **request.json)
if tasks:
task_filters = [
'play_id', 'task_id', 'task_name', 'role', 'create_time',
'start_time', 'end_time', 'status', 'elapsed'
]
data['f_tasks'] = [task.to_json(task_filters) for task in tasks]
return get_json_result(retmsg="Query play successfully.", data=data)
def fast_test():
request_data = request.json
required_parameters = [
'host', 'guest_party_id', 'host_party_id', 'arbiter_party_id'
]
check_config(request_data, required_parameters)
request_data['test_type'] = 'fast'
retcode, retmsg = status_utils.min_test(**request_data)
return get_json_result(retcode=retcode, retmsg=retmsg)
def query_job():
jobs = JobSaver.query_job(**request.json)
if not jobs:
return get_json_result(retcode=101,
retmsg='Query job failed, no job found.')
data = jobs[0].to_json(filters=[
'job_id', 'create_time', 'start_time', 'end_time', 'status', 'elapsed'
])
plays = JobSaver.query_play(reverse=False,
order_by='play_id',
**request.json)
if plays:
play_filters = [
'job_id', 'play_id', 'create_time', 'start_time', 'end_time',
'status', 'elapsed'
]
data['f_plays'] = [
play.to_json(filters=play_filters) for play in plays
]
return get_json_result(retmsg="Query job successfully.", data=data)
def retry_play():
request_data = request.json
required_parameters = ['job_id', 'play_id']
check_config(request_data, required_parameters)
retcode, retmsg = PlayController.retry_play(
job_id=request_data['job_id'],
play_id=request_data['play_id'],
test_mode=TEST_MODE)
return get_json_result(retcode=retcode,
retmsg=retmsg,
data={'play_id': request_data['play_id']})
def get_host_log():
'''
:arg host
:arg job_id
:arg limit [OPTIONAL]
:return:
'''
data = request.json
if data.get('limit'):
retcode, log_data = job_utils.get_host_log(job_id=data.get('job_id'),
host=data.get('host'),
limit=data.get('limit'))
else:
retcode, log_data = job_utils.get_host_log(job_id=data.get('job_id'),
host=data.get('host'))
if retcode == 0:
return get_json_result(retcode=retcode,
retmsg='Query host log successfully',
data=log_data)
return get_json_result(retcode=retcode,
retmsg='Query host log failed, no log files found.')
def status_check():
request_data = request.json
required_parameters = ['party_id']
check_config(request_data, required_parameters)
if request_data.get('module_name'):
retcode, retmsg, data = do_status_check(
party_id=request_data['party_id'],
module_name=request_data['module_name'])
else:
retcode, retmsg, data = do_status_check(
party_id=request_data['party_id'])
return get_json_result(retcode=retcode, retmsg=retmsg, data=data)
def check_ansible():
request_data = request.json
if request_data.get('party_id'):
# TODO consider case that there are more than one records of specified party
parties = JobSaver.query_party(party_id=request_data.get('party_id'))
if parties:
party_info = parties[0].to_json(
filters=['modules', 'version', 'role'])
modules = party_info.get('f_modules', {}).get('data', [])
status = {}
host = None
for module in modules:
if module.get('name') == 'supervisor':
host = module.get('ips')[0]
break
if host:
result = distribute_status_check_task(host=host)
if result and result.get('list'):
for item in result.get('list'):
status[item['name']] = item['status']
if status:
for module in modules:
module['status'] = status.get(module['name'], None)
return get_json_result(retmsg='Query party info successfully',
data={
'party_id':
request_data.get('party_id'),
'list': modules,
'fate_version':
party_info.get('f_version'),
"role": party_info.get('f_role')
})
return get_json_result(
retcode=0,
retmsg=
f"can not found info party {request_data.get('party_id')} in database."
)
return get_json_result(data={'status': 'success'})
def pre_check():
data = request.json
result = []
for ip_addr in data.get('managerNode', []):
if data.get('check_name'):
check_name = data['check_name']
else:
check_name = 'check'
if not status_utils.if_exists(
host=ip_addr, fp=f'/data/projects/check/{check_name}.sh'):
distribute_res = status_utils.distribute_pre_check_script(
host=ip_addr)
if not distribute_res:
result.append({'ip': ip_addr, 'list': []})
continue
result.append(
status_utils.distribute_pre_check_task(host=ip_addr,
task_name=check_name))
return get_json_result(retmsg='pre check success', data=result)
def upload_data():
request_data = request.json
required_parameters = ['host']
check_config(request_data, required_parameters)
retcode, retmsg = status_utils.upload_data_for_min_test(**request_data)
return get_json_result(retcode=retcode, retmsg=retmsg)
def single_test():
request_data = request.json
required_parameters = ['host', 'party_id']
check_config(request_data, required_parameters)
retcode, retmsg = status_utils.single_test(**request_data)
return get_json_result(retcode=retcode, retmsg=retmsg)
def internal_server_error(e):
stat_logger.exception(e)
return get_json_result(retcode=100, retmsg=str(e))
def get_log():
request_data = request.json
retcode, retmsg, data = get_server_log(level=request_data.get('level'))
return get_json_result(retcode=retcode, retmsg=retmsg, data=data)
def query_log():
request_data = request.json
required_parameters = ['host', 'test_type']
check_config(request_data, required_parameters)
retcode, retmsg, data = status_utils.get_test_log(**request_data)
return get_json_result(retcode=retcode, retmsg=retmsg, data=data)
