def start_clean_queue(**kwargs):
tasks = query_task(**kwargs)
if tasks:
for task in tasks:
task_info = get_task_info(task.f_job_id, task.f_role,
task.f_party_id, task.f_component_name)
try:
# clean session
stat_logger.info('start {} {} {} {} session stop'.format(
task.f_job_id, task.f_role, task.f_party_id,
task.f_component_name))
start_session_stop(task)
except:
pass
try:
# clean data table
stat_logger.info('start delete {} {} {} {} data table'.format(
task.f_job_id, task.f_role, task.f_party_id,
task.f_component_name))
data_views = query_data_view(**task_info)
if data_views:
delete_table(data_views)
except:
pass
try:
# clean metric data
stat_logger.info('start delete {} {} {} {} metric data'.format(
task.f_job_id, task.f_role, task.f_party_id,
task.f_component_name))
delete_metric_data(task_info)
except:
pass
else:
raise Exception('no found task')
def component_output_data_table():
request_data = request.json
data_views = query_data_view(**request_data)
if data_views:
return get_json_result(retcode=0, retmsg='success', data={'table_name': data_views[0].f_table_name,
'table_namespace': data_views[0].f_table_namespace})
else:
return get_json_result(retcode=100, retmsg='No found table, please check if the parameters are correct')
def query_data_view():
data_views = data_manager.query_data_view(**request.json)
if not data_views:
return get_json_result(retcode=101, retmsg='find data view failed')
return get_json_result(
retcode=0,
retmsg='success',
data=[data_view.to_json() for data_view in data_views])
def table_delete():
request_data = request.json
data_views = query_data_view(**request_data)
table_name = request_data.get('table_name')
namespace = request_data.get('namespace')
status = False
data = []
if table_name and namespace:
table = session.get_data_table(name=table_name, namespace=namespace)
table.destroy()
data.append({'table_name': table_name, 'namespace': namespace})
elif data_views:
status, data = delete_table(data_views)
else:
return get_json_result(retcode=101, retmsg='no find table')
return get_json_result(retcode=(0 if status else 101),
retmsg=('success' if status else 'failed'),
data=data)
def get_upload_info(jobs_run_conf):
data = []
for job_id, job_run_conf in jobs_run_conf.items():
data_views = query_data_view(job_id=job_id,
component_name='upload_0')[0]
info = {}
table_name = job_run_conf["table_name"][0]
namespace = job_run_conf["namespace"][0]
partition = job_run_conf["partition"][0]
info["upload_info"] = {
"table_name": table_name,
"namespace": namespace,
"partition": partition,
'upload_count': data_views.f_table_count_upload,
'actual_count': data_views.f_table_count_actual
}
info["notes"] = job_run_conf["notes"]
info["meta"] = session.get_data_table_metas(table_name, namespace)
data.append({job_id: info})
return data