def test_get_requests(request_id):
reqs = get_requests(request_id=request_id, with_detail=False)
for req in reqs:
# show_request(req)
print(req)
print(json_dumps(req, sort_keys=True, indent=4))
pass
def get(self, request_id, workload_id):
""" Get details about a specific Request with given id.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: dictionary of an request.
"""
try:
if request_id == 'null':
request_id = None
if workload_id == 'null':
workload_id = None
reqs = get_requests(request_id=request_id,
workload_id=workload_id,
with_request=True,
with_transform=False,
with_processing=False,
with_detail=False,
with_metadata=True)
for req in reqs:
req['relation_map'] = []
workflow = req['request_metadata']['workflow']
if hasattr(workflow, 'get_relation_map'):
req['relation_map'] = workflow.get_relation_map()
# return reqs
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK, data=reqs)
def get(self, request_id, workload_id, with_detail, with_metadata=False):
""" Get details about a specific Request with given id.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: dictionary of an request.
"""
try:
if request_id == 'null':
request_id = None
if workload_id == 'null':
workload_id = None
if with_detail and with_detail.lower() in ['true']:
with_detail = True
else:
with_detail = False
if with_metadata and with_metadata.lower() in ['true']:
with_metadata = True
else:
with_metadata = False
# reqs = get_requests(request_id=request_id, workload_id=workload_id, to_json=True)
reqs = get_requests(request_id=request_id,
workload_id=workload_id,
with_detail=with_detail,
with_metadata=with_metadata)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK, data=reqs)
def get(self):
"""
Get requests.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: A list containing requests.
"""
try:
request_id = self.get_request().args.get('request_id', None)
workload_id = self.get_request().args.get('workload_id', None)
if request_id is None and workload_id is None:
self.generate_http_response(
HTTP_STATUS_CODE.BadRequest,
exc_cls=exceptions.BadRequest.__name__,
exc_msg=
"request_id and workload_id are both None. One should not be None"
)
# reqs = get_requests(request_id=request_id, workload_id=workload_id, to_json=True)
reqs = get_requests(request_id=request_id, workload_id=workload_id)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK, data=reqs)
def show_req_transforms(request_id):
reqs = get_requests(request_id=request_id, with_detail=False)
trfs = get_transforms(request_id=request_id)
prs = get_processings(request_id=request_id)
reqs_times = {}
trfs_times = {}
prs_times = {}
for req in reqs:
reqs_times[req['request_id']] = {'created_at': req['created_at']}
for trf in trfs:
trfs_times[trf['transform_id']] = {'request_id': trf['request_id'], 'created_at': trf['created_at']}
for pr in prs:
prs_times[pr['processing_id']] = {'request_id': pr['request_id'], 'transform_id': pr['transform_id'], 'created_at': pr['created_at']}
table = []
title = ['request_id', 'req_created_at', 'transform_id', 'tf_created_at', 'processing_id', 'pr_created_at', 'request-processing']
table.append(title)
pr_ids = sorted(list(prs_times.keys()))
for pr_id in pr_ids:
req_id = prs_times[pr_id]['request_id']
tf_id = prs_times[pr_id]['transform_id']
row = [req_id, reqs_times[req_id]['created_at'], tf_id, trfs_times[tf_id]['created_at'], pr_id, prs_times[pr_id]['created_at'], (prs_times[pr_id]['created_at'] - reqs_times[req_id]['created_at']).seconds / 60]
table.append(row)
print(tabulate(table))
print(len(workflow.work_sequence.keys()))
print(workflow.works.keys())
print(len(workflow.works.keys()))
work_ids = []
for i_id in workflow.works:
work = workflow.works[i_id]
print(i_id)
print(work.work_name)
print(work.task_name)
print(work.work_id)
work_ids.append(work.work_id)
print(work_ids)
reqs = get_requests(request_id=118, with_detail=False, with_metadata=True)
for req in reqs:
# print(req['request_id'])
# print(rets)
print(json_dumps(req, sort_keys=True, indent=4))
# show_works(req)
pass
sys.exit(0)
# reqs = get_requests()
# print(len(reqs))
"""
for req in reqs:
if req['request_id'] == 113:
# print(req)
dest_rse=ori_work.dest_rse,
rule_id=ori_work.rule_id)
workload_id = req['workload_id']
if not workload_id and 'workload_id' in request_metadata:
workload_id = request_metadata['workload_id']
wf = Workflow()
wf.set_workload_id(workload_id)
wf.add_work(work)
host = get_rest_host()
wm = ClientManager(host=host)
request_id = wm.submit(wf)
# print(request_id)
return request_id
reqs = get_requests()
print(len(reqs))
for req in reqs:
# if req['request_id'] in [3743, 3755, 3769, 3775]:
# if req['request_id'] in [3787, 3789, 3791]:
if req['request_id'] in [11673, 17615, 19289, 19397]:
# print(req)
# print(req['request_metadata']['workflow'].to_dict())
new_req_id = convert_req2reqv2(req)
print("convert old request %s to new request %s" % (req['request_id'], new_req_id))
# print(req['request_metadata']['workflow'].to_dict())
pass
print(work.work_id)
work_ids.append(work.work_id)
print(work_ids)
print("start at: ", datetime.datetime.utcnow())
# 283511, 283517
# reqs = get_requests(request_id=599, with_detail=True, with_metadata=True)
# reqs = get_requests(request_id=283511, with_request=True, with_detail=False, with_metadata=True)
# reqs = get_requests(request_id=298163, with_request=True, with_detail=False, with_metadata=True)
# reqs = get_requests(request_id=298557, with_request=True, with_detail=False, with_metadata=True)
# reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True)
# reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True)
# reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True)
reqs = get_requests(request_id=274,
with_request=True,
with_detail=False,
with_metadata=True)
print("got requests at: ", datetime.datetime.utcnow())
for req in reqs:
# print(req['request_id'])
# print(rets)
# print(json_dumps(req, sort_keys=True, indent=4))
# show_works(req)
pass
workflow = req['request_metadata']['workflow']
print("generating works at: ", datetime.datetime.utcnow())
works = workflow.get_new_works()
print("generated works at: ", datetime.datetime.utcnow())
for work in works:
def get_requests(self, request_id, workload_id, with_request=False, with_transform=False, with_processing=False):
if with_request:
rets, ret_reqs = [], {}
reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=True, with_processing=False, with_metadata=False)
for req in reqs:
if req['request_id'] not in ret_reqs:
ret_reqs[req['request_id']] = {'request_id': req['request_id'],
'workload_id': req['workload_id'],
'status': req['status'].name if req['status'] else req['status'],
'created_at': req['created_at'],
'updated_at': req['updated_at'],
'transforms': {},
'input_total_files': 0,
'input_coll_bytes': 0,
'input_processed_files': 0,
'input_processing_files': 0,
'output_total_files': 0,
'output_coll_bytes': 0,
'output_processed_files': 0,
'output_processing_files': 0
}
if req['transform_status']:
if req['transform_status'].name not in ret_reqs[req['request_id']]['transforms']:
ret_reqs[req['request_id']]['transforms'][req['transform_status'].name] = 0
ret_reqs[req['request_id']]['transforms'][req['transform_status'].name] += 1
if req['input_total_files']:
ret_reqs[req['request_id']]['input_total_files'] += req['input_total_files']
if req['input_coll_bytes']:
ret_reqs[req['request_id']]['input_coll_bytes'] += req['input_coll_bytes']
if req['input_processed_files']:
ret_reqs[req['request_id']]['input_processed_files'] += req['input_processed_files']
if req['input_processing_files']:
ret_reqs[req['request_id']]['input_processing_files'] += req['input_processing_files']
if req['output_total_files']:
ret_reqs[req['request_id']]['output_total_files'] += req['output_total_files']
if req['output_coll_bytes']:
ret_reqs[req['request_id']]['output_coll_bytes'] += req['output_coll_bytes']
if req['output_processed_files']:
ret_reqs[req['request_id']]['output_processed_files'] += req['output_processed_files']
if req['output_processing_files']:
ret_reqs[req['request_id']]['output_processing_files'] += req['output_processing_files']
for req_id in ret_reqs:
rets.append(ret_reqs[req_id])
return rets
elif with_transform:
rets = []
reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=True, with_processing=False, with_metadata=False)
for req in reqs:
ret = {'request_id': req['request_id'],
'transform_id': req['transform_id'],
'workload_id': req['workload_id'],
'transform_workload_id': req['transform_workload_id'],
'transform_type': req['transform_type'].name if req['transform_type'] else req['transform_type'],
'output_coll_scope': req['output_coll_scope'],
'output_coll_name': req['output_coll_name'],
'transform_status': req['transform_status'].name if req['transform_status'] else req['transform_status'],
'transform_created_at': req['transform_created_at'],
'transform_updated_at': req['transform_updated_at'],
'transform_finished_at': req['transform_finished_at'],
'input_total_files': req['input_total_files'] if req['input_total_files'] else 0,
'input_coll_bytes': req['input_coll_bytes'] if req['input_coll_bytes'] else 0,
'input_processed_files': req['input_processed_files'] if req['input_processed_files'] else 0,
'input_processing_files': req['input_processing_files'] if req['input_processing_files'] else 0,
'output_total_files': req['output_total_files'] if req['output_total_files'] else 0,
'output_coll_bytes': req['output_coll_bytes'] if req['output_coll_bytes'] else 0,
'output_processed_files': req['output_processed_files'] if req['output_processed_files'] else 0,
'output_processing_files': req['output_processing_files'] if req['output_processing_files'] else 0,
'errors': req['errors']
}
rets.append(ret)
return rets
elif with_processing:
rets = []
reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=False, with_processing=True, with_metadata=False)
for req in reqs:
ret = {'request_id': req['request_id'],
'workload_id': req['workload_id'],
'processing_id': req['processing_id'],
'processing_status': req['processing_status'].name if req['processing_status'] else req['processing_status'],
'processing_created_at': req['processing_created_at'],
'processing_updated_at': req['processing_updated_at'],
'processing_finished_at': req['processing_finished_at']
}
rets.append(ret)
return rets
else:
rets = []
reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=False, with_processing=False, with_metadata=False)
for req in reqs:
ret = {'request_id': req['request_id'],
'workload_id': req['workload_id'],
'status': req['status'].name if req['status'] else req['status'],
'created_at': req['created_at'],
'updated_at': req['updated_at']
}
rets.append(ret)
return rets
