def release_inputs_by_collection_old(to_release_inputs):
update_contents = []
for coll_id in to_release_inputs:
to_release_contents = to_release_inputs[coll_id]
if to_release_contents:
to_release = to_release_contents[0]
to_release_names_available = []
to_release_names_fake_available = []
to_release_names_final_failed = []
to_release_names_missing = []
for to_release_content in to_release_contents:
if (to_release_content['status'] in [ContentStatus.Available] # noqa: W503
or to_release_content['substatus'] in [ContentStatus.Available]): # noqa: W503
to_release_names_available.append(to_release_content['name'])
elif (to_release_content['status'] in [ContentStatus.FakeAvailable] # noqa: W503
or to_release_content['substatus'] in [ContentStatus.FakeAvailable]): # noqa: W503
to_release_names_fake_available.append(to_release_content['name'])
elif (to_release_content['status'] in [ContentStatus.FinalFailed] # noqa: W503
or to_release_content['substatus'] in [ContentStatus.FinalFailed]): # noqa: W503
to_release_names_final_failed.append(to_release_content['name'])
elif (to_release_content['status'] in [ContentStatus.Missing] # noqa: W503
or to_release_content['substatus'] in [ContentStatus.Missing]): # noqa: W503
to_release_names_missing.append(to_release_content['name'])
contents = orm_contents.get_input_contents(request_id=to_release['request_id'],
coll_id=to_release['coll_id'],
name=None)
for content in contents:
if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503
if (content['status'] not in [ContentStatus.Available] # noqa: W503
and content['name'] in to_release_names_available): # noqa: W503
update_content = {'content_id': content['content_id'],
'substatus': ContentStatus.Available,
'status': ContentStatus.Available}
update_contents.append(update_content)
elif (content['status'] not in [ContentStatus.FakeAvailable] # noqa: W503
and content['name'] in to_release_names_fake_available): # noqa: W503
update_content = {'content_id': content['content_id'],
'substatus': ContentStatus.FakeAvailable,
'status': ContentStatus.FakeAvailable}
update_contents.append(update_content)
elif (content['status'] not in [ContentStatus.FinalFailed] # noqa: W503
and content['name'] in to_release_names_final_failed): # noqa: W503
update_content = {'content_id': content['content_id'],
'substatus': ContentStatus.FinalFailed,
'status': ContentStatus.FinalFailed}
update_contents.append(update_content)
elif (content['status'] not in [ContentStatus.Missing] # noqa: W503
and content['name'] in to_release_names_missing): # noqa: W503
update_content = {'content_id': content['content_id'],
'substatus': ContentStatus.Missing,
'status': ContentStatus.Missing}
update_contents.append(update_content)
return update_contents
def release_inputs(to_release_inputs):
update_contents = []
for to_release in to_release_inputs:
contents = orm_contents.get_input_contents(request_id=to_release['request_id'],
coll_id=to_release['coll_id'],
name=to_release['name'])
for content in contents:
if content['content_relation_type'] == ContentRelationType.InputDependency:
update_content = {'content_id': content['content_id'],
'substatus': to_release['substatus'],
'status': to_release['status']}
update_contents.append(update_content)
return update_contents
def release_inputs_by_collection(to_release_inputs, final=False):
update_contents = []
status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing]
for coll_id in to_release_inputs:
to_release_contents = to_release_inputs[coll_id]
if to_release_contents:
to_release_status = {}
for to_release_content in to_release_contents:
if (to_release_content['status'] in status_to_check):
to_release_status[to_release_content['name']] = to_release_content['status']
elif (to_release_content['substatus'] in status_to_check):
to_release_status[to_release_content['name']] = to_release_content['substatus']
# print("to_release_status: %s" % str(to_release_status))
contents = orm_contents.get_input_contents(request_id=to_release_contents[0]['request_id'],
coll_id=to_release_contents[0]['coll_id'],
name=None)
# print("contents: %s" % str(contents))
unfinished_contents_dict = {}
for content in contents:
if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503
if content['status'] not in status_to_check:
if content['name'] not in unfinished_contents_dict:
unfinished_contents_dict[content['name']] = []
content_short = {'content_id': content['content_id'], 'status': content['status']}
unfinished_contents_dict[content['name']].append(content_short)
intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys()
intersection_keys = list(intersection_keys)
logging.debug("release_inputs_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10])))
for name in intersection_keys:
matched_content_status = to_release_status[name]
matched_contents = unfinished_contents_dict[name]
for matched_content in matched_contents:
if (matched_content['status'] != matched_content_status):
update_content = {'content_id': matched_content['content_id'],
'substatus': matched_content_status,
'status': matched_content_status}
update_contents.append(update_content)
return update_contents
def release_inputs_test():
to_release_inputs = {3498: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset',
'substatus': ContentStatus.Available, 'path': None,
'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+3_isr_3020111900038_94+qgraphNodeId:3+qgraphId:1626129062.5744567-119392',
'content_id': 2248918, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None,
'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678',
'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output,
'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411522}, 'transform_id': 1749, 'storage_id': None},
{'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset',
'substatus': ContentStatus.Available, 'path': None,
'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+2_isr_3020111900032_94+qgraphNodeId:2+qgraphId:1626129062.5744567-119392',
'content_id': 2248919, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None,
'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678',
'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output,
'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411523}, 'transform_id': 1749, 'storage_id': None},
{'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset',
'substatus': ContentStatus.Available, 'path': None,
'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+4_isr_3020111900040_94+qgraphNodeId:4+qgraphId:1626129062.5744567-119392',
'content_id': 2248920, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None,
'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678',
'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output,
'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411524}, 'transform_id': 1749, 'storage_id': None},
{'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset',
'substatus': ContentStatus.Available, 'path': None,
'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+1_isr_3020111900036_94+qgraphNodeId:1+qgraphId:1626129062.5744567-119392',
'content_id': 2248921, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None,
'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678',
'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output,
'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411525}, 'transform_id': 1749, 'storage_id': None},
{'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset',
'substatus': ContentStatus.Available, 'path': None,
'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+0_isr_3020111900034_94+qgraphNodeId:0+qgraphId:1626129062.5744567-119392',
'content_id': 2248922, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None,
'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678',
'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output,
'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411526}, 'transform_id': 1749, 'storage_id': None}
]}
to_release_inputs = {4042: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+13_isr_257768_161+1626299263.3909254-24148+13', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254913, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412272}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+2_isr_212071_54+1626299263.3909254-24148+2', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254914, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412273}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+10_isr_456716_99+1626299263.3909254-24148+10', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254915, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412274}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+34_isr_407919_130+1626299263.3909254-24148+34', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254916, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412275}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+23_isr_254379_48+1626299263.3909254-24148+23', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254917, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412276}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 6, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+11_isr_37657_141+1626299263.3909254-24148+11', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254918, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412277}, 'transform_id': 2021, 'storage_id': None}, # noqa E501
{'map_id': 7, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+31_isr_226983_36+1626299263.3909254-24148+31', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254919, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412278}, 'transform_id': 2021, 'storage_id': None}]} # noqa E501
for coll_id in to_release_inputs:
contents = get_input_contents(request_id=to_release_inputs[coll_id][0]['request_id'],
coll_id=coll_id,
name=None)
print(len(contents))
in_dep_contents = []
for content in contents:
if (content['content_relation_type'] == ContentRelationType.InputDependency):
in_dep_contents.append(content)
print(len(in_dep_contents))
update_contents = release_inputs_by_collection(to_release_inputs)
print(update_contents)
update_contents = release_inputs_by_collection(to_release_inputs, final=True)
print(update_contents)
def poll_inputs_dependency_by_collection(unfinished_inputs):
update_contents = []
status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing]
for coll_id in unfinished_inputs:
unfinished_contents = unfinished_inputs[coll_id]
contents = orm_contents.get_input_contents(request_id=unfinished_contents[0]['request_id'],
coll_id=unfinished_contents[0]['coll_id'],
name=None)
logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): unfinished_contents[:10]: %s" % (coll_id, str(unfinished_contents[:10])))
to_release_status = {}
for content in contents:
if (content['content_relation_type'] == ContentRelationType.Output): # noqa: W503
if content['status'] in status_to_check:
to_release_status[content['name']] = content['status']
elif content['substatus'] in status_to_check:
to_release_status[content['name']] = content['substatus']
unfinished_contents_dict = {}
for content in unfinished_contents:
if content['name'] not in unfinished_contents_dict:
unfinished_contents_dict[content['name']] = []
content_short = {'content_id': content['content_id'], 'status': content['status']}
unfinished_contents_dict[content['name']].append(content_short)
intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys()
intersection_keys = list(intersection_keys)
logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10])))
for name in intersection_keys:
matched_content_status = to_release_status[name]
matched_contents = unfinished_contents_dict[name]
for matched_content in matched_contents:
if (matched_content['status'] != matched_content_status):
update_content = {'content_id': matched_content['content_id'],
'substatus': matched_content_status,
'status': matched_content_status}
update_contents.append(update_content)
# if len(unfinished_contents_dict.keys()) < len(to_release_status.keys()):
# for name, content in unfinished_contents_dict.items():
# if name in to_release_status:
# matched_content_status = to_release_status[name]
# if (content['status'] != matched_content_status):
# update_content = {'content_id': content['content_id'],
# 'substatus': matched_content_status,
# 'status': matched_content_status}
# update_contents.append(update_content)
# else:
# for name, status in to_release_status.items():
# if name in unfinished_contents_dict:
# matched_content = unfinished_contents_dict[name]
# if (matched_content['status'] != status):
# update_content = {'content_id': matched_content['content_id'],
# 'substatus': status,
# 'status': status}
# update_contents.append(update_content)
# for content in unfinished_contents:
# if content['name'] in to_release_status:
# matched_content_status = to_release_status[content['name']]
# if (content['status'] != matched_content_status):
# update_content = {'content_id': content['content_id'],
# 'substatus': matched_content_status,
# 'status': matched_content_status}
# update_contents.append(update_content)
return update_contents