def release_inputs_by_collection_old(to_release_inputs): update_contents = [] for coll_id in to_release_inputs: to_release_contents = to_release_inputs[coll_id] if to_release_contents: to_release = to_release_contents[0] to_release_names_available = [] to_release_names_fake_available = [] to_release_names_final_failed = [] to_release_names_missing = [] for to_release_content in to_release_contents: if (to_release_content['status'] in [ContentStatus.Available] # noqa: W503 or to_release_content['substatus'] in [ContentStatus.Available]): # noqa: W503 to_release_names_available.append(to_release_content['name']) elif (to_release_content['status'] in [ContentStatus.FakeAvailable] # noqa: W503 or to_release_content['substatus'] in [ContentStatus.FakeAvailable]): # noqa: W503 to_release_names_fake_available.append(to_release_content['name']) elif (to_release_content['status'] in [ContentStatus.FinalFailed] # noqa: W503 or to_release_content['substatus'] in [ContentStatus.FinalFailed]): # noqa: W503 to_release_names_final_failed.append(to_release_content['name']) elif (to_release_content['status'] in [ContentStatus.Missing] # noqa: W503 or to_release_content['substatus'] in [ContentStatus.Missing]): # noqa: W503 to_release_names_missing.append(to_release_content['name']) contents = orm_contents.get_input_contents(request_id=to_release['request_id'], coll_id=to_release['coll_id'], name=None) for content in contents: if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503 if (content['status'] not in [ContentStatus.Available] # noqa: W503 and content['name'] in to_release_names_available): # noqa: W503 update_content = {'content_id': content['content_id'], 'substatus': ContentStatus.Available, 'status': ContentStatus.Available} update_contents.append(update_content) elif (content['status'] not in [ContentStatus.FakeAvailable] # noqa: W503 and content['name'] in to_release_names_fake_available): # noqa: W503 update_content = {'content_id': content['content_id'], 'substatus': ContentStatus.FakeAvailable, 'status': ContentStatus.FakeAvailable} update_contents.append(update_content) elif (content['status'] not in [ContentStatus.FinalFailed] # noqa: W503 and content['name'] in to_release_names_final_failed): # noqa: W503 update_content = {'content_id': content['content_id'], 'substatus': ContentStatus.FinalFailed, 'status': ContentStatus.FinalFailed} update_contents.append(update_content) elif (content['status'] not in [ContentStatus.Missing] # noqa: W503 and content['name'] in to_release_names_missing): # noqa: W503 update_content = {'content_id': content['content_id'], 'substatus': ContentStatus.Missing, 'status': ContentStatus.Missing} update_contents.append(update_content) return update_contents
def release_inputs(to_release_inputs): update_contents = [] for to_release in to_release_inputs: contents = orm_contents.get_input_contents(request_id=to_release['request_id'], coll_id=to_release['coll_id'], name=to_release['name']) for content in contents: if content['content_relation_type'] == ContentRelationType.InputDependency: update_content = {'content_id': content['content_id'], 'substatus': to_release['substatus'], 'status': to_release['status']} update_contents.append(update_content) return update_contents
def release_inputs_by_collection(to_release_inputs, final=False): update_contents = [] status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] for coll_id in to_release_inputs: to_release_contents = to_release_inputs[coll_id] if to_release_contents: to_release_status = {} for to_release_content in to_release_contents: if (to_release_content['status'] in status_to_check): to_release_status[to_release_content['name']] = to_release_content['status'] elif (to_release_content['substatus'] in status_to_check): to_release_status[to_release_content['name']] = to_release_content['substatus'] # print("to_release_status: %s" % str(to_release_status)) contents = orm_contents.get_input_contents(request_id=to_release_contents[0]['request_id'], coll_id=to_release_contents[0]['coll_id'], name=None) # print("contents: %s" % str(contents)) unfinished_contents_dict = {} for content in contents: if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503 if content['status'] not in status_to_check: if content['name'] not in unfinished_contents_dict: unfinished_contents_dict[content['name']] = [] content_short = {'content_id': content['content_id'], 'status': content['status']} unfinished_contents_dict[content['name']].append(content_short) intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys() intersection_keys = list(intersection_keys) logging.debug("release_inputs_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10]))) for name in intersection_keys: matched_content_status = to_release_status[name] matched_contents = unfinished_contents_dict[name] for matched_content in matched_contents: if (matched_content['status'] != matched_content_status): update_content = {'content_id': matched_content['content_id'], 'substatus': matched_content_status, 'status': matched_content_status} update_contents.append(update_content) return update_contents
def release_inputs_test(): to_release_inputs = {3498: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+3_isr_3020111900038_94+qgraphNodeId:3+qgraphId:1626129062.5744567-119392', 'content_id': 2248918, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411522}, 'transform_id': 1749, 'storage_id': None}, {'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+2_isr_3020111900032_94+qgraphNodeId:2+qgraphId:1626129062.5744567-119392', 'content_id': 2248919, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411523}, 'transform_id': 1749, 'storage_id': None}, {'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+4_isr_3020111900040_94+qgraphNodeId:4+qgraphId:1626129062.5744567-119392', 'content_id': 2248920, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411524}, 'transform_id': 1749, 'storage_id': None}, {'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+1_isr_3020111900036_94+qgraphNodeId:1+qgraphId:1626129062.5744567-119392', 'content_id': 2248921, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411525}, 'transform_id': 1749, 'storage_id': None}, {'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+0_isr_3020111900034_94+qgraphNodeId:0+qgraphId:1626129062.5744567-119392', 'content_id': 2248922, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411526}, 'transform_id': 1749, 'storage_id': None} ]} to_release_inputs = {4042: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+13_isr_257768_161+1626299263.3909254-24148+13', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254913, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412272}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+2_isr_212071_54+1626299263.3909254-24148+2', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254914, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412273}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+10_isr_456716_99+1626299263.3909254-24148+10', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254915, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412274}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+34_isr_407919_130+1626299263.3909254-24148+34', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254916, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412275}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+23_isr_254379_48+1626299263.3909254-24148+23', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254917, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412276}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 6, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+11_isr_37657_141+1626299263.3909254-24148+11', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254918, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412277}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 {'map_id': 7, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+31_isr_226983_36+1626299263.3909254-24148+31', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254919, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412278}, 'transform_id': 2021, 'storage_id': None}]} # noqa E501 for coll_id in to_release_inputs: contents = get_input_contents(request_id=to_release_inputs[coll_id][0]['request_id'], coll_id=coll_id, name=None) print(len(contents)) in_dep_contents = [] for content in contents: if (content['content_relation_type'] == ContentRelationType.InputDependency): in_dep_contents.append(content) print(len(in_dep_contents)) update_contents = release_inputs_by_collection(to_release_inputs) print(update_contents) update_contents = release_inputs_by_collection(to_release_inputs, final=True) print(update_contents)
def poll_inputs_dependency_by_collection(unfinished_inputs): update_contents = [] status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] for coll_id in unfinished_inputs: unfinished_contents = unfinished_inputs[coll_id] contents = orm_contents.get_input_contents(request_id=unfinished_contents[0]['request_id'], coll_id=unfinished_contents[0]['coll_id'], name=None) logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): unfinished_contents[:10]: %s" % (coll_id, str(unfinished_contents[:10]))) to_release_status = {} for content in contents: if (content['content_relation_type'] == ContentRelationType.Output): # noqa: W503 if content['status'] in status_to_check: to_release_status[content['name']] = content['status'] elif content['substatus'] in status_to_check: to_release_status[content['name']] = content['substatus'] unfinished_contents_dict = {} for content in unfinished_contents: if content['name'] not in unfinished_contents_dict: unfinished_contents_dict[content['name']] = [] content_short = {'content_id': content['content_id'], 'status': content['status']} unfinished_contents_dict[content['name']].append(content_short) intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys() intersection_keys = list(intersection_keys) logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10]))) for name in intersection_keys: matched_content_status = to_release_status[name] matched_contents = unfinished_contents_dict[name] for matched_content in matched_contents: if (matched_content['status'] != matched_content_status): update_content = {'content_id': matched_content['content_id'], 'substatus': matched_content_status, 'status': matched_content_status} update_contents.append(update_content) # if len(unfinished_contents_dict.keys()) < len(to_release_status.keys()): # for name, content in unfinished_contents_dict.items(): # if name in to_release_status: # matched_content_status = to_release_status[name] # if (content['status'] != matched_content_status): # update_content = {'content_id': content['content_id'], # 'substatus': matched_content_status, # 'status': matched_content_status} # update_contents.append(update_content) # else: # for name, status in to_release_status.items(): # if name in unfinished_contents_dict: # matched_content = unfinished_contents_dict[name] # if (matched_content['status'] != status): # update_content = {'content_id': matched_content['content_id'], # 'substatus': status, # 'status': status} # update_contents.append(update_content) # for content in unfinished_contents: # if content['name'] in to_release_status: # matched_content_status = to_release_status[content['name']] # if (content['status'] != matched_content_status): # update_content = {'content_id': content['content_id'], # 'substatus': matched_content_status, # 'status': matched_content_status} # update_contents.append(update_content) return update_contents