def get(self, coll_scope, coll_name, request_id, workload_id,
relation_type, status):
""" Get contents by coll_scope, coll_name, request_id, workload_id and relation_type.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: contents.
"""
try:
if coll_scope in ['null', 'None']:
coll_scope = None
if coll_name in ['null', 'None']:
coll_name = None
if request_id in ['null', 'None']:
request_id = None
else:
request_id = int(request_id)
if workload_id in ['null', 'None']:
workload_id = None
else:
workload_id = int(workload_id)
if relation_type in ['null', 'None']:
relation_type = None
else:
relation_type = int(relation_type)
if status in ['null', 'None']:
status = None
else:
status = int(status)
rets = get_contents(coll_scope=coll_scope,
coll_name=coll_name,
request_id=request_id,
workload_id=workload_id,
relation_type=relation_type,
status=status,
to_json=False)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(traceback.format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1):
collections = core_catalog.get_collections(request_id=request_id)
contents = core_catalog.get_contents(request_id=request_id)
get_dep_link(collections,
contents,
request_id,
transform_id,
coll_id,
scope,
name,
step=step)
def release_inputs(self, request_id):
contents = core_catalog.get_contents(request_id=request_id, status=ContentStatus.Available)
ret_contents = {}
for content in contents:
if content['content_relation_type'] == ContentRelationType.Output: # InputDependency
if content['coll_id'] not in ret_contents:
ret_contents[content['coll_id']] = []
ret_contents[content['coll_id']].append(content)
updated_contents = core_transforms.release_inputs_by_collection(ret_contents)
core_catalog.update_contents(updated_contents)
def get_contents(coll_scope=None,
coll_name=None,
request_id=None,
workload_id=None,
relation_type=None):
"""
Get contents with collection scope, collection name, request id, workload id and relation type.
:param coll_scope: scope of the collection.
:param coll_name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param relation_type: The relation type between the collection and transform: input, outpu, logs and etc.
:returns: dict of contents
"""
return catalog.get_contents(coll_scope=coll_scope,
coll_name=coll_name,
request_id=request_id,
workload_id=workload_id,
relation_type=relation_type)
def process_output_collection(self, coll):
"""
Process output collection
"""
if coll['coll_metadata'] and 'to_register' in coll[
'coll_metadata'] and coll['coll_metadata'][
'to_register'] is True:
contents = core_catalog.get_contents(coll_id=coll['coll_id'])
registered_contents = self.get_contents(coll['scope'],
coll['name'])
registered_content_names = [
con['name'] for con in registered_contents
]
new_contents = []
for content in contents:
if content['name'] not in registered_content_names:
new_contents.append(content)
if new_contents:
self.register_contents(coll['scope'], coll['name'],
new_contents)
is_input_collection_all_processed = False
if coll['coll_metadata'] and 'input_collections' in coll[
'coll_metadata'] and coll['coll_metadata']['input_collections']:
input_coll_list = coll['coll_metadata']['input_collections']
is_input_collection_all_processed = self.is_input_collection_all_processed(
input_coll_list)
contents_statistics = core_catalog.get_content_status_statistics(
coll_id=coll['coll_id'])
contents_statistics_with_name = {}
for key in contents_statistics:
contents_statistics_with_name[key.name] = contents_statistics[key]
content_status_keys = list(contents_statistics.keys())
total_files = sum(contents_statistics.values())
new_files = 0
processed_files = 0
output_metadata = None
if ContentStatus.Available in contents_statistics:
processed_files += contents_statistics[ContentStatus.Available]
if ContentStatus.Available.value in contents_statistics:
processed_files += contents_statistics[
ContentStatus.Available.value]
if ContentStatus.New in contents_statistics:
new_files += contents_statistics[ContentStatus.New]
if ContentStatus.New.value in contents_statistics:
new_files += contents_statistics[ContentStatus.New.value]
coll_msg = None
if not is_input_collection_all_processed:
coll_status = CollectionStatus.Processing
elif content_status_keys == [
ContentStatus.Available
] or content_status_keys == [ContentStatus.Available.value]:
coll_status = CollectionStatus.Closed
transform = core_transforms.get_transform(
coll['coll_metadata']['transform_id'])
if 'processing_id' in transform['transform_metadata']:
processing = core_processings.get_processing(
transform['transform_metadata']['processing_id'])
output_metadata = processing['output_metadata']
if transform['transform_type'] in [
TransformType.StageIn, TransformType.StageIn.value
]:
msg_type = 'collection_stagein'
msg_type_c = MessageType.StageInCollection
elif transform['transform_type'] in [
TransformType.ActiveLearning,
TransformType.ActiveLearning.value
]:
msg_type = 'collection_activelearning'
msg_type_c = MessageType.ActiveLearningCollection
elif transform['transform_type'] in [
TransformType.HyperParameterOpt,
TransformType.HyperParameterOpt.value
]:
msg_type = 'collection_hyperparameteropt'
msg_type_c = MessageType.HyperParameterOptCollection
else:
msg_type = 'collection_unknown'
msg_type_c = MessageType.UnknownCollection
msg_content = {
'msg_type':
msg_type,
'workload_id':
coll['coll_metadata']['workload_id']
if 'workload_id' in coll['coll_metadata'] else None,
'collections': [{
'scope': coll['scope'],
'name': coll['name'],
'status': 'Available'
}],
'output':
output_metadata
}
coll_msg = {
'msg_type':
msg_type_c,
'status':
MessageStatus.New,
'source':
MessageSource.Transporter,
'transform_id':
coll['coll_metadata']['transform_id']
if 'transform_id' in coll['coll_metadata'] else None,
'num_contents':
1,
'msg_content':
msg_content
}
elif content_status_keys == [
ContentStatus.FinalFailed
] or content_status_keys == [ContentStatus.FinalFailed.value]:
coll_status = CollectionStatus.Failed
elif (len(content_status_keys) == 2 # noqa: W503
and (ContentStatus.FinalFailed in content_status_keys
or ContentStatus.FinalFailed.value in content_status_keys
) # noqa: W503
and (ContentStatus.Available in content_status_keys
or ContentStatus.Available.value
in content_status_keys)): # noqa: W503
coll_status = CollectionStatus.SubClosed
elif (ContentStatus.New in content_status_keys
or ContentStatus.New.value in content_status_keys # noqa: W503
or ContentStatus.Failed in content_status_keys or
ContentStatus.Failed.value in content_status_keys): # noqa: W503
coll_status = CollectionStatus.Processing
coll_metadata = coll['coll_metadata']
if not coll_metadata:
coll_metadata = {}
coll_metadata['status_statistics'] = contents_statistics_with_name
coll_metadata['output_metadata'] = output_metadata
ret_coll = {
'coll_id': coll['coll_id'],
'total_files': total_files,
'status': coll_status,
'new_files': new_files,
'processing_files': 0,
'processed_files': processed_files,
'coll_metadata': coll_metadata,
'coll_msg': coll_msg
}
return ret_coll
def test_catalog_core(self):
""" Catalog (Core): Test catalog core functions """
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
coll_log_properties = copy.deepcopy(coll_properties)
coll_log_properties['relation_type'] = CollectionRelationType.Log
coll_log_properties['name'] = coll_log_properties['name'] + '_log'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
origin_coll_log_id = add_collection(**coll_log_properties)
origin_coll_id_list = [origin_coll_input_id, origin_coll_output_id, origin_coll_log_id]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_input_properties['name'], request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Input)
if key == 'name':
assert_equal(coll[key], coll_input_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_output_properties['name'], request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Output)
if key == 'name':
assert_equal(coll[key], coll_output_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_log_properties['name'], request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
assert_equal(coll[key], coll_log_properties[key])
trans_properties1 = copy.deepcopy(trans_properties)
origin_trans_id1 = add_transform(**trans_properties1)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties1['name'] + '_1'
coll_properties1['transform_id'] = origin_trans_id1
coll_input_properties1 = copy.deepcopy(coll_properties1)
coll_input_properties1['name'] = coll_input_properties1['name'] + '_input'
coll_output_properties1 = copy.deepcopy(coll_properties1)
coll_output_properties1['relation_type'] = CollectionRelationType.Output
coll_output_properties1['name'] = coll_output_properties1['name'] + '_output'
coll_log_properties1 = copy.deepcopy(coll_properties1)
coll_log_properties1['relation_type'] = CollectionRelationType.Log
coll_log_properties1['name'] = coll_log_properties1['name'] + '_log'
origin_coll_input_id1 = add_collection(**coll_input_properties1)
origin_coll_output_id1 = add_collection(**coll_output_properties1)
origin_coll_log_id1 = add_collection(**coll_log_properties1)
origin_coll_id_list1 = [origin_coll_input_id1, origin_coll_output_id1, origin_coll_log_id1]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
content_input_properties = get_content_properties()
content_input_properties['coll_id'] = origin_coll_input_id
origin_content_input_id_0_100 = add_content(returning_id=True, **content_input_properties)
content_input_properties_100_200 = copy.deepcopy(content_input_properties)
content_input_properties_100_200['min_id'] = 100
content_input_properties_100_200['max_id'] = 200
origin_content_input_id_100_200 = add_content(returning_id=True, **content_input_properties_100_200)
content_input_properties_name1 = copy.deepcopy(content_input_properties)
content_input_properties_name1['name'] = content_input_properties_name1['name'] + '_1'
content_input_properties_name1_id = add_content(returning_id=True, **content_input_properties_name1)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_output_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
with assert_raises(exceptions.WrongParameterException):
get_contents(coll_scope=None, coll_name=None, request_id=None, workload_id=None, relation_type=None)
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_input_scope_name, coll_output_scope_name])
input_contents = coll_contents[coll_input_scope_name]['contents']
output_contents = coll_contents[coll_output_scope_name]['contents']
assert_equal(len(input_contents), 3)
assert_equal(len(output_contents), 4)
input_content_ids = [input_content['content_id'] for input_content in input_contents]
assert_equal(input_content_ids, [origin_content_input_id_0_100, origin_content_input_id_100_200, content_input_properties_name1_id])
output_content_ids = [output_content['content_id'] for output_content in output_contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Input)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_input_scope_name])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_output_scope_name])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
register_output_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
contents=contents, request_id=origin_request_id, workload_id=None)
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
def put(self, workload_id, request_id, id, loss):
""" Update the loss for the hyper parameter.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
404 Not Found
500 Internal Error
"""
try:
if workload_id == 'null':
workload_id = None
if request_id == 'null':
request_id = None
if workload_id is None and request_id is None:
error = "One of workload_id and request_id should not be None or empty"
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
try:
contents = catalog.get_contents(
request_id=request_id,
workload_id=workload_id,
relation_type=CollectionRelationType.Output)
if id:
new_contents = []
for content in contents:
if str(content['name']) == str(id):
new_contents.append(content)
contents = new_contents
content = contents[0]
loss = float(loss)
content_id = content['content_id']
point = content['path']
param, origin_loss = json.loads(point)
params = {
'path': json.dumps((param, loss)),
'substatus': ContentStatus.Available
}
catalog.update_content(content_id, params)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data={
'status': 0,
'message': 'update successfully'
})
def get(self, workload_id, request_id, id=None, status=None, limit=None):
""" Get hyper parameters.
:param request_id: The id of the request.
:param status: status of the hyper parameters. None for all statuses.
:param limit: Limit number of hyperparameters.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: list of hyper parameters.
"""
try:
if workload_id == 'null':
workload_id = None
if request_id == 'null':
request_id = None
if status == 'null':
status = None
if limit == 'null':
limit = None
if id == 'null':
id = None
contents = catalog.get_contents(
request_id=request_id,
workload_id=workload_id,
status=status,
relation_type=CollectionRelationType.Output)
if id:
new_contents = []
for content in contents:
if str(content['name']) == str(id):
new_contents.append(content)
contents = new_contents
if contents and limit and len(contents) > limit:
contents = contents[:limit]
hyperparameters = []
for content in contents:
point = content['path']
parameter, loss = json.loads(point)
param = {
'id': content['name'],
'parameters': parameter,
'loss': loss
}
hyperparameters.append(param)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data=hyperparameters)
from idds.common.utils import json_dumps # noqa F401
from idds.common.constants import ContentStatus, ContentRelationType # noqa F401
from idds.core.requests import get_requests # noqa F401
from idds.core.messages import retrieve_messages # noqa F401
from idds.core.transforms import get_transforms, release_inputs # noqa F401
from idds.core.workprogress import get_workprogresses # noqa F401
from idds.core.processings import get_processings # noqa F401
from idds.core import transforms as core_transforms # noqa F401
from idds.core.catalog import get_contents, update_contents # noqa F401
from idds.orm.contents import get_input_contents # noqa F401
request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376]
request_ids = [1689]
for request_id in request_ids:
contents = get_contents(request_id=request_id,
status=ContentStatus.Available)
ret_contents = {}
for content in contents:
if content[
'content_relation_type'] == ContentRelationType.Output: # InputDependency
if content['coll_id'] not in ret_contents:
ret_contents[content['coll_id']] = []
ret_contents[content['coll_id']].append(content)
for ret_content in ret_contents:
print("coll_id: %s, num_contents: %s" %
(ret_content, len(ret_contents[ret_content])))
# break
updated_contents = core_transforms.release_inputs_by_collection(
ret_contents)