def test_catalog_rest(self):
""" Catalog (Rest): Test catalog rest functions """
host = get_rest_host()
client = Client(host=host)
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_properties['relation_type'] = CollectionRelationType.Output
origin_coll_id = add_collection(**coll_properties)
req_trans_colls = client.get_collections(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
assert_equal(colls[0]['coll_id'], origin_coll_id)
req_trans_colls1 = client.get_collections(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = client.get_collections(scope=coll_properties['scope'], name=coll_properties['name'],
request_id=None, workload_id=None)
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
req_trans_coll_contents = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_scope_name = '%s:%s' % (coll_properties['scope'], coll_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_scope_name])
contents = coll_contents[coll_scope_name]['contents']
assert_equal(len(contents), 4)
output_content_ids = [output_content['content_id'] for output_content in contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents1 = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
assert_equal(is_same_req_trans_coll_contents(req_trans_coll_contents, req_trans_coll_contents1), True)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
client.register_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
contents=contents, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'])
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
# props['request_metadata']['result_parser'] = 'default'
scope = 'data16_13TeV'
name = 'data16_13TeV.00298862.physics_Main.daq.RAW.idds.stagein'
request_id = 12
workload_id = 1601235010
relation_type = CollectionRelationType.Output # Input, Log
status = ContentStatus.Available # New, Processing, Available, ...
colls = client.get_collections(scope=scope,
name=name,
request_id=request_id,
workload_id=workload_id,
relation_type=relation_type)
print(colls)
# example outputs
# [{'relation_type': <CollectionRelationType.Output: 1>, 'next_poll_at': 'Sun, 27 Sep 2020 19:30:16 UTC', 'storage_id': None, 'scope': 'data16_13TeV', 'accessed_at': 'Sun, 27 Sep 2020 21:31:40 UTC', 'new_files': 0, 'name': 'data16_13TeV.00298862.physics_Main.daq.RAW.idds.stagein', 'expired_at': 'Tue, 27 Oct 2020 19:30:10 UTC', 'processed_files': 0, 'bytes': 0, 'coll_metadata': {'internal_id': 'db1bb0dc-00f7-11eb-a5d2-fa163eb98fd2'}, 'request_id': 12, 'processing_files': 0, 'status': <CollectionStatus.Open: 3>, 'coll_id': 16, 'processing_id': None, 'substatus': None, 'workload_id': 1601235010, 'retries': 0, 'locking': <CollectionLocking.Idle: 0>, 'coll_type': <CollectionType.Dataset: 1>, 'created_at': 'Sun, 27 Sep 2020 19:30:16 UTC', 'total_files': 0, 'transform_id': 9, 'updated_at': 'Sun, 27 Sep 2020 21:31:40 UTC'}] # noqa: E501
contents = client.get_contents(coll_scope=scope,
coll_name=name,
request_id=request_id,
workload_id=workload_id,
relation_type=relation_type,
status=status)
for content in contents:
# print(content)
pass
# example outputs
# {'substatus': <ContentStatus.Available: 2>, 'transform_id': 9, 'path': None, 'locking': <ContentLocking.Idle: 0>, 'map_id': 8895, 'created_at': 'Sun, 27 Sep 2020 19:30:40 UTC', 'bytes': 534484132, 'scope': 'data16_13TeV', 'updated_at': 'Sun, 27 Sep 2020 19:44:55 UTC', 'md5': None, 'name': 'data16_13TeV.00298862.physics_Main.daq.RAW._lb0538._SFO-5._0003.data', 'accessed_at': 'Sun, 27 Sep 2020 19:44:56 UTC', 'adler32': 'e8543989', 'content_id': 2361650, 'min_id': 0, 'expired_at': 'Tue, 27 Oct 2020 19:30:26 UTC', 'coll_id': 16, 'processing_id': None, 'max_id': 579, 'content_metadata': {'events': 579}, 'storage_id': None, 'request_id': 12, 'content_type': <ContentType.File: 0>, 'workload_id': 1601235010, 'retries': 0, 'status': <ContentStatus.Available: 2>} # noqa: E501