def test_create_and_check_for_request_orm(self):
""" Request (ORM): Test the creation, query, and cancel of a Request """
properties = get_request_properties()
request_id = add_request(**properties)
request = get_request(request_id=request_id)
assert_equal(request_id, request['request_id'])
for key in properties:
if key in ['lifetime']:
continue
assert_equal(request[key], properties[key])
request_id1 = add_request(**properties)
delete_requests(request_id=request_id1)
update_request(request_id, parameters={'status': RequestStatus.Failed})
request = get_request(request_id=request_id)
assert_equal(request['status'], RequestStatus.Failed)
req = get_request(request_id=999999)
assert_equal(req, None)
delete_requests(request_id=request_id)
req = get_request(request_id=request_id)
assert_equal(req, None)
def add_request(scope, name, requester=None, request_type=None, transform_tag=None,
status=RequestStatus.New, locking=RequestLocking.Idle, priority=0,
lifetime=30, workload_id=None, request_metadata=None,
processing_metadata=None, session=None):
"""
Add a request.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param requestr: The requester, such as panda, user and so on.
:param request_type: The type of the request, such as ESS, DAOD.
:param transform_tag: Transform tag, such as ATLAS AMI tag.
:param status: The request status as integer.
:param locking: The request locking as integer.
:param priority: The priority as integer.
:param lifetime: The life time as umber of days.
:param workload_id: The external workload id.
:param request_metadata: The metadata as json.
:param processing_metadata: The metadata as json.
:returns: request id.
"""
kwargs = {'scope': scope, 'name': name, 'requester': requester, 'request_type': request_type,
'transform_tag': transform_tag, 'status': status, 'locking': locking,
'priority': priority, 'lifetime': lifetime, 'workload_id': workload_id,
'request_metadata': request_metadata, 'processing_metadata': processing_metadata,
'session': session}
if request_metadata and 'workload_id' in request_metadata:
kwargs['workload_id'] = int(request_metadata['workload_id'])
return orm_requests.add_request(**kwargs)
def test_get_collections_orm(self):
""" Collections (ORM): Test get collections """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_properties['request_id'] = request_id
origin_coll_id = add_collection(**coll_properties)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties['name'] + '_1'
origin_coll_id1 = add_collection(**coll_properties1)
origin_coll_id_list = [origin_coll_id, origin_coll_id1]
colls = get_collections_by_request_transform_id(transform_id=trans_id)
assert_equal(len(colls), 2)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
coll_ids = get_collection_ids_by_request_transform_id(
transform_id=trans_id)
assert_equal(len(coll_ids), 2)
for coll_id in coll_ids:
assert_in(coll_id, origin_coll_id_list)
colls = get_collections(scope=coll_properties['scope'],
name=coll_properties['name'],
request_id=request_id)
assert_equal(len(colls), 2)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
coll_id = get_collection_id_by_scope_name(
scope=coll_properties['scope'],
name=coll_properties['name'],
request_id=request_id)
assert_equal(coll_id, origin_coll_id)
def test_create_and_check_for_request_transform_orm(self):
""" Transform (ORM): Test to create and delete a Transform """
req_properties = get_request_properties()
req_properties['workload_id'] = int(time.time()) + random.randint(
1, 1000000)
trans_properties = get_transform_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
transform = get_transform(transform_id=trans_id)
for key in trans_properties:
if key in ['request_id']:
continue
assert_equal(transform[key], trans_properties[key])
trans_ids1 = get_transform_ids(request_id=request_id)
trans_ids2 = get_transform_ids(
workload_id=req_properties['workload_id'])
trans_ids3 = get_transform_ids(transform_id=trans_id)
trans_ids4 = get_transform_ids(
request_id=request_id, workload_id=req_properties['workload_id'])
trans_ids5 = get_transform_ids(
request_id=request_id,
workload_id=req_properties['workload_id'],
transform_id=trans_id)
trans_ids6 = get_transform_ids(request_id=request_id,
transform_id=trans_id)
trans_ids7 = get_transform_ids(
workload_id=req_properties['workload_id'], transform_id=trans_id)
assert_equal(trans_ids1, trans_ids2)
assert_equal(trans_ids2, trans_ids3)
assert_equal(trans_ids3, trans_ids4)
assert_equal(trans_ids4, trans_ids5)
assert_equal(trans_ids5, trans_ids6)
assert_equal(trans_ids6, trans_ids7)
delete_transform(trans_id)
delete_requests(request_id=request_id)
req = get_request(request_id=request_id)
assert_equal(req, None)
trans = get_transform(transform_id=trans_id)
assert_equal(trans, None)
def test_contents_orm(self):
""" Contents (ORM): Test contents """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
content_properties = get_content_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
content_properties['coll_id'] = coll_id
origin_content_id = add_content(**content_properties)
content_properties1 = copy.deepcopy(content_properties)
content_properties1['min_id'] = 101
content_properties1['max_id'] = 200
origin_content_id1 = add_content(**content_properties1)
content_properties2 = copy.deepcopy(content_properties)
content_properties2['min_id'] = 0
content_properties2['max_id'] = 200
origin_content_id2 = add_content(**content_properties2)
content_properties3 = copy.deepcopy(content_properties)
content_properties3['name'] = content_properties3['name'] + '_1'
origin_content_id3 = add_content(**content_properties3)
origin_content_ids = [
origin_content_id, origin_content_id1, origin_content_id2,
origin_content_id3
]
contents = get_contents(coll_id=coll_id)
assert_equal(len(contents), 4)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties['scope'],
name=content_properties['name'],
coll_id=coll_id)
assert_equal(len(contents), 3)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties3['scope'],
name=content_properties3['name'],
coll_id=coll_id)
assert_equal(len(contents), 1)
assert_equal(contents[0]['content_id'], origin_content_id3)
contents = get_match_contents(coll_id=content_properties['coll_id'],
scope=content_properties['scope'],
name=content_properties['name'],
min_id=content_properties['min_id'],
max_id=content_properties['max_id'])
assert_equal(len(contents), 2)
for content in contents:
assert_in(content['content_id'],
[origin_content_id, origin_content_id2])
to_updates = [{
'path': 'test_path1',
'status': ContentStatus.Processing,
'content_id': origin_content_id
}, {
'path': 'test_path2',
'status': ContentStatus.Processing,
'content_id': origin_content_id1
}]
update_contents(to_updates)
content = get_content(content_id=origin_content_id)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path1')
content = get_content(content_id=origin_content_id1)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path2')
def test_catalog_rest(self):
""" Catalog (Rest): Test catalog rest functions """
host = get_rest_host()
client = Client(host=host)
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_properties['relation_type'] = CollectionRelationType.Output
origin_coll_id = add_collection(**coll_properties)
req_trans_colls = client.get_collections(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
assert_equal(colls[0]['coll_id'], origin_coll_id)
req_trans_colls1 = client.get_collections(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = client.get_collections(scope=coll_properties['scope'], name=coll_properties['name'],
request_id=None, workload_id=None)
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
req_trans_coll_contents = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_scope_name = '%s:%s' % (coll_properties['scope'], coll_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_scope_name])
contents = coll_contents[coll_scope_name]['contents']
assert_equal(len(contents), 4)
output_content_ids = [output_content['content_id'] for output_content in contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents1 = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
assert_equal(is_same_req_trans_coll_contents(req_trans_coll_contents, req_trans_coll_contents1), True)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
client.register_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
contents=contents, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'])
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
def test_catalog_core(self):
""" Catalog (Core): Test catalog core functions """
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
coll_log_properties = copy.deepcopy(coll_properties)
coll_log_properties['relation_type'] = CollectionRelationType.Log
coll_log_properties['name'] = coll_log_properties['name'] + '_log'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
origin_coll_log_id = add_collection(**coll_log_properties)
origin_coll_id_list = [origin_coll_input_id, origin_coll_output_id, origin_coll_log_id]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_input_properties['name'], request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Input)
if key == 'name':
assert_equal(coll[key], coll_input_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_output_properties['name'], request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Output)
if key == 'name':
assert_equal(coll[key], coll_output_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_log_properties['name'], request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
assert_equal(coll[key], coll_log_properties[key])
trans_properties1 = copy.deepcopy(trans_properties)
origin_trans_id1 = add_transform(**trans_properties1)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties1['name'] + '_1'
coll_properties1['transform_id'] = origin_trans_id1
coll_input_properties1 = copy.deepcopy(coll_properties1)
coll_input_properties1['name'] = coll_input_properties1['name'] + '_input'
coll_output_properties1 = copy.deepcopy(coll_properties1)
coll_output_properties1['relation_type'] = CollectionRelationType.Output
coll_output_properties1['name'] = coll_output_properties1['name'] + '_output'
coll_log_properties1 = copy.deepcopy(coll_properties1)
coll_log_properties1['relation_type'] = CollectionRelationType.Log
coll_log_properties1['name'] = coll_log_properties1['name'] + '_log'
origin_coll_input_id1 = add_collection(**coll_input_properties1)
origin_coll_output_id1 = add_collection(**coll_output_properties1)
origin_coll_log_id1 = add_collection(**coll_log_properties1)
origin_coll_id_list1 = [origin_coll_input_id1, origin_coll_output_id1, origin_coll_log_id1]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
content_input_properties = get_content_properties()
content_input_properties['coll_id'] = origin_coll_input_id
origin_content_input_id_0_100 = add_content(returning_id=True, **content_input_properties)
content_input_properties_100_200 = copy.deepcopy(content_input_properties)
content_input_properties_100_200['min_id'] = 100
content_input_properties_100_200['max_id'] = 200
origin_content_input_id_100_200 = add_content(returning_id=True, **content_input_properties_100_200)
content_input_properties_name1 = copy.deepcopy(content_input_properties)
content_input_properties_name1['name'] = content_input_properties_name1['name'] + '_1'
content_input_properties_name1_id = add_content(returning_id=True, **content_input_properties_name1)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_output_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
with assert_raises(exceptions.WrongParameterException):
get_contents(coll_scope=None, coll_name=None, request_id=None, workload_id=None, relation_type=None)
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_input_scope_name, coll_output_scope_name])
input_contents = coll_contents[coll_input_scope_name]['contents']
output_contents = coll_contents[coll_output_scope_name]['contents']
assert_equal(len(input_contents), 3)
assert_equal(len(output_contents), 4)
input_content_ids = [input_content['content_id'] for input_content in input_contents]
assert_equal(input_content_ids, [origin_content_input_id_0_100, origin_content_input_id_100_200, content_input_properties_name1_id])
output_content_ids = [output_content['content_id'] for output_content in output_contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Input)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_input_scope_name])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_output_scope_name])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
register_output_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
contents=contents, request_id=origin_request_id, workload_id=None)
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
