def test(num_threads=1, total_contents=1, num_colls_per_session=1):
trans_properties = get_transform_prop()
trans_id = add_transform(**trans_properties)
coll_properties = get_collection_prop()
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
time_start = time.time()
threads = [
threading.Thread(target=test_thread,
args=(total_contents // num_threads,
num_colls_per_session, coll_id))
for i in range(num_threads)
]
[thread.start() for thread in threads]
while len(threads) > 0:
left_threads = []
for thread in threads:
if thread.is_alive():
left_threads.append(thread)
time.sleep(0.1)
threads = left_threads
time_end = time.time()
print(
"num_threads=%s, total_contents=%s, num_colls_per_session=%s, time used: %s"
% (num_threads, total_contents, num_colls_per_session,
time_end - time_start))
def update_workprogress(workprogress_id,
parameters,
new_transforms=None,
update_transforms=None,
session=None):
"""
update a workprogress.
:param workprogress_id: the workprogress id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no workprogress is founded.
:raises DatabaseException: If there is a database error.
"""
if new_transforms:
for tf in new_transforms:
orginal_work = tf['transform_metadata']['orginal_work']
del tf['transform_metadata']['orginal_work']
tf_id = orm_transforms.add_transform(**tf, session=session)
# work = tf['transform_metadata']['work']
orginal_work.set_work_id(tf_id, transforming=True)
orginal_work.set_status(WorkStatus.New)
if update_transforms:
for tr_id in update_transforms:
orm_transforms.update_transform(
transform_id=tr_id,
parameters=update_transforms[tr_id],
session=session)
return orm_workprogress.update_workprogress(
workprogress_id=workprogress_id,
parameters=parameters,
session=session)
def add_transform(request_id, workload_id, transform_type, transform_tag=None, priority=0,
status=TransformStatus.New, substatus=TransformStatus.New, locking=TransformLocking.Idle,
retries=0, expired_at=None, transform_metadata=None, workprogress_id=None, session=None):
"""
Add a transform.
:param request_id: The request id.
:param workload_id: The workload id.
:param transform_type: Transform type.
:param transform_tag: Transform tag.
:param priority: priority.
:param status: Transform status.
:param locking: Transform locking.
:param retries: The number of retries.
:param expired_at: The datetime when it expires.
:param transform_metadata: The metadata as json.
:raises DuplicatedObject: If a transform with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: transform id.
"""
transform_id = orm_transforms.add_transform(request_id=request_id, workload_id=workload_id,
transform_type=transform_type, transform_tag=transform_tag,
priority=priority, status=status, substatus=substatus,
locking=locking, retries=retries,
expired_at=expired_at, transform_metadata=transform_metadata,
workprogress_id=workprogress_id, session=session)
return transform_id
def update_request_with_transforms(request_id,
parameters,
transforms_to_add,
transforms_to_extend,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param transforms_to_add: list of transforms
:param transforms_to_extend: list of transforms
"""
for transform in transforms_to_add:
if 'collections' not in transform or len(
transform['collections']) == 0:
msg = "Transform must have collections, such as input collection, output collection and log collection"
raise exceptions.WrongParameterException(msg)
collections = transform['collections']
del transform['collections']
transform_id = orm_transforms.add_transform(**transform,
session=session)
input_coll_ids = []
log_coll_ids = []
for collection in collections['input_collections']:
collection['transform_id'] = transform_id
input_coll_id = orm_collections.add_collection(**collection,
session=session)
input_coll_ids.append(input_coll_id)
for collection in collections['log_collections']:
collection['transform_id'] = transform_id
log_coll_id = orm_collections.add_collection(**collection,
session=session)
log_coll_ids.append(log_coll_id)
for collection in collections['output_collections']:
collection['transform_id'] = transform_id
workload_id = transform['transform_metadata'][
'workload_id'] if 'workload_id' in transform[
'transform_metadata'] else None
collection['coll_metadata'] = {
'transform_id': transform_id,
'workload_id': workload_id,
'input_collections': input_coll_ids,
'log_collections': log_coll_ids
}
orm_collections.add_collection(**collection, session=session)
for transform in transforms_to_extend:
transform_id = transform['transform_id']
del transform['transform_id']
# orm_transforms.add_req2transform(request_id, transform_id, session=session)
orm_transforms.update_transform(transform_id,
parameters=transform,
session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def test_get_collections_orm(self):
""" Collections (ORM): Test get collections """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_properties['request_id'] = request_id
origin_coll_id = add_collection(**coll_properties)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties['name'] + '_1'
origin_coll_id1 = add_collection(**coll_properties1)
origin_coll_id_list = [origin_coll_id, origin_coll_id1]
colls = get_collections_by_request_transform_id(transform_id=trans_id)
assert_equal(len(colls), 2)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
coll_ids = get_collection_ids_by_request_transform_id(
transform_id=trans_id)
assert_equal(len(coll_ids), 2)
for coll_id in coll_ids:
assert_in(coll_id, origin_coll_id_list)
colls = get_collections(scope=coll_properties['scope'],
name=coll_properties['name'],
request_id=request_id)
assert_equal(len(colls), 2)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
coll_id = get_collection_id_by_scope_name(
scope=coll_properties['scope'],
name=coll_properties['name'],
request_id=request_id)
assert_equal(coll_id, origin_coll_id)
def test_create_and_check_for_transform_collection_content_orm(self):
""" Transform/Collection/Content (ORM): Test the creation, query, and cancel of a Transform/Collection/Content """
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
content_properties = get_content_properties()
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
coll_id_1 = get_collection_id(
transform_id=trans_id,
relation_type=coll_properties['relation_type'])
assert_equal(coll_id, coll_id_1)
coll = get_collection(coll_id=coll_id)
for key in coll_properties:
assert_equal(coll[key], coll_properties[key])
update_collection(coll_id, {'status': CollectionStatus.Closed})
coll = get_collection(coll_id=coll_id)
assert_equal(coll['status'], CollectionStatus.Closed)
content_properties['coll_id'] = coll_id
content_id = add_content(**content_properties)
content_1 = get_content(coll_id=coll_id,
scope=content_properties['scope'],
name=content_properties['name'],
content_type=ContentType.File)
content_id_1 = content_1['content_id']
assert_equal(content_id, content_id_1)
content = get_content(content_id=content_id)
update_content(content_id=content_id,
parameters={'status': ContentStatus.Lost})
content = get_content(content_id=content_id)
assert_equal(content['status'], ContentStatus.Lost)
delete_content(content_id=content_id)
c = get_content(content_id=content_id)
assert_equal(c, None)
delete_collection(coll_id=coll_id)
coll = get_collection(coll_id=coll_id)
assert_equal(coll, None)
delete_transform(transform_id=trans_id)
t = get_transform(transform_id=trans_id)
assert_equal(t, None)
def test_create_and_check_for_transform_orm(self):
""" Transform (ORM): Test the creation, query, and cancel of a Transform """
trans_properties = get_transform_properties()
trans_id = add_transform(**trans_properties)
transform = get_transform(transform_id=trans_id)
for key in trans_properties:
assert_equal(transform[key], trans_properties[key])
update_transform(trans_id, {'status': TransformStatus.Failed})
transform = get_transform(transform_id=trans_id)
assert_equal(transform['status'], TransformStatus.Failed)
delete_transform(trans_id)
t = get_transform(transform_id=trans_id)
assert_equal(t, None)
def test_create_and_check_for_request_transform_orm(self):
""" Transform (ORM): Test to create and delete a Transform """
req_properties = get_request_properties()
req_properties['workload_id'] = int(time.time()) + random.randint(
1, 1000000)
trans_properties = get_transform_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
transform = get_transform(transform_id=trans_id)
for key in trans_properties:
if key in ['request_id']:
continue
assert_equal(transform[key], trans_properties[key])
trans_ids1 = get_transform_ids(request_id=request_id)
trans_ids2 = get_transform_ids(
workload_id=req_properties['workload_id'])
trans_ids3 = get_transform_ids(transform_id=trans_id)
trans_ids4 = get_transform_ids(
request_id=request_id, workload_id=req_properties['workload_id'])
trans_ids5 = get_transform_ids(
request_id=request_id,
workload_id=req_properties['workload_id'],
transform_id=trans_id)
trans_ids6 = get_transform_ids(request_id=request_id,
transform_id=trans_id)
trans_ids7 = get_transform_ids(
workload_id=req_properties['workload_id'], transform_id=trans_id)
assert_equal(trans_ids1, trans_ids2)
assert_equal(trans_ids2, trans_ids3)
assert_equal(trans_ids3, trans_ids4)
assert_equal(trans_ids4, trans_ids5)
assert_equal(trans_ids5, trans_ids6)
assert_equal(trans_ids6, trans_ids7)
delete_transform(trans_id)
delete_requests(request_id=request_id)
req = get_request(request_id=request_id)
assert_equal(req, None)
trans = get_transform(transform_id=trans_id)
assert_equal(trans, None)
def add_transform(transform_type,
transform_tag=None,
priority=0,
status=TransformStatus.New,
locking=TransformLocking.Idle,
retries=0,
expired_at=None,
transform_metadata=None,
request_id=None,
collections=None,
session=None):
"""
Add a transform.
:param transform_type: Transform type.
:param transform_tag: Transform tag.
:param priority: priority.
:param status: Transform status.
:param locking: Transform locking.
:param retries: The number of retries.
:param expired_at: The datetime when it expires.
:param transform_metadata: The metadata as json.
:raises DuplicatedObject: If a transform with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: transform id.
"""
if collections is None or len(collections) == 0:
msg = "Transform must have collections, such as input collection, output collection and log collection"
raise exceptions.WrongParameterException(msg)
transform_id = orm_transforms.add_transform(
transform_type=transform_type,
transform_tag=transform_tag,
priority=priority,
status=status,
locking=locking,
retries=retries,
expired_at=expired_at,
transform_metadata=transform_metadata,
request_id=request_id,
session=session)
for collection in collections:
collection['transform_id'] = transform_id
orm_collections.add_collection(**collection, session=session)
def test_create_and_check_for_transform_processing_orm(self):
""" Transform/Processing (ORM): Test the creation, query, and cancel of a Transform/Processing """
trans_properties = get_transform_properties()
proc_properties = get_processing_properties()
trans_id = add_transform(**trans_properties)
proc_properties['transform_id'] = trans_id
processing_id = add_processing(**proc_properties)
processing = get_processing(processing_id=processing_id)
for key in proc_properties:
assert_equal(processing[key], proc_properties[key])
update_processing(processing_id, {'status': ProcessingStatus.Failed})
processing = get_processing(processing_id=processing_id)
assert_equal(processing['status'], ProcessingStatus.Failed)
delete_processing(processing_id)
with assert_raises(exceptions.NoObject):
get_processing(processing_id=processing_id)
delete_transform(trans_id)
def test_contents_orm(self):
""" Contents (ORM): Test contents """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
content_properties = get_content_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
content_properties['coll_id'] = coll_id
origin_content_id = add_content(**content_properties)
content_properties1 = copy.deepcopy(content_properties)
content_properties1['min_id'] = 101
content_properties1['max_id'] = 200
origin_content_id1 = add_content(**content_properties1)
content_properties2 = copy.deepcopy(content_properties)
content_properties2['min_id'] = 0
content_properties2['max_id'] = 200
origin_content_id2 = add_content(**content_properties2)
content_properties3 = copy.deepcopy(content_properties)
content_properties3['name'] = content_properties3['name'] + '_1'
origin_content_id3 = add_content(**content_properties3)
origin_content_ids = [
origin_content_id, origin_content_id1, origin_content_id2,
origin_content_id3
]
contents = get_contents(coll_id=coll_id)
assert_equal(len(contents), 4)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties['scope'],
name=content_properties['name'],
coll_id=coll_id)
assert_equal(len(contents), 3)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties3['scope'],
name=content_properties3['name'],
coll_id=coll_id)
assert_equal(len(contents), 1)
assert_equal(contents[0]['content_id'], origin_content_id3)
contents = get_match_contents(coll_id=content_properties['coll_id'],
scope=content_properties['scope'],
name=content_properties['name'],
min_id=content_properties['min_id'],
max_id=content_properties['max_id'])
assert_equal(len(contents), 2)
for content in contents:
assert_in(content['content_id'],
[origin_content_id, origin_content_id2])
to_updates = [{
'path': 'test_path1',
'status': ContentStatus.Processing,
'content_id': origin_content_id
}, {
'path': 'test_path2',
'status': ContentStatus.Processing,
'content_id': origin_content_id1
}]
update_contents(to_updates)
content = get_content(content_id=origin_content_id)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path1')
content = get_content(content_id=origin_content_id1)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path2')
def update_request_with_transforms(request_id,
parameters,
new_transforms=None,
update_transforms=None,
new_messages=None,
update_messages=None,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param new_transforms: list of transforms
:param update_transforms: list of transforms
"""
if new_transforms:
for tf in new_transforms:
# tf_id = orm_transforms.add_transform(**tf, session=session)
# original_work = tf['transform_metadata']['original_work']
# del tf['transform_metadata']['original_work']
workflow = tf['transform_metadata']['workflow']
del tf['transform_metadata']['workflow']
work = tf['transform_metadata']['work']
tf_copy = copy.deepcopy(tf)
tf_id = orm_transforms.add_transform(**tf_copy, session=session)
tf['transform_id'] = tf_id
# work = tf['transform_metadata']['work']
# original_work.set_work_id(tf_id, transforming=True)
# original_work.set_status(WorkStatus.New)
work.set_work_id(tf_id, transforming=True)
work.set_status(WorkStatus.New)
workflow.refresh_works()
collections = generate_collections(tf)
for coll in collections:
collection = coll['collection']
del coll['collection']
coll['transform_id'] = tf_id
coll_id = orm_collections.add_collection(**coll,
session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
# update transform to record the coll_id
work.refresh_work()
orm_transforms.update_transform(
transform_id=tf_id,
parameters={'transform_metadata': tf['transform_metadata']},
session=session)
if update_transforms:
for tr_id in update_transforms:
orm_transforms.update_transform(
transform_id=tr_id,
parameters=update_transforms[tr_id],
session=session)
if new_messages:
orm_messages.add_messages(new_messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def test_catalog_rest(self):
""" Catalog (Rest): Test catalog rest functions """
host = get_rest_host()
client = Client(host=host)
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_properties['relation_type'] = CollectionRelationType.Output
origin_coll_id = add_collection(**coll_properties)
req_trans_colls = client.get_collections(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
assert_equal(colls[0]['coll_id'], origin_coll_id)
req_trans_colls1 = client.get_collections(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = client.get_collections(scope=coll_properties['scope'], name=coll_properties['name'],
request_id=None, workload_id=None)
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
req_trans_coll_contents = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_scope_name = '%s:%s' % (coll_properties['scope'], coll_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_scope_name])
contents = coll_contents[coll_scope_name]['contents']
assert_equal(len(contents), 4)
output_content_ids = [output_content['content_id'] for output_content in contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents1 = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
assert_equal(is_same_req_trans_coll_contents(req_trans_coll_contents, req_trans_coll_contents1), True)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
client.register_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
contents=contents, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'])
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
def test_catalog_core(self):
""" Catalog (Core): Test catalog core functions """
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
coll_log_properties = copy.deepcopy(coll_properties)
coll_log_properties['relation_type'] = CollectionRelationType.Log
coll_log_properties['name'] = coll_log_properties['name'] + '_log'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
origin_coll_log_id = add_collection(**coll_log_properties)
origin_coll_id_list = [origin_coll_input_id, origin_coll_output_id, origin_coll_log_id]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_input_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_input_properties['name'], request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Input)
if key == 'name':
assert_equal(coll[key], coll_input_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_output_properties['name'], request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Output)
if key == 'name':
assert_equal(coll[key], coll_output_properties[key])
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_log_properties['name'], request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
coll = colls[0]
for key in coll_properties:
if key == 'relation_type':
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
assert_equal(coll[key], coll_log_properties[key])
trans_properties1 = copy.deepcopy(trans_properties)
origin_trans_id1 = add_transform(**trans_properties1)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties1['name'] + '_1'
coll_properties1['transform_id'] = origin_trans_id1
coll_input_properties1 = copy.deepcopy(coll_properties1)
coll_input_properties1['name'] = coll_input_properties1['name'] + '_input'
coll_output_properties1 = copy.deepcopy(coll_properties1)
coll_output_properties1['relation_type'] = CollectionRelationType.Output
coll_output_properties1['name'] = coll_output_properties1['name'] + '_output'
coll_log_properties1 = copy.deepcopy(coll_properties1)
coll_log_properties1['relation_type'] = CollectionRelationType.Log
coll_log_properties1['name'] = coll_log_properties1['name'] + '_log'
origin_coll_input_id1 = add_collection(**coll_input_properties1)
origin_coll_output_id1 = add_collection(**coll_output_properties1)
origin_coll_log_id1 = add_collection(**coll_log_properties1)
origin_coll_id_list1 = [origin_coll_input_id1, origin_coll_output_id1, origin_coll_log_id1]
req_trans_colls = get_collections_by_request(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_trans_colls1 = get_collections_by_request(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = get_collections_by_request(request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'], request_id=None, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 2)
for trans_id in req_trans_colls[req_id]:
if trans_id == origin_trans_id:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id:
assert_equal(coll[key], coll_input_properties[key])
elif coll['coll_id'] == origin_coll_output_id:
assert_equal(coll[key], coll_output_properties[key])
else:
assert_equal(coll[key], coll_log_properties[key])
if trans_id == origin_trans_id1:
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 3)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list1)
for key in coll_properties:
if key == 'relation_type':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], CollectionRelationType.Input)
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], CollectionRelationType.Output)
else:
assert_equal(coll['coll_id'], origin_coll_log_id1)
assert_equal(coll[key], CollectionRelationType.Log)
if key == 'name':
if coll['coll_id'] == origin_coll_input_id1:
assert_equal(coll[key], coll_input_properties1[key])
elif coll['coll_id'] == origin_coll_output_id1:
assert_equal(coll[key], coll_output_properties1[key])
else:
assert_equal(coll[key], coll_log_properties1[key])
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_input_properties = copy.deepcopy(coll_properties)
coll_input_properties['name'] = coll_input_properties['name'] + '_input'
coll_output_properties = copy.deepcopy(coll_properties)
coll_output_properties['relation_type'] = CollectionRelationType.Output
coll_output_properties['name'] = coll_output_properties['name'] + '_output'
origin_coll_input_id = add_collection(**coll_input_properties)
origin_coll_output_id = add_collection(**coll_output_properties)
content_input_properties = get_content_properties()
content_input_properties['coll_id'] = origin_coll_input_id
origin_content_input_id_0_100 = add_content(returning_id=True, **content_input_properties)
content_input_properties_100_200 = copy.deepcopy(content_input_properties)
content_input_properties_100_200['min_id'] = 100
content_input_properties_100_200['max_id'] = 200
origin_content_input_id_100_200 = add_content(returning_id=True, **content_input_properties_100_200)
content_input_properties_name1 = copy.deepcopy(content_input_properties)
content_input_properties_name1['name'] = content_input_properties_name1['name'] + '_1'
content_input_properties_name1_id = add_content(returning_id=True, **content_input_properties_name1)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_output_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
with assert_raises(exceptions.WrongParameterException):
get_contents(coll_scope=None, coll_name=None, request_id=None, workload_id=None, relation_type=None)
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_input_scope_name, coll_output_scope_name])
input_contents = coll_contents[coll_input_scope_name]['contents']
output_contents = coll_contents[coll_output_scope_name]['contents']
assert_equal(len(input_contents), 3)
assert_equal(len(output_contents), 4)
input_content_ids = [input_content['content_id'] for input_content in input_contents]
assert_equal(input_content_ids, [origin_content_input_id_0_100, origin_content_input_id_100_200, content_input_properties_name1_id])
output_content_ids = [output_content['content_id'] for output_content in output_contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Input)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_input_scope_name])
req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
coll_scope_names = list(coll_contents.keys())
assert_equal(coll_scope_names, [coll_output_scope_name])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
register_output_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
contents=contents, request_id=origin_request_id, workload_id=None)
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
