def get_collections(scope=None,
name=None,
request_id=None,
workload_id=None,
transform_id=None,
relation_type=None,
to_json=False,
session=None):
"""
Get collections by scope, name, request_id and workload id.
:param scope: scope of the collection.
:param name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param transform_id: The transform id related to this collection.
:param relation_type: The relation between this collection and its transform,
such as Input, Output, Log and so on.
:param to_json: return json format.
:param session: The database session in use.
:returns: dict of collections
"""
if request_id or workload_id or transform_id:
transform_ids = orm_transforms.get_transform_ids(
request_id=request_id,
workload_id=workload_id,
transform_id=transform_id,
session=session)
if transform_ids:
collections = orm_collections.get_collections(
scope=scope,
name=name,
transform_id=transform_ids,
relation_type=relation_type,
to_json=to_json,
session=session)
else:
collections = []
else:
collections = orm_collections.get_collections(
scope=scope,
name=name,
to_json=to_json,
relation_type=relation_type,
session=session)
rets = {}
for collection in collections:
if request_id not in rets:
rets[request_id] = {}
transform_id = collection['transform_id']
if transform_id not in rets[request_id]:
rets[request_id][transform_id] = []
rets[request_id][transform_id].append(collection)
return rets
def test_get_collections_orm(self):
""" Collections (ORM): Test get collections """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
origin_coll_id = add_collection(**coll_properties)
coll_properties1 = copy.deepcopy(coll_properties)
coll_properties1['name'] = coll_properties['name'] + '_1'
origin_coll_id1 = add_collection(**coll_properties1)
origin_coll_id_list = [origin_coll_id, origin_coll_id1]
colls = get_collections(transform_id=trans_id)
assert_equal(len(colls), 2)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
coll_ids = get_collection_ids_by_transform_id(transform_id=trans_id)
assert_equal(len(coll_ids), 2)
for coll_id in coll_ids:
assert_in(coll_id, origin_coll_id_list)
colls = get_collections(scope=coll_properties['scope'],
name=coll_properties['name'],
transform_id=[trans_id])
assert_equal(len(colls), 1)
for coll in colls:
assert_in(coll['coll_id'], origin_coll_id_list)
for key in coll_properties:
if key == 'name':
continue
assert_equal(coll[key], coll_properties[key])
def get_match_contents(coll_scope, coll_name, scope, name, min_id=None, max_id=None,
request_id=None, workload_id=None, relation_type=None,
only_return_best_match=False, to_json=False, session=None):
"""
Get matched contents with collection scope, collection name, scope, name, min_id, max_id,
request id, workload id and only_return_best_match.
:param coll_scope: scope of the collection.
:param coll_name: name the the collection.
:param scope: scope of the content.
:param name: name of the content.
:param min_id: min_id of the content.
:param max_id: max_id of the content.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param only_return_best_match: only return best matched content if it's true.
:param session: The database session in use.
:returns: list of contents
"""
transform_ids = orm_transforms.get_transform_ids(request_id=request_id,
workload_id=workload_id,
session=session)
if transform_ids:
collections = orm_collections.get_collections(scope=coll_scope, name=coll_name, transform_id=transform_ids,
relation_type=relation_type, session=session)
else:
collections = []
coll_def = "request_id=%s, workload_id=%s, coll_scope=%s" % (request_id, workload_id, coll_scope)
coll_def += ", coll_name=%s, relation_type: %s" % (coll_name, relation_type)
if len(collections) != 1:
msg = "There should be only one collection matched. However there are %s collections" % len(collections)
msg += coll_def
raise exceptions.WrongParameterException(msg)
coll_id = collections[0]['coll_id']
contents = orm_contents.get_match_contents(coll_id=coll_id, scope=scope, name=name, min_id=min_id, max_id=max_id,
to_json=to_json, session=session)
if not only_return_best_match:
return contents
if len(contents) == 1:
return contents
content = None
for row in contents:
if (not content) or (content['max_id'] - content['min_id'] > row['max_id'] - row['min_id']):
content = row
return [content]
def register_output_contents(coll_scope, coll_name, contents, request_id=None, workload_id=None,
relation_type=CollectionRelationType.Output, session=None):
"""
register contents with collection scope, collection name, request id, workload id and contents.
:param coll_scope: scope of the collection.
:param coll_name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param contents: list of contents [{'scope': <scope>, 'name': <name>, 'min_id': min_id, 'max_id': max_id,
'status': <status>, 'path': <path>}].
:param session: The database session in use.
"""
transform_ids = orm_transforms.get_transform_ids(request_id=request_id,
workload_id=workload_id,
session=session)
if transform_ids:
collections = orm_collections.get_collections(scope=coll_scope, name=coll_name, transform_id=transform_ids,
relation_type=relation_type, session=session)
else:
collections = []
coll_def = "request_id=%s, workload_id=%s, coll_scope=%s" % (request_id, workload_id, coll_scope)
coll_def += ", coll_name=%s, relation_type: %s" % (coll_name, relation_type)
if len(collections) != 1:
msg = "There should be only one collection matched. However there are %s collections" % len(collections)
msg += coll_def
raise exceptions.WrongParameterException(msg)
coll_id = collections[0]['coll_id']
keys = ['scope', 'name', 'min_id', 'max_id']
for content in contents:
ex_content = orm_contents.get_content(coll_id=coll_id, scope=content['scope'],
name=content['name'], min_id=content['min_id'],
max_id=content['max_id'], session=session)
content_def = "scope: %s, name: %s, min_id: %s, max_id: %s" % (content['scope'],
content['name'],
content['min_id'],
content['max_id'])
if not ex_content:
msg = "No matched content in collection(%s) with content(%s)" % (coll_def, content_def)
raise exceptions.WrongParameterException(msg)
for key in keys:
if key in content:
del content[key]
content['content_id'] = ex_content['content_id']
orm_contents.update_contents(contents, session=session)
def get_work_name_to_coll_map(request_id):
tfs = orm_transforms.get_transforms(request_id=request_id)
colls = orm_collections.get_collections(request_id=request_id)
work_name_to_coll_map = {}
for tf in tfs:
if ('transform_metadata' in tf and tf['transform_metadata']
and 'work_name' in tf['transform_metadata'] and tf['transform_metadata']['work_name']): # noqa: W503
work_name = tf['transform_metadata']['work_name']
transform_id = tf['transform_id']
if work_name not in work_name_to_coll_map:
work_name_to_coll_map[work_name] = {'inputs': [], 'outputs': []}
for coll in colls:
if coll['transform_id'] == transform_id:
if coll['relation_type'] == CollectionRelationType.Input:
work_name_to_coll_map[work_name]['inputs'].append({'coll_id': coll['coll_id'], 'scope': coll['scope'], 'name': coll['name']})
elif coll['relation_type'] == CollectionRelationType.Output:
work_name_to_coll_map[work_name]['outputs'].append({'coll_id': coll['coll_id'], 'scope': coll['scope'], 'name': coll['name']})
return work_name_to_coll_map
def get_collections(scope,
name,
request_id=None,
workload_id=None,
session=None):
"""
Get collections by scope, name, request_id and workload id.
:param scope: scope of the collection.
:param name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param session: The database session in use.
:returns: dict of collections
"""
if scope is None and name is None:
return get_collections_by_request(request_id=request_id,
workload_id=workload_id,
session=session)
if request_id is None and workload_id is not None:
request_id = orm_requests.get_request_id(request_id,
workload_id,
session=session)
transform_ids = None
if request_id:
transform_ids = orm_transforms.get_transform_ids(request_id,
session=session)
collections = orm_collections.get_collections(scope=scope,
name=name,
transform_ids=transform_ids,
session=session)
rets = {}
for collection in collections:
if request_id not in rets:
rets[request_id] = {}
transform_id = collection['transform_id']
if transform_id not in rets[request_id]:
rets[request_id][transform_id] = []
rets[request_id][transform_id].append(collection)
return rets
def get_collections(scope=None, name=None, request_id=None, workload_id=None, transform_id=None,
relation_type=None, to_json=False, session=None):
"""
Get collections by scope, name, request_id and workload id.
:param scope: scope of the collection.
:param name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param transform_id: The transform id related to this collection.
:param relation_type: The relation between this collection and its transform,
such as Input, Output, Log and so on.
:param to_json: return json format.
:param session: The database session in use.
:returns: dict of collections
"""
collections = orm_collections.get_collections(scope=scope, name=name, request_id=request_id,
workload_id=workload_id, transform_id=transform_id,
to_json=to_json,
relation_type=relation_type, session=session)
return collections