def update_processing_with_collection_contents(updated_processing,
new_processing=None,
updated_collection=None,
updated_files=None,
new_files=None,
coll_msg_content=None,
file_msg_content=None,
transform_updates=None,
message_bulk_size=1000,
session=None):
"""
Update processing with collection, contents, file messages and collection messages.
:param updated_processing: dict with processing id and parameters.
:param updated_collection: dict with collection id and parameters.
:param updated_files: list of content files.
:param coll_msg_content: message with collection info.
:param file_msg_content: message with files info.
"""
if updated_files:
orm_contents.update_contents(updated_files, session=session)
if new_files:
orm_contents.add_contents(contents=new_files, session=session)
if file_msg_content:
if not type(file_msg_content) in [list, tuple]:
file_msg_content = [file_msg_content]
for file_msg_con in file_msg_content:
orm_messages.add_message(msg_type=file_msg_con['msg_type'],
status=file_msg_con['status'],
source=file_msg_con['source'],
transform_id=file_msg_con['transform_id'],
num_contents=file_msg_con['num_contents'],
msg_content=file_msg_con['msg_content'],
bulk_size=message_bulk_size,
session=session)
if updated_collection:
orm_collections.update_collection(
coll_id=updated_collection['coll_id'],
parameters=updated_collection['parameters'],
session=session)
if coll_msg_content:
orm_messages.add_message(msg_type=coll_msg_content['msg_type'],
status=coll_msg_content['status'],
source=coll_msg_content['source'],
transform_id=coll_msg_content['transform_id'],
num_contents=coll_msg_content['num_contents'],
msg_content=coll_msg_content['msg_content'],
session=session)
if updated_processing:
orm_processings.update_processing(
processing_id=updated_processing['processing_id'],
parameters=updated_processing['parameters'],
session=session)
if new_processing:
orm_processings.add_processing(**new_processing, session=session)
if transform_updates:
orm_transforms.update_transform(
transform_id=transform_updates['transform_id'],
parameters=transform_updates['parameters'],
session=session)
def register_output_contents(coll_scope, coll_name, contents, request_id=None, workload_id=None,
relation_type=CollectionRelationType.Output, session=None):
"""
register contents with collection scope, collection name, request id, workload id and contents.
:param coll_scope: scope of the collection.
:param coll_name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param contents: list of contents [{'scope': <scope>, 'name': <name>, 'min_id': min_id, 'max_id': max_id,
'status': <status>, 'path': <path>}].
:param session: The database session in use.
"""
transform_ids = orm_transforms.get_transform_ids(request_id=request_id,
workload_id=workload_id,
session=session)
if transform_ids:
collections = orm_collections.get_collections(scope=coll_scope, name=coll_name, transform_id=transform_ids,
relation_type=relation_type, session=session)
else:
collections = []
coll_def = "request_id=%s, workload_id=%s, coll_scope=%s" % (request_id, workload_id, coll_scope)
coll_def += ", coll_name=%s, relation_type: %s" % (coll_name, relation_type)
if len(collections) != 1:
msg = "There should be only one collection matched. However there are %s collections" % len(collections)
msg += coll_def
raise exceptions.WrongParameterException(msg)
coll_id = collections[0]['coll_id']
keys = ['scope', 'name', 'min_id', 'max_id']
for content in contents:
ex_content = orm_contents.get_content(coll_id=coll_id, scope=content['scope'],
name=content['name'], min_id=content['min_id'],
max_id=content['max_id'], session=session)
content_def = "scope: %s, name: %s, min_id: %s, max_id: %s" % (content['scope'],
content['name'],
content['min_id'],
content['max_id'])
if not ex_content:
msg = "No matched content in collection(%s) with content(%s)" % (coll_def, content_def)
raise exceptions.WrongParameterException(msg)
for key in keys:
if key in content:
del content[key]
content['content_id'] = ex_content['content_id']
orm_contents.update_contents(contents, session=session)
def update_processing_contents(processing_update,
content_updates,
session=None):
"""
Update processing with contents.
:param processing_update: dict with processing id and parameters.
:param content_updates: list of content files.
"""
if content_updates:
orm_contents.update_contents(content_updates, session=session)
if processing_update:
orm_processings.update_processing(
processing_id=processing_update['processing_id'],
parameters=processing_update['parameters'],
session=session)
def update_contents(parameters, session=None):
"""
updatecontents.
:param parameters: list of dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
return orm_contents.update_contents(parameters, session=session)
def register_output_contents(coll_scope,
coll_name,
contents,
request_id=None,
workload_id=None,
relation_type=CollectionRelationType.Output,
session=None):
"""
register contents with collection scope, collection name, request id, workload id and contents.
:param coll_scope: scope of the collection.
:param coll_name: name the the collection.
:param request_id: the request id.
:param workload_id: The workload_id of the request.
:param contents: list of contents [{'scope': <scope>, 'name': <name>, 'min_id': min_id, 'max_id': max_id,
'status': <status>, 'path': <path>}].
:param session: The database session in use.
"""
if (request_id is None and
workload_id is None) or coll_scope is None or coll_name is None:
msg = "Only one of (request_id, workload_id) can be None. All other parameters should not be None: "
msg += "request_id=%s, workload_id=%s, coll_scope=%s, coll_name=%s" % (
request_id, workload_id, coll_scope, coll_name)
raise exceptions.WrongParameterException(msg)
if request_id is None and workload_id is not None:
request_id = orm_requests.get_request_id(request_id,
workload_id,
session=session)
coll_id = orm_collections.get_collection_id_by_scope_name(coll_scope,
coll_name,
request_id,
relation_type,
session=session)
parameters = []
for content in contents:
if 'status' not in content or content['status'] is None:
raise exceptions.WrongParameterException(
"Content status is required and should not be None: %s" %
content)
if content['status'] in [
ContentStatus.Available, ContentStatus.Available.value
]:
content_keys = [
'scope', 'name', 'min_id', 'max_id', 'status', 'path'
]
else:
content_keys = ['scope', 'name', 'min_id', 'max_id', 'status']
parameter = {}
for key in content_keys:
if content[key] is None:
raise exceptions.WrongParameterException(
"Content %s should not be None" % key)
parameter[key] = content[key]
if isinstance(parameter['status'], ContentStatus):
parameter['status'] = parameter['status'].value
parameter['coll_id'] = coll_id
parameters.append(parameter)
orm_contents.update_contents(parameters, session=session)
def add_transform_outputs(transform,
input_collection,
output_collection,
input_contents,
output_contents,
processing,
to_cancel_processing=None,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collection: The input collection.
:param output_collection: The output collection.
:param input_contents: The input contents.
:param output_contents: The corresponding output contents.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if input_contents:
update_input_contents = []
for input_content in input_contents:
update_input_content = {
'content_id': input_content['content_id'],
'status': ContentStatus.Mapped,
'path': None
}
update_input_contents.append(update_input_content)
if update_input_contents:
orm_contents.update_contents(update_input_contents,
with_content_id=True,
session=session)
if output_collection:
# TODO, the status and new_files should be updated
orm_collections.update_collection(
output_collection['coll_id'],
{'status': CollectionStatus.Processing},
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id,
parameters=to_cancel_params,
session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing,
session=session)
if transform:
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {
'processing_id': processing_id
}
else:
transform['transform_metadata'][
'processing_id'] = processing_id
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)
def test_contents_orm(self):
""" Contents (ORM): Test contents """
req_properties = get_request_properties()
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
content_properties = get_content_properties()
request_id = add_request(**req_properties)
trans_properties['request_id'] = request_id
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
content_properties['coll_id'] = coll_id
origin_content_id = add_content(**content_properties)
content_properties1 = copy.deepcopy(content_properties)
content_properties1['min_id'] = 101
content_properties1['max_id'] = 200
origin_content_id1 = add_content(**content_properties1)
content_properties2 = copy.deepcopy(content_properties)
content_properties2['min_id'] = 0
content_properties2['max_id'] = 200
origin_content_id2 = add_content(**content_properties2)
content_properties3 = copy.deepcopy(content_properties)
content_properties3['name'] = content_properties3['name'] + '_1'
origin_content_id3 = add_content(**content_properties3)
origin_content_ids = [
origin_content_id, origin_content_id1, origin_content_id2,
origin_content_id3
]
contents = get_contents(coll_id=coll_id)
assert_equal(len(contents), 4)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties['scope'],
name=content_properties['name'],
coll_id=coll_id)
assert_equal(len(contents), 3)
for content in contents:
assert_in(content['content_id'], origin_content_ids)
contents = get_contents(scope=content_properties3['scope'],
name=content_properties3['name'],
coll_id=coll_id)
assert_equal(len(contents), 1)
assert_equal(contents[0]['content_id'], origin_content_id3)
contents = get_match_contents(coll_id=content_properties['coll_id'],
scope=content_properties['scope'],
name=content_properties['name'],
min_id=content_properties['min_id'],
max_id=content_properties['max_id'])
assert_equal(len(contents), 2)
for content in contents:
assert_in(content['content_id'],
[origin_content_id, origin_content_id2])
to_updates = [{
'path': 'test_path1',
'status': ContentStatus.Processing,
'content_id': origin_content_id
}, {
'path': 'test_path2',
'status': ContentStatus.Processing,
'content_id': origin_content_id1
}]
update_contents(to_updates)
content = get_content(content_id=origin_content_id)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path1')
content = get_content(content_id=origin_content_id1)
assert_equal(content['status'], ContentStatus.Processing)
assert_equal(content['path'], 'test_path2')
def add_transform_outputs(transform,
transform_parameters,
input_collections=None,
output_collections=None,
log_collections=None,
update_input_collections=None,
update_output_collections=None,
update_log_collections=None,
new_contents=None,
update_contents=None,
new_processing=None,
update_processing=None,
messages=None,
update_messages=None,
message_bulk_size=10000,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collections: The new input collections.
:param output_collections: The new output collections.
:param log_collections: The new log collections.
:param update_input_collections: The updated input collections.
:param update_output_collections: The updated output collections.
:param update_log_collections: The updated log collections.
:param new_contents: The new contents.
:param update_contents: The updated contents.
:param new_processing: The new processing.
:param messages: Messages.
:param message_bulk_size: The message bulk size.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
work = transform['transform_metadata']['work']
if input_collections:
for coll in input_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if output_collections:
for coll in output_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if log_collections:
for coll in log_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if update_input_collections:
update_input_colls = [
coll.collection for coll in update_input_collections
]
orm_collections.update_collections(update_input_colls, session=session)
if update_output_collections:
update_output_colls = [
coll.collection for coll in update_output_collections
]
orm_collections.update_collections(update_output_colls,
session=session)
if update_log_collections:
update_log_colls = [coll.collection for coll in update_log_collections]
orm_collections.update_collections(update_log_colls, session=session)
if new_contents:
orm_contents.add_contents(new_contents, session=session)
if update_contents:
orm_contents.update_contents(update_contents, session=session)
processing_id = None
if new_processing:
# print(new_processing)
processing_id = orm_processings.add_processing(**new_processing,
session=session)
if update_processing:
for proc_id in update_processing:
orm_processings.update_processing(
processing_id=proc_id,
parameters=update_processing[proc_id],
session=session)
if messages:
if not type(messages) in [list, tuple]:
messages = [messages]
# for message in messages:
# orm_messages.add_message(msg_type=message['msg_type'],
# status=message['status'],
# source=message['source'],
# request_id=message['request_id'],
# workload_id=message['workload_id'],
# transform_id=message['transform_id'],
# num_contents=message['num_contents'],
# msg_content=message['msg_content'],
# bulk_size=message_bulk_size,
# session=session)
orm_messages.add_messages(messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
if transform:
if processing_id:
# work.set_processing_id(new_processing, processing_id)
work.set_processing_id(
new_processing['processing_metadata']['processing'],
processing_id)
work.refresh_work()
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=transform_parameters,
session=session)
