def update_processing_with_collection_contents(updated_processing,
new_processing=None,
updated_collection=None,
updated_files=None,
new_files=None,
coll_msg_content=None,
file_msg_content=None,
transform_updates=None,
message_bulk_size=1000,
session=None):
"""
Update processing with collection, contents, file messages and collection messages.
:param updated_processing: dict with processing id and parameters.
:param updated_collection: dict with collection id and parameters.
:param updated_files: list of content files.
:param coll_msg_content: message with collection info.
:param file_msg_content: message with files info.
"""
if updated_files:
orm_contents.update_contents(updated_files, session=session)
if new_files:
orm_contents.add_contents(contents=new_files, session=session)
if file_msg_content:
if not type(file_msg_content) in [list, tuple]:
file_msg_content = [file_msg_content]
for file_msg_con in file_msg_content:
orm_messages.add_message(msg_type=file_msg_con['msg_type'],
status=file_msg_con['status'],
source=file_msg_con['source'],
transform_id=file_msg_con['transform_id'],
num_contents=file_msg_con['num_contents'],
msg_content=file_msg_con['msg_content'],
bulk_size=message_bulk_size,
session=session)
if updated_collection:
orm_collections.update_collection(
coll_id=updated_collection['coll_id'],
parameters=updated_collection['parameters'],
session=session)
if coll_msg_content:
orm_messages.add_message(msg_type=coll_msg_content['msg_type'],
status=coll_msg_content['status'],
source=coll_msg_content['source'],
transform_id=coll_msg_content['transform_id'],
num_contents=coll_msg_content['num_contents'],
msg_content=coll_msg_content['msg_content'],
session=session)
if updated_processing:
orm_processings.update_processing(
processing_id=updated_processing['processing_id'],
parameters=updated_processing['parameters'],
session=session)
if new_processing:
orm_processings.add_processing(**new_processing, session=session)
if transform_updates:
orm_transforms.update_transform(
transform_id=transform_updates['transform_id'],
parameters=transform_updates['parameters'],
session=session)
def get_collections_by_status(status,
relation_type=CollectionRelationType.Input,
time_period=None,
locking=False,
bulk_size=None,
session=None):
"""
Get collections by status, relation_type and time_period or raise a NoObject exception.
:param status: The collection status.
:param relation_type: The relation_type of the collection to the transform.
:param time_period: time period in seconds since last update.
:param locking: Whether to retrieve unlocked files and lock them.
:param session: The database session in use.
:raises NoObject: If no collections are founded.
:returns: list of Collections.
"""
colls = orm_collections.get_collections_by_status(
status=status,
relation_type=relation_type,
bulk_size=bulk_size,
time_period=time_period,
locking=locking,
session=session)
if locking:
parameters = {'locking': CollectionLocking.Locking}
for coll in colls:
orm_collections.update_collection(coll_id=coll['coll_id'],
parameters=parameters,
session=session)
return colls
def update_collection(coll_id, parameters, msg=None, session=None):
"""
update a collection.
:param coll_id: the collection id.
:param parameters: A dictionary of parameters.
:param msg: messages.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
"""
orm_collections.update_collection(coll_id=coll_id,
parameters=parameters,
session=session)
if msg:
orm_messages.add_message(msg_type=msg['msg_type'],
status=msg['status'],
source=msg['source'],
transform_id=msg['transform_id'],
num_contents=msg['num_contents'],
msg_content=msg['msg_content'],
session=session)
def test_create_and_check_for_transform_collection_content_orm(self):
""" Transform/Collection/Content (ORM): Test the creation, query, and cancel of a Transform/Collection/Content """
trans_properties = get_transform_properties()
coll_properties = get_collection_properties()
content_properties = get_content_properties()
trans_id = add_transform(**trans_properties)
coll_properties['transform_id'] = trans_id
coll_id = add_collection(**coll_properties)
coll_id_1 = get_collection_id(
transform_id=trans_id,
relation_type=coll_properties['relation_type'])
assert_equal(coll_id, coll_id_1)
coll = get_collection(coll_id=coll_id)
for key in coll_properties:
assert_equal(coll[key], coll_properties[key])
update_collection(coll_id, {'status': CollectionStatus.Closed})
coll = get_collection(coll_id=coll_id)
assert_equal(coll['status'], CollectionStatus.Closed)
content_properties['coll_id'] = coll_id
content_id = add_content(**content_properties)
content_1 = get_content(coll_id=coll_id,
scope=content_properties['scope'],
name=content_properties['name'],
content_type=ContentType.File)
content_id_1 = content_1['content_id']
assert_equal(content_id, content_id_1)
content = get_content(content_id=content_id)
update_content(content_id=content_id,
parameters={'status': ContentStatus.Lost})
content = get_content(content_id=content_id)
assert_equal(content['status'], ContentStatus.Lost)
delete_content(content_id=content_id)
c = get_content(content_id=content_id)
assert_equal(c, None)
delete_collection(coll_id=coll_id)
coll = get_collection(coll_id=coll_id)
assert_equal(coll, None)
delete_transform(transform_id=trans_id)
t = get_transform(transform_id=trans_id)
assert_equal(t, None)
def update_collection(coll_id, parameters, session=None):
"""
update a collection.
:param coll_id: the collection id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
"""
orm_collections.update_collection(coll_id=coll_id,
parameters=parameters,
session=session)
def add_transform_outputs(transform,
input_collection,
output_collection,
input_contents,
output_contents,
processing,
to_cancel_processing=None,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collection: The input collection.
:param output_collection: The output collection.
:param input_contents: The input contents.
:param output_contents: The corresponding output contents.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if input_contents:
update_input_contents = []
for input_content in input_contents:
update_input_content = {
'content_id': input_content['content_id'],
'status': ContentStatus.Mapped,
'path': None
}
update_input_contents.append(update_input_content)
if update_input_contents:
orm_contents.update_contents(update_input_contents,
with_content_id=True,
session=session)
if output_collection:
# TODO, the status and new_files should be updated
orm_collections.update_collection(
output_collection['coll_id'],
{'status': CollectionStatus.Processing},
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id,
parameters=to_cancel_params,
session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing,
session=session)
if transform:
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {
'processing_id': processing_id
}
else:
transform['transform_metadata'][
'processing_id'] = processing_id
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)