def update_processing_with_collection_contents(updated_processing,
new_processing=None,
updated_collection=None,
updated_files=None,
new_files=None,
coll_msg_content=None,
file_msg_content=None,
transform_updates=None,
message_bulk_size=1000,
session=None):
"""
Update processing with collection, contents, file messages and collection messages.
:param updated_processing: dict with processing id and parameters.
:param updated_collection: dict with collection id and parameters.
:param updated_files: list of content files.
:param coll_msg_content: message with collection info.
:param file_msg_content: message with files info.
"""
if updated_files:
orm_contents.update_contents(updated_files, session=session)
if new_files:
orm_contents.add_contents(contents=new_files, session=session)
if file_msg_content:
if not type(file_msg_content) in [list, tuple]:
file_msg_content = [file_msg_content]
for file_msg_con in file_msg_content:
orm_messages.add_message(msg_type=file_msg_con['msg_type'],
status=file_msg_con['status'],
source=file_msg_con['source'],
transform_id=file_msg_con['transform_id'],
num_contents=file_msg_con['num_contents'],
msg_content=file_msg_con['msg_content'],
bulk_size=message_bulk_size,
session=session)
if updated_collection:
orm_collections.update_collection(
coll_id=updated_collection['coll_id'],
parameters=updated_collection['parameters'],
session=session)
if coll_msg_content:
orm_messages.add_message(msg_type=coll_msg_content['msg_type'],
status=coll_msg_content['status'],
source=coll_msg_content['source'],
transform_id=coll_msg_content['transform_id'],
num_contents=coll_msg_content['num_contents'],
msg_content=coll_msg_content['msg_content'],
session=session)
if updated_processing:
orm_processings.update_processing(
processing_id=updated_processing['processing_id'],
parameters=updated_processing['parameters'],
session=session)
if new_processing:
orm_processings.add_processing(**new_processing, session=session)
if transform_updates:
orm_transforms.update_transform(
transform_id=transform_updates['transform_id'],
parameters=transform_updates['parameters'],
session=session)
def add_processing(transform_id,
status,
submitter=None,
granularity=None,
granularity_type=None,
expired_at=None,
processing_metadata=None,
session=None):
"""
Add a processing.
:param transform_id: Transform id.
:param status: processing status.
:param submitter: submitter name.
:param granularity: Granularity size.
:param granularity_type: Granularity type.
:param expired_at: The datetime when it expires.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If a processing with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: processing id.
"""
return orm_processings.add_processing(
transform_id=transform_id,
status=status,
submitter=submitter,
granularity=granularity,
granularity_type=granularity_type,
expired_at=expired_at,
processing_metadata=processing_metadata,
session=session)
def test_create_and_check_for_transform_processing_orm(self):
""" Transform/Processing (ORM): Test the creation, query, and cancel of a Transform/Processing """
trans_properties = get_transform_properties()
proc_properties = get_processing_properties()
trans_id = add_transform(**trans_properties)
proc_properties['transform_id'] = trans_id
processing_id = add_processing(**proc_properties)
processing = get_processing(processing_id=processing_id)
for key in proc_properties:
assert_equal(processing[key], proc_properties[key])
update_processing(processing_id, {'status': ProcessingStatus.Failed})
processing = get_processing(processing_id=processing_id)
assert_equal(processing['status'], ProcessingStatus.Failed)
delete_processing(processing_id)
with assert_raises(exceptions.NoObject):
get_processing(processing_id=processing_id)
delete_transform(trans_id)
def add_transform_outputs(transform,
input_collections=None,
output_collections=None,
log_collections=None,
update_input_collections=None,
update_output_collections=None,
update_log_collections=None,
new_contents=None,
update_contents=None,
new_processing=None,
messages=None,
message_bulk_size=1000,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collections: The new input collections.
:param output_collections: The new output collections.
:param log_collections: The new log collections.
:param update_input_collections: The updated input collections.
:param update_output_collections: The updated output collections.
:param update_log_collections: The updated log collections.
:param new_contents: The new contents.
:param update_contents: The updated contents.
:param new_processing: The new processing.
:param messages: Messages.
:param message_bulk_size: The message bulk size.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
work = transform['transform_metadata']['work']
if input_collections:
for coll in input_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if output_collections:
for coll in output_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if log_collections:
for coll in log_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if update_input_collections:
orm_collections.update_collections(update_input_collections,
session=session)
if update_output_collections:
orm_collections.update_collections(update_output_collections,
session=session)
if update_log_collections:
orm_collections.update_collections(update_log_collections,
session=session)
if new_contents:
orm_contents.add_contents(new_contents, session=session)
if update_contents:
orm_contents.add_contents(update_contents, session=session)
processing_id = None
if new_processing:
processing_id = orm_processings.add_processing(**new_processing,
session=session)
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if messages:
if not type(messages) in [list, tuple]:
messages = [messages]
for message in messages:
orm_messages.add_message(msg_type=message['msg_type'],
status=message['status'],
source=message['source'],
transform_id=message['transform_id'],
num_contents=message['num_contents'],
msg_content=message['msg_content'],
bulk_size=message_bulk_size,
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing, session=session)
"""
if transform:
"""
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {'processing_id': processing_id}
else:
transform['transform_metadata']['processing_id'] = processing_id
"""
if processing_id:
work.set_processing_id(new_processing, processing_id)
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)
def add_transform_outputs(transform,
input_collection,
output_collection,
input_contents,
output_contents,
processing,
to_cancel_processing=None,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collection: The input collection.
:param output_collection: The output collection.
:param input_contents: The input contents.
:param output_contents: The corresponding output contents.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if input_contents:
update_input_contents = []
for input_content in input_contents:
update_input_content = {
'content_id': input_content['content_id'],
'status': ContentStatus.Mapped,
'path': None
}
update_input_contents.append(update_input_content)
if update_input_contents:
orm_contents.update_contents(update_input_contents,
with_content_id=True,
session=session)
if output_collection:
# TODO, the status and new_files should be updated
orm_collections.update_collection(
output_collection['coll_id'],
{'status': CollectionStatus.Processing},
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id,
parameters=to_cancel_params,
session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing,
session=session)
if transform:
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {
'processing_id': processing_id
}
else:
transform['transform_metadata'][
'processing_id'] = processing_id
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)
def add_transform_outputs(transform,
transform_parameters,
input_collections=None,
output_collections=None,
log_collections=None,
update_input_collections=None,
update_output_collections=None,
update_log_collections=None,
new_contents=None,
update_contents=None,
new_processing=None,
update_processing=None,
messages=None,
update_messages=None,
message_bulk_size=10000,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collections: The new input collections.
:param output_collections: The new output collections.
:param log_collections: The new log collections.
:param update_input_collections: The updated input collections.
:param update_output_collections: The updated output collections.
:param update_log_collections: The updated log collections.
:param new_contents: The new contents.
:param update_contents: The updated contents.
:param new_processing: The new processing.
:param messages: Messages.
:param message_bulk_size: The message bulk size.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
work = transform['transform_metadata']['work']
if input_collections:
for coll in input_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if output_collections:
for coll in output_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if log_collections:
for coll in log_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if update_input_collections:
update_input_colls = [
coll.collection for coll in update_input_collections
]
orm_collections.update_collections(update_input_colls, session=session)
if update_output_collections:
update_output_colls = [
coll.collection for coll in update_output_collections
]
orm_collections.update_collections(update_output_colls,
session=session)
if update_log_collections:
update_log_colls = [coll.collection for coll in update_log_collections]
orm_collections.update_collections(update_log_colls, session=session)
if new_contents:
orm_contents.add_contents(new_contents, session=session)
if update_contents:
orm_contents.update_contents(update_contents, session=session)
processing_id = None
if new_processing:
# print(new_processing)
processing_id = orm_processings.add_processing(**new_processing,
session=session)
if update_processing:
for proc_id in update_processing:
orm_processings.update_processing(
processing_id=proc_id,
parameters=update_processing[proc_id],
session=session)
if messages:
if not type(messages) in [list, tuple]:
messages = [messages]
# for message in messages:
# orm_messages.add_message(msg_type=message['msg_type'],
# status=message['status'],
# source=message['source'],
# request_id=message['request_id'],
# workload_id=message['workload_id'],
# transform_id=message['transform_id'],
# num_contents=message['num_contents'],
# msg_content=message['msg_content'],
# bulk_size=message_bulk_size,
# session=session)
orm_messages.add_messages(messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
if transform:
if processing_id:
# work.set_processing_id(new_processing, processing_id)
work.set_processing_id(
new_processing['processing_metadata']['processing'],
processing_id)
work.refresh_work()
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=transform_parameters,
session=session)