def get_transforms_by_status(status,
period=None,
locking=False,
bulk_size=None,
to_json=False,
session=None):
"""
Get transforms or raise a NoObject exception.
:param status: Transform status or list of transform status.
:param session: The database session in use.
:param locking: Whether to lock retrieved items.
:param to_json: return json format.
:raises NoObject: If no transform is founded.
:returns: list of transform.
"""
transforms = orm_transforms.get_transforms_by_status(status=status,
period=period,
locking=locking,
bulk_size=bulk_size,
to_json=to_json,
session=session)
if locking:
parameters = {'locking': TransformLocking.Locking}
for transform in transforms:
orm_transforms.update_transform(
transform_id=transform['transform_id'],
parameters=parameters,
session=session)
return transforms
def get_transforms_with_messaging(locking=False, bulk_size=None, session=None):
msgs = core_messages.retrieve_transform_messages(transform_id=None, bulk_size=bulk_size, session=session)
if msgs:
tf_ids = [msg['transform_id'] for msg in msgs]
if locking:
tf2s = orm_transforms.get_transforms_by_status(status=None, transform_ids=tf_ids,
locking=locking, locking_for_update=True,
bulk_size=None, session=session)
if tf2s:
transforms = []
for tf_id in tf_ids:
if len(transforms) >= bulk_size:
break
for tf in tf2s:
if tf['transform_id'] == tf_id:
transforms.append(tf)
break
else:
transforms = []
parameters = {'locking': TransformLocking.Locking}
for tf in transforms:
orm_transforms.update_transform(transform_id=tf['transform_id'], parameters=parameters, session=session)
return transforms
else:
transforms = orm_transforms.get_transforms_by_status(status=None, transform_ids=tf_ids, locking=locking,
locking_for_update=locking,
bulk_size=bulk_size, session=session)
return transforms
else:
return []
def update_processing_with_collection_contents(updated_processing,
new_processing=None,
updated_collection=None,
updated_files=None,
new_files=None,
coll_msg_content=None,
file_msg_content=None,
transform_updates=None,
message_bulk_size=1000,
session=None):
"""
Update processing with collection, contents, file messages and collection messages.
:param updated_processing: dict with processing id and parameters.
:param updated_collection: dict with collection id and parameters.
:param updated_files: list of content files.
:param coll_msg_content: message with collection info.
:param file_msg_content: message with files info.
"""
if updated_files:
orm_contents.update_contents(updated_files, session=session)
if new_files:
orm_contents.add_contents(contents=new_files, session=session)
if file_msg_content:
if not type(file_msg_content) in [list, tuple]:
file_msg_content = [file_msg_content]
for file_msg_con in file_msg_content:
orm_messages.add_message(msg_type=file_msg_con['msg_type'],
status=file_msg_con['status'],
source=file_msg_con['source'],
transform_id=file_msg_con['transform_id'],
num_contents=file_msg_con['num_contents'],
msg_content=file_msg_con['msg_content'],
bulk_size=message_bulk_size,
session=session)
if updated_collection:
orm_collections.update_collection(
coll_id=updated_collection['coll_id'],
parameters=updated_collection['parameters'],
session=session)
if coll_msg_content:
orm_messages.add_message(msg_type=coll_msg_content['msg_type'],
status=coll_msg_content['status'],
source=coll_msg_content['source'],
transform_id=coll_msg_content['transform_id'],
num_contents=coll_msg_content['num_contents'],
msg_content=coll_msg_content['msg_content'],
session=session)
if updated_processing:
orm_processings.update_processing(
processing_id=updated_processing['processing_id'],
parameters=updated_processing['parameters'],
session=session)
if new_processing:
orm_processings.add_processing(**new_processing, session=session)
if transform_updates:
orm_transforms.update_transform(
transform_id=transform_updates['transform_id'],
parameters=transform_updates['parameters'],
session=session)
def update_workprogress(workprogress_id,
parameters,
new_transforms=None,
update_transforms=None,
session=None):
"""
update a workprogress.
:param workprogress_id: the workprogress id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no workprogress is founded.
:raises DatabaseException: If there is a database error.
"""
if new_transforms:
for tf in new_transforms:
orginal_work = tf['transform_metadata']['orginal_work']
del tf['transform_metadata']['orginal_work']
tf_id = orm_transforms.add_transform(**tf, session=session)
# work = tf['transform_metadata']['work']
orginal_work.set_work_id(tf_id, transforming=True)
orginal_work.set_status(WorkStatus.New)
if update_transforms:
for tr_id in update_transforms:
orm_transforms.update_transform(
transform_id=tr_id,
parameters=update_transforms[tr_id],
session=session)
return orm_workprogress.update_workprogress(
workprogress_id=workprogress_id,
parameters=parameters,
session=session)
def update_request_with_transforms(request_id,
parameters,
transforms_to_add,
transforms_to_extend,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param transforms_to_add: list of transforms
:param transforms_to_extend: list of transforms
"""
for transform in transforms_to_add:
if 'collections' not in transform or len(
transform['collections']) == 0:
msg = "Transform must have collections, such as input collection, output collection and log collection"
raise exceptions.WrongParameterException(msg)
collections = transform['collections']
del transform['collections']
transform_id = orm_transforms.add_transform(**transform,
session=session)
input_coll_ids = []
log_coll_ids = []
for collection in collections['input_collections']:
collection['transform_id'] = transform_id
input_coll_id = orm_collections.add_collection(**collection,
session=session)
input_coll_ids.append(input_coll_id)
for collection in collections['log_collections']:
collection['transform_id'] = transform_id
log_coll_id = orm_collections.add_collection(**collection,
session=session)
log_coll_ids.append(log_coll_id)
for collection in collections['output_collections']:
collection['transform_id'] = transform_id
workload_id = transform['transform_metadata'][
'workload_id'] if 'workload_id' in transform[
'transform_metadata'] else None
collection['coll_metadata'] = {
'transform_id': transform_id,
'workload_id': workload_id,
'input_collections': input_coll_ids,
'log_collections': log_coll_ids
}
orm_collections.add_collection(**collection, session=session)
for transform in transforms_to_extend:
transform_id = transform['transform_id']
del transform['transform_id']
# orm_transforms.add_req2transform(request_id, transform_id, session=session)
orm_transforms.update_transform(transform_id,
parameters=transform,
session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def update_transform(transform_id, parameters, session=None):
"""
update a transform.
:param transform_id: the transform id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
orm_transforms.update_transform(transform_id=transform_id, parameters=parameters, session=session)
def test_create_and_check_for_transform_orm(self):
""" Transform (ORM): Test the creation, query, and cancel of a Transform """
trans_properties = get_transform_properties()
trans_id = add_transform(**trans_properties)
transform = get_transform(transform_id=trans_id)
for key in trans_properties:
assert_equal(transform[key], trans_properties[key])
update_transform(trans_id, {'status': TransformStatus.Failed})
transform = get_transform(transform_id=trans_id)
assert_equal(transform['status'], TransformStatus.Failed)
delete_transform(trans_id)
t = get_transform(transform_id=trans_id)
assert_equal(t, None)
def add_transform_outputs(transform,
input_collections=None,
output_collections=None,
log_collections=None,
update_input_collections=None,
update_output_collections=None,
update_log_collections=None,
new_contents=None,
update_contents=None,
new_processing=None,
messages=None,
message_bulk_size=1000,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collections: The new input collections.
:param output_collections: The new output collections.
:param log_collections: The new log collections.
:param update_input_collections: The updated input collections.
:param update_output_collections: The updated output collections.
:param update_log_collections: The updated log collections.
:param new_contents: The new contents.
:param update_contents: The updated contents.
:param new_processing: The new processing.
:param messages: Messages.
:param message_bulk_size: The message bulk size.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
work = transform['transform_metadata']['work']
if input_collections:
for coll in input_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if output_collections:
for coll in output_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if log_collections:
for coll in log_collections:
coll_id = orm_collections.add_collection(**coll, session=session)
work.set_collection_id(coll, coll_id)
if update_input_collections:
orm_collections.update_collections(update_input_collections,
session=session)
if update_output_collections:
orm_collections.update_collections(update_output_collections,
session=session)
if update_log_collections:
orm_collections.update_collections(update_log_collections,
session=session)
if new_contents:
orm_contents.add_contents(new_contents, session=session)
if update_contents:
orm_contents.add_contents(update_contents, session=session)
processing_id = None
if new_processing:
processing_id = orm_processings.add_processing(**new_processing,
session=session)
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if messages:
if not type(messages) in [list, tuple]:
messages = [messages]
for message in messages:
orm_messages.add_message(msg_type=message['msg_type'],
status=message['status'],
source=message['source'],
transform_id=message['transform_id'],
num_contents=message['num_contents'],
msg_content=message['msg_content'],
bulk_size=message_bulk_size,
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing, session=session)
"""
if transform:
"""
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {'processing_id': processing_id}
else:
transform['transform_metadata']['processing_id'] = processing_id
"""
if processing_id:
work.set_processing_id(new_processing, processing_id)
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)
def add_transform_outputs(transform,
input_collection,
output_collection,
input_contents,
output_contents,
processing,
to_cancel_processing=None,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collection: The input collection.
:param output_collection: The output collection.
:param input_contents: The input contents.
:param output_contents: The corresponding output contents.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
if output_contents:
orm_contents.add_contents(output_contents, session=session)
if input_contents:
update_input_contents = []
for input_content in input_contents:
update_input_content = {
'content_id': input_content['content_id'],
'status': ContentStatus.Mapped,
'path': None
}
update_input_contents.append(update_input_content)
if update_input_contents:
orm_contents.update_contents(update_input_contents,
with_content_id=True,
session=session)
if output_collection:
# TODO, the status and new_files should be updated
orm_collections.update_collection(
output_collection['coll_id'],
{'status': CollectionStatus.Processing},
session=session)
if to_cancel_processing:
to_cancel_params = {'status': ProcessingStatus.Cancel}
for to_cancel_id in to_cancel_processing:
orm_processings.update_processing(processing_id=to_cancel_id,
parameters=to_cancel_params,
session=session)
processing_id = None
if processing:
processing_id = orm_processings.add_processing(**processing,
session=session)
if transform:
if processing_id is not None:
if not transform['transform_metadata']:
transform['transform_metadata'] = {
'processing_id': processing_id
}
else:
transform['transform_metadata'][
'processing_id'] = processing_id
parameters = {
'status': transform['status'],
'locking': transform['locking'],
'transform_metadata': transform['transform_metadata']
}
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=parameters,
session=session)
def trigger_update_transform_status(transform_id,
input_collection_changed=False,
output_collection_changed=False,
session=None):
"""
update transform status based on input/output collection changes.
:param transform_id: the transform id.
:param input_collection_changed: Whether input collection is changed.
:param output_collection_changed: Whether output collection is changed.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
if not input_collection_changed and not output_collection_changed:
return
transform = orm_transforms.get_transform(transform_id, session=session)
status = transform['status']
transform_metadata = transform['transform_metadata']
if 'input_collection_changed' not in transform_metadata:
transform_metadata[
'input_collection_changed'] = input_collection_changed
else:
transform_metadata['input_collection_changed'] = transform_metadata[
'input_collection_changed'] or input_collection_changed
if 'output_collection_changed' not in transform_metadata:
transform_metadata[
'output_collection_changed'] = output_collection_changed
else:
transform_metadata['output_collection_changed'] = transform_metadata[
'output_collection_changed'] or output_collection_changed
if isinstance(status, TransformStatus):
status = status.value
new_status = status
if input_collection_changed:
if status in [
TransformStatus.ToCancel.value,
TransformStatus.Cancelling.value, TransformStatus.Failed.value,
TransformStatus.Cancelled.value
]:
new_status = status
elif status in [
TransformStatus.New.value, TransformStatus.Extend.value
]:
new_status = TransformStatus.Ready.value
elif status in [TransformStatus.Transforming.value]:
new_status = TransformStatus.Transforming.value
elif status in [
TransformStatus.Finished.value,
TransformStatus.SubFinished.value
]:
new_status = TransformStatus.Transforming.value
elif input_collection_changed or output_collection_changed:
if status in [
TransformStatus.ToCancel.value,
TransformStatus.Cancelling.value, TransformStatus.Failed.value,
TransformStatus.Cancelled.value
]:
new_status = status
else:
new_status = TransformStatus.Transforming.value
parameters = {
'status': new_status,
'transform_metadata': transform_metadata
}
orm_transforms.update_transform(transform_id=transform_id,
parameters=parameters,
session=session)
def update_request_with_transforms(request_id,
parameters,
new_transforms=None,
update_transforms=None,
new_messages=None,
update_messages=None,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param new_transforms: list of transforms
:param update_transforms: list of transforms
"""
if new_transforms:
for tf in new_transforms:
# tf_id = orm_transforms.add_transform(**tf, session=session)
# original_work = tf['transform_metadata']['original_work']
# del tf['transform_metadata']['original_work']
workflow = tf['transform_metadata']['workflow']
del tf['transform_metadata']['workflow']
work = tf['transform_metadata']['work']
tf_copy = copy.deepcopy(tf)
tf_id = orm_transforms.add_transform(**tf_copy, session=session)
tf['transform_id'] = tf_id
# work = tf['transform_metadata']['work']
# original_work.set_work_id(tf_id, transforming=True)
# original_work.set_status(WorkStatus.New)
work.set_work_id(tf_id, transforming=True)
work.set_status(WorkStatus.New)
workflow.refresh_works()
collections = generate_collections(tf)
for coll in collections:
collection = coll['collection']
del coll['collection']
coll['transform_id'] = tf_id
coll_id = orm_collections.add_collection(**coll,
session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
# update transform to record the coll_id
work.refresh_work()
orm_transforms.update_transform(
transform_id=tf_id,
parameters={'transform_metadata': tf['transform_metadata']},
session=session)
if update_transforms:
for tr_id in update_transforms:
orm_transforms.update_transform(
transform_id=tr_id,
parameters=update_transforms[tr_id],
session=session)
if new_messages:
orm_messages.add_messages(new_messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def add_transform_outputs(transform,
transform_parameters,
input_collections=None,
output_collections=None,
log_collections=None,
update_input_collections=None,
update_output_collections=None,
update_log_collections=None,
new_contents=None,
update_contents=None,
new_processing=None,
update_processing=None,
messages=None,
update_messages=None,
message_bulk_size=10000,
session=None):
"""
For input contents, add corresponding output contents.
:param transform: the transform.
:param input_collections: The new input collections.
:param output_collections: The new output collections.
:param log_collections: The new log collections.
:param update_input_collections: The updated input collections.
:param update_output_collections: The updated output collections.
:param update_log_collections: The updated log collections.
:param new_contents: The new contents.
:param update_contents: The updated contents.
:param new_processing: The new processing.
:param messages: Messages.
:param message_bulk_size: The message bulk size.
:param session: The database session in use.
:raises DatabaseException: If there is a database error.
"""
work = transform['transform_metadata']['work']
if input_collections:
for coll in input_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if output_collections:
for coll in output_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if log_collections:
for coll in log_collections:
collection = coll['collection']
del coll['collection']
coll_id = orm_collections.add_collection(**coll, session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
if update_input_collections:
update_input_colls = [
coll.collection for coll in update_input_collections
]
orm_collections.update_collections(update_input_colls, session=session)
if update_output_collections:
update_output_colls = [
coll.collection for coll in update_output_collections
]
orm_collections.update_collections(update_output_colls,
session=session)
if update_log_collections:
update_log_colls = [coll.collection for coll in update_log_collections]
orm_collections.update_collections(update_log_colls, session=session)
if new_contents:
orm_contents.add_contents(new_contents, session=session)
if update_contents:
orm_contents.update_contents(update_contents, session=session)
processing_id = None
if new_processing:
# print(new_processing)
processing_id = orm_processings.add_processing(**new_processing,
session=session)
if update_processing:
for proc_id in update_processing:
orm_processings.update_processing(
processing_id=proc_id,
parameters=update_processing[proc_id],
session=session)
if messages:
if not type(messages) in [list, tuple]:
messages = [messages]
# for message in messages:
# orm_messages.add_message(msg_type=message['msg_type'],
# status=message['status'],
# source=message['source'],
# request_id=message['request_id'],
# workload_id=message['workload_id'],
# transform_id=message['transform_id'],
# num_contents=message['num_contents'],
# msg_content=message['msg_content'],
# bulk_size=message_bulk_size,
# session=session)
orm_messages.add_messages(messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
if transform:
if processing_id:
# work.set_processing_id(new_processing, processing_id)
work.set_processing_id(
new_processing['processing_metadata']['processing'],
processing_id)
work.refresh_work()
orm_transforms.update_transform(transform_id=transform['transform_id'],
parameters=transform_parameters,
session=session)
def get_transforms_by_status(status,
period=None,
locking=False,
bulk_size=None,
to_json=False,
by_substatus=False,
with_messaging=False,
session=None):
"""
Get transforms or raise a NoObject exception.
:param status: Transform status or list of transform status.
:param session: The database session in use.
:param locking: Whether to lock retrieved items.
:param to_json: return json format.
:raises NoObject: If no transform is founded.
:returns: list of transform.
"""
if with_messaging:
transforms = get_transforms_with_messaging(locking=locking,
bulk_size=bulk_size,
session=session)
if transforms:
return transforms
if locking:
if bulk_size:
# order by cannot work together with locking. So first select 2 * bulk_size without locking with order by.
# then select with locking.
tf_ids = orm_transforms.get_transforms_by_status(
status=status,
period=period,
locking=locking,
bulk_size=bulk_size * 2,
locking_for_update=False,
to_json=False,
only_return_id=True,
by_substatus=by_substatus,
session=session)
if tf_ids:
transform2s = orm_transforms.get_transforms_by_status(
status=status,
period=period,
locking=locking,
bulk_size=None,
locking_for_update=True,
to_json=to_json,
transform_ids=tf_ids,
by_substatus=by_substatus,
session=session)
if transform2s:
# reqs = req2s[:bulk_size]
# order requests
transforms = []
for tf_id in tf_ids:
if len(transforms) >= bulk_size:
break
for tf in transform2s:
if tf['transform_id'] == tf_id:
transforms.append(tf)
break
# transforms = transforms[:bulk_size]
else:
transforms = []
else:
transforms = []
else:
transforms = orm_transforms.get_transforms_by_status(
status=status,
period=period,
locking=locking,
locking_for_update=locking,
bulk_size=bulk_size,
to_json=to_json,
by_substatus=by_substatus,
session=session)
parameters = {'locking': TransformLocking.Locking}
for transform in transforms:
orm_transforms.update_transform(
transform_id=transform['transform_id'],
parameters=parameters,
session=session)
else:
transforms = orm_transforms.get_transforms_by_status(
status=status,
period=period,
locking=locking,
bulk_size=bulk_size,
to_json=to_json,
by_substatus=by_substatus,
session=session)
return transforms
