def get_transforms_by_status(status, period=None, locking=False, bulk_size=None, to_json=False, session=None): """ Get transforms or raise a NoObject exception. :param status: Transform status or list of transform status. :param session: The database session in use. :param locking: Whether to lock retrieved items. :param to_json: return json format. :raises NoObject: If no transform is founded. :returns: list of transform. """ transforms = orm_transforms.get_transforms_by_status(status=status, period=period, locking=locking, bulk_size=bulk_size, to_json=to_json, session=session) if locking: parameters = {'locking': TransformLocking.Locking} for transform in transforms: orm_transforms.update_transform( transform_id=transform['transform_id'], parameters=parameters, session=session) return transforms
def get_transforms_with_messaging(locking=False, bulk_size=None, session=None): msgs = core_messages.retrieve_transform_messages(transform_id=None, bulk_size=bulk_size, session=session) if msgs: tf_ids = [msg['transform_id'] for msg in msgs] if locking: tf2s = orm_transforms.get_transforms_by_status(status=None, transform_ids=tf_ids, locking=locking, locking_for_update=True, bulk_size=None, session=session) if tf2s: transforms = [] for tf_id in tf_ids: if len(transforms) >= bulk_size: break for tf in tf2s: if tf['transform_id'] == tf_id: transforms.append(tf) break else: transforms = [] parameters = {'locking': TransformLocking.Locking} for tf in transforms: orm_transforms.update_transform(transform_id=tf['transform_id'], parameters=parameters, session=session) return transforms else: transforms = orm_transforms.get_transforms_by_status(status=None, transform_ids=tf_ids, locking=locking, locking_for_update=locking, bulk_size=bulk_size, session=session) return transforms else: return []
def update_processing_with_collection_contents(updated_processing, new_processing=None, updated_collection=None, updated_files=None, new_files=None, coll_msg_content=None, file_msg_content=None, transform_updates=None, message_bulk_size=1000, session=None): """ Update processing with collection, contents, file messages and collection messages. :param updated_processing: dict with processing id and parameters. :param updated_collection: dict with collection id and parameters. :param updated_files: list of content files. :param coll_msg_content: message with collection info. :param file_msg_content: message with files info. """ if updated_files: orm_contents.update_contents(updated_files, session=session) if new_files: orm_contents.add_contents(contents=new_files, session=session) if file_msg_content: if not type(file_msg_content) in [list, tuple]: file_msg_content = [file_msg_content] for file_msg_con in file_msg_content: orm_messages.add_message(msg_type=file_msg_con['msg_type'], status=file_msg_con['status'], source=file_msg_con['source'], transform_id=file_msg_con['transform_id'], num_contents=file_msg_con['num_contents'], msg_content=file_msg_con['msg_content'], bulk_size=message_bulk_size, session=session) if updated_collection: orm_collections.update_collection( coll_id=updated_collection['coll_id'], parameters=updated_collection['parameters'], session=session) if coll_msg_content: orm_messages.add_message(msg_type=coll_msg_content['msg_type'], status=coll_msg_content['status'], source=coll_msg_content['source'], transform_id=coll_msg_content['transform_id'], num_contents=coll_msg_content['num_contents'], msg_content=coll_msg_content['msg_content'], session=session) if updated_processing: orm_processings.update_processing( processing_id=updated_processing['processing_id'], parameters=updated_processing['parameters'], session=session) if new_processing: orm_processings.add_processing(**new_processing, session=session) if transform_updates: orm_transforms.update_transform( transform_id=transform_updates['transform_id'], parameters=transform_updates['parameters'], session=session)
def update_workprogress(workprogress_id, parameters, new_transforms=None, update_transforms=None, session=None): """ update a workprogress. :param workprogress_id: the workprogress id. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no workprogress is founded. :raises DatabaseException: If there is a database error. """ if new_transforms: for tf in new_transforms: orginal_work = tf['transform_metadata']['orginal_work'] del tf['transform_metadata']['orginal_work'] tf_id = orm_transforms.add_transform(**tf, session=session) # work = tf['transform_metadata']['work'] orginal_work.set_work_id(tf_id, transforming=True) orginal_work.set_status(WorkStatus.New) if update_transforms: for tr_id in update_transforms: orm_transforms.update_transform( transform_id=tr_id, parameters=update_transforms[tr_id], session=session) return orm_workprogress.update_workprogress( workprogress_id=workprogress_id, parameters=parameters, session=session)
def update_request_with_transforms(request_id, parameters, transforms_to_add, transforms_to_extend, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param transforms_to_add: list of transforms :param transforms_to_extend: list of transforms """ for transform in transforms_to_add: if 'collections' not in transform or len( transform['collections']) == 0: msg = "Transform must have collections, such as input collection, output collection and log collection" raise exceptions.WrongParameterException(msg) collections = transform['collections'] del transform['collections'] transform_id = orm_transforms.add_transform(**transform, session=session) input_coll_ids = [] log_coll_ids = [] for collection in collections['input_collections']: collection['transform_id'] = transform_id input_coll_id = orm_collections.add_collection(**collection, session=session) input_coll_ids.append(input_coll_id) for collection in collections['log_collections']: collection['transform_id'] = transform_id log_coll_id = orm_collections.add_collection(**collection, session=session) log_coll_ids.append(log_coll_id) for collection in collections['output_collections']: collection['transform_id'] = transform_id workload_id = transform['transform_metadata'][ 'workload_id'] if 'workload_id' in transform[ 'transform_metadata'] else None collection['coll_metadata'] = { 'transform_id': transform_id, 'workload_id': workload_id, 'input_collections': input_coll_ids, 'log_collections': log_coll_ids } orm_collections.add_collection(**collection, session=session) for transform in transforms_to_extend: transform_id = transform['transform_id'] del transform['transform_id'] # orm_transforms.add_req2transform(request_id, transform_id, session=session) orm_transforms.update_transform(transform_id, parameters=transform, session=session) return orm_requests.update_request(request_id, parameters, session=session)
def update_transform(transform_id, parameters, session=None): """ update a transform. :param transform_id: the transform id. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no content is founded. :raises DatabaseException: If there is a database error. """ orm_transforms.update_transform(transform_id=transform_id, parameters=parameters, session=session)
def test_create_and_check_for_transform_orm(self): """ Transform (ORM): Test the creation, query, and cancel of a Transform """ trans_properties = get_transform_properties() trans_id = add_transform(**trans_properties) transform = get_transform(transform_id=trans_id) for key in trans_properties: assert_equal(transform[key], trans_properties[key]) update_transform(trans_id, {'status': TransformStatus.Failed}) transform = get_transform(transform_id=trans_id) assert_equal(transform['status'], TransformStatus.Failed) delete_transform(trans_id) t = get_transform(transform_id=trans_id) assert_equal(t, None)
def add_transform_outputs(transform, input_collections=None, output_collections=None, log_collections=None, update_input_collections=None, update_output_collections=None, update_log_collections=None, new_contents=None, update_contents=None, new_processing=None, messages=None, message_bulk_size=1000, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collections: The new input collections. :param output_collections: The new output collections. :param log_collections: The new log collections. :param update_input_collections: The updated input collections. :param update_output_collections: The updated output collections. :param update_log_collections: The updated log collections. :param new_contents: The new contents. :param update_contents: The updated contents. :param new_processing: The new processing. :param messages: Messages. :param message_bulk_size: The message bulk size. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ work = transform['transform_metadata']['work'] if input_collections: for coll in input_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if output_collections: for coll in output_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if log_collections: for coll in log_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if update_input_collections: orm_collections.update_collections(update_input_collections, session=session) if update_output_collections: orm_collections.update_collections(update_output_collections, session=session) if update_log_collections: orm_collections.update_collections(update_log_collections, session=session) if new_contents: orm_contents.add_contents(new_contents, session=session) if update_contents: orm_contents.add_contents(update_contents, session=session) processing_id = None if new_processing: processing_id = orm_processings.add_processing(**new_processing, session=session) """ if output_contents: orm_contents.add_contents(output_contents, session=session) if messages: if not type(messages) in [list, tuple]: messages = [messages] for message in messages: orm_messages.add_message(msg_type=message['msg_type'], status=message['status'], source=message['source'], transform_id=message['transform_id'], num_contents=message['num_contents'], msg_content=message['msg_content'], bulk_size=message_bulk_size, session=session) if to_cancel_processing: to_cancel_params = {'status': ProcessingStatus.Cancel} for to_cancel_id in to_cancel_processing: orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session) processing_id = None if processing: processing_id = orm_processings.add_processing(**processing, session=session) """ if transform: """ if processing_id is not None: if not transform['transform_metadata']: transform['transform_metadata'] = {'processing_id': processing_id} else: transform['transform_metadata']['processing_id'] = processing_id """ if processing_id: work.set_processing_id(new_processing, processing_id) parameters = { 'status': transform['status'], 'locking': transform['locking'], 'transform_metadata': transform['transform_metadata'] } orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=parameters, session=session)
def add_transform_outputs(transform, input_collection, output_collection, input_contents, output_contents, processing, to_cancel_processing=None, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collection: The input collection. :param output_collection: The output collection. :param input_contents: The input contents. :param output_contents: The corresponding output contents. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ if output_contents: orm_contents.add_contents(output_contents, session=session) if input_contents: update_input_contents = [] for input_content in input_contents: update_input_content = { 'content_id': input_content['content_id'], 'status': ContentStatus.Mapped, 'path': None } update_input_contents.append(update_input_content) if update_input_contents: orm_contents.update_contents(update_input_contents, with_content_id=True, session=session) if output_collection: # TODO, the status and new_files should be updated orm_collections.update_collection( output_collection['coll_id'], {'status': CollectionStatus.Processing}, session=session) if to_cancel_processing: to_cancel_params = {'status': ProcessingStatus.Cancel} for to_cancel_id in to_cancel_processing: orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session) processing_id = None if processing: processing_id = orm_processings.add_processing(**processing, session=session) if transform: if processing_id is not None: if not transform['transform_metadata']: transform['transform_metadata'] = { 'processing_id': processing_id } else: transform['transform_metadata'][ 'processing_id'] = processing_id parameters = { 'status': transform['status'], 'locking': transform['locking'], 'transform_metadata': transform['transform_metadata'] } orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=parameters, session=session)
def trigger_update_transform_status(transform_id, input_collection_changed=False, output_collection_changed=False, session=None): """ update transform status based on input/output collection changes. :param transform_id: the transform id. :param input_collection_changed: Whether input collection is changed. :param output_collection_changed: Whether output collection is changed. :param session: The database session in use. :raises NoObject: If no content is founded. :raises DatabaseException: If there is a database error. """ if not input_collection_changed and not output_collection_changed: return transform = orm_transforms.get_transform(transform_id, session=session) status = transform['status'] transform_metadata = transform['transform_metadata'] if 'input_collection_changed' not in transform_metadata: transform_metadata[ 'input_collection_changed'] = input_collection_changed else: transform_metadata['input_collection_changed'] = transform_metadata[ 'input_collection_changed'] or input_collection_changed if 'output_collection_changed' not in transform_metadata: transform_metadata[ 'output_collection_changed'] = output_collection_changed else: transform_metadata['output_collection_changed'] = transform_metadata[ 'output_collection_changed'] or output_collection_changed if isinstance(status, TransformStatus): status = status.value new_status = status if input_collection_changed: if status in [ TransformStatus.ToCancel.value, TransformStatus.Cancelling.value, TransformStatus.Failed.value, TransformStatus.Cancelled.value ]: new_status = status elif status in [ TransformStatus.New.value, TransformStatus.Extend.value ]: new_status = TransformStatus.Ready.value elif status in [TransformStatus.Transforming.value]: new_status = TransformStatus.Transforming.value elif status in [ TransformStatus.Finished.value, TransformStatus.SubFinished.value ]: new_status = TransformStatus.Transforming.value elif input_collection_changed or output_collection_changed: if status in [ TransformStatus.ToCancel.value, TransformStatus.Cancelling.value, TransformStatus.Failed.value, TransformStatus.Cancelled.value ]: new_status = status else: new_status = TransformStatus.Transforming.value parameters = { 'status': new_status, 'transform_metadata': transform_metadata } orm_transforms.update_transform(transform_id=transform_id, parameters=parameters, session=session)
def update_request_with_transforms(request_id, parameters, new_transforms=None, update_transforms=None, new_messages=None, update_messages=None, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param new_transforms: list of transforms :param update_transforms: list of transforms """ if new_transforms: for tf in new_transforms: # tf_id = orm_transforms.add_transform(**tf, session=session) # original_work = tf['transform_metadata']['original_work'] # del tf['transform_metadata']['original_work'] workflow = tf['transform_metadata']['workflow'] del tf['transform_metadata']['workflow'] work = tf['transform_metadata']['work'] tf_copy = copy.deepcopy(tf) tf_id = orm_transforms.add_transform(**tf_copy, session=session) tf['transform_id'] = tf_id # work = tf['transform_metadata']['work'] # original_work.set_work_id(tf_id, transforming=True) # original_work.set_status(WorkStatus.New) work.set_work_id(tf_id, transforming=True) work.set_status(WorkStatus.New) workflow.refresh_works() collections = generate_collections(tf) for coll in collections: collection = coll['collection'] del coll['collection'] coll['transform_id'] = tf_id coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id # update transform to record the coll_id work.refresh_work() orm_transforms.update_transform( transform_id=tf_id, parameters={'transform_metadata': tf['transform_metadata']}, session=session) if update_transforms: for tr_id in update_transforms: orm_transforms.update_transform( transform_id=tr_id, parameters=update_transforms[tr_id], session=session) if new_messages: orm_messages.add_messages(new_messages, session=session) if update_messages: orm_messages.update_messages(update_messages, session=session) return orm_requests.update_request(request_id, parameters, session=session)
def add_transform_outputs(transform, transform_parameters, input_collections=None, output_collections=None, log_collections=None, update_input_collections=None, update_output_collections=None, update_log_collections=None, new_contents=None, update_contents=None, new_processing=None, update_processing=None, messages=None, update_messages=None, message_bulk_size=10000, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collections: The new input collections. :param output_collections: The new output collections. :param log_collections: The new log collections. :param update_input_collections: The updated input collections. :param update_output_collections: The updated output collections. :param update_log_collections: The updated log collections. :param new_contents: The new contents. :param update_contents: The updated contents. :param new_processing: The new processing. :param messages: Messages. :param message_bulk_size: The message bulk size. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ work = transform['transform_metadata']['work'] if input_collections: for coll in input_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if output_collections: for coll in output_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if log_collections: for coll in log_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if update_input_collections: update_input_colls = [ coll.collection for coll in update_input_collections ] orm_collections.update_collections(update_input_colls, session=session) if update_output_collections: update_output_colls = [ coll.collection for coll in update_output_collections ] orm_collections.update_collections(update_output_colls, session=session) if update_log_collections: update_log_colls = [coll.collection for coll in update_log_collections] orm_collections.update_collections(update_log_colls, session=session) if new_contents: orm_contents.add_contents(new_contents, session=session) if update_contents: orm_contents.update_contents(update_contents, session=session) processing_id = None if new_processing: # print(new_processing) processing_id = orm_processings.add_processing(**new_processing, session=session) if update_processing: for proc_id in update_processing: orm_processings.update_processing( processing_id=proc_id, parameters=update_processing[proc_id], session=session) if messages: if not type(messages) in [list, tuple]: messages = [messages] # for message in messages: # orm_messages.add_message(msg_type=message['msg_type'], # status=message['status'], # source=message['source'], # request_id=message['request_id'], # workload_id=message['workload_id'], # transform_id=message['transform_id'], # num_contents=message['num_contents'], # msg_content=message['msg_content'], # bulk_size=message_bulk_size, # session=session) orm_messages.add_messages(messages, session=session) if update_messages: orm_messages.update_messages(update_messages, session=session) if transform: if processing_id: # work.set_processing_id(new_processing, processing_id) work.set_processing_id( new_processing['processing_metadata']['processing'], processing_id) work.refresh_work() orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=transform_parameters, session=session)
def get_transforms_by_status(status, period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, with_messaging=False, session=None): """ Get transforms or raise a NoObject exception. :param status: Transform status or list of transform status. :param session: The database session in use. :param locking: Whether to lock retrieved items. :param to_json: return json format. :raises NoObject: If no transform is founded. :returns: list of transform. """ if with_messaging: transforms = get_transforms_with_messaging(locking=locking, bulk_size=bulk_size, session=session) if transforms: return transforms if locking: if bulk_size: # order by cannot work together with locking. So first select 2 * bulk_size without locking with order by. # then select with locking. tf_ids = orm_transforms.get_transforms_by_status( status=status, period=period, locking=locking, bulk_size=bulk_size * 2, locking_for_update=False, to_json=False, only_return_id=True, by_substatus=by_substatus, session=session) if tf_ids: transform2s = orm_transforms.get_transforms_by_status( status=status, period=period, locking=locking, bulk_size=None, locking_for_update=True, to_json=to_json, transform_ids=tf_ids, by_substatus=by_substatus, session=session) if transform2s: # reqs = req2s[:bulk_size] # order requests transforms = [] for tf_id in tf_ids: if len(transforms) >= bulk_size: break for tf in transform2s: if tf['transform_id'] == tf_id: transforms.append(tf) break # transforms = transforms[:bulk_size] else: transforms = [] else: transforms = [] else: transforms = orm_transforms.get_transforms_by_status( status=status, period=period, locking=locking, locking_for_update=locking, bulk_size=bulk_size, to_json=to_json, by_substatus=by_substatus, session=session) parameters = {'locking': TransformLocking.Locking} for transform in transforms: orm_transforms.update_transform( transform_id=transform['transform_id'], parameters=parameters, session=session) else: transforms = orm_transforms.get_transforms_by_status( status=status, period=period, locking=locking, bulk_size=bulk_size, to_json=to_json, by_substatus=by_substatus, session=session) return transforms