def update_processing_with_collection_contents(updated_processing, new_processing=None, updated_collection=None, updated_files=None, new_files=None, coll_msg_content=None, file_msg_content=None, transform_updates=None, message_bulk_size=1000, session=None): """ Update processing with collection, contents, file messages and collection messages. :param updated_processing: dict with processing id and parameters. :param updated_collection: dict with collection id and parameters. :param updated_files: list of content files. :param coll_msg_content: message with collection info. :param file_msg_content: message with files info. """ if updated_files: orm_contents.update_contents(updated_files, session=session) if new_files: orm_contents.add_contents(contents=new_files, session=session) if file_msg_content: if not type(file_msg_content) in [list, tuple]: file_msg_content = [file_msg_content] for file_msg_con in file_msg_content: orm_messages.add_message(msg_type=file_msg_con['msg_type'], status=file_msg_con['status'], source=file_msg_con['source'], transform_id=file_msg_con['transform_id'], num_contents=file_msg_con['num_contents'], msg_content=file_msg_con['msg_content'], bulk_size=message_bulk_size, session=session) if updated_collection: orm_collections.update_collection( coll_id=updated_collection['coll_id'], parameters=updated_collection['parameters'], session=session) if coll_msg_content: orm_messages.add_message(msg_type=coll_msg_content['msg_type'], status=coll_msg_content['status'], source=coll_msg_content['source'], transform_id=coll_msg_content['transform_id'], num_contents=coll_msg_content['num_contents'], msg_content=coll_msg_content['msg_content'], session=session) if updated_processing: orm_processings.update_processing( processing_id=updated_processing['processing_id'], parameters=updated_processing['parameters'], session=session) if new_processing: orm_processings.add_processing(**new_processing, session=session) if transform_updates: orm_transforms.update_transform( transform_id=transform_updates['transform_id'], parameters=transform_updates['parameters'], session=session)
def test_insert_contents(coll_id, num_contents=1, session=None): # print("test_insert_contents, num_contents: %s" % num_contents) list_contents = [] for i in range(num_contents): content_properties = get_content_prop() content_properties['coll_id'] = coll_id list_contents.append(content_properties) add_contents(list_contents, bulk_size=num_contents, session=session)
def update_processing_contents(processing_update, content_updates, update_messages=None, new_contents=None, session=None): """ Update processing with contents. :param processing_update: dict with processing id and parameters. :param content_updates: list of content files. """ if content_updates: orm_contents.update_contents(content_updates, session=session) if new_contents: orm_contents.add_contents(new_contents, session=session) if processing_update: orm_processings.update_processing( processing_id=processing_update['processing_id'], parameters=processing_update['parameters'], session=session) if update_messages: orm_messages.update_messages(update_messages, session=session)
def add_contents(contents, bulk_size=1000, session=None): """ Add contents. :param contents: dict of contents. :param bulk_size: bulk per insert to db. :param session: session. :raises DuplicatedObject: If a collection with the same name exists. :raises DatabaseException: If there is a database error. :returns: content id. """ return orm_contents.add_contents(contents=contents, bulk_size=bulk_size, session=session)
def add_transform_outputs(transform, input_collections=None, output_collections=None, log_collections=None, update_input_collections=None, update_output_collections=None, update_log_collections=None, new_contents=None, update_contents=None, new_processing=None, messages=None, message_bulk_size=1000, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collections: The new input collections. :param output_collections: The new output collections. :param log_collections: The new log collections. :param update_input_collections: The updated input collections. :param update_output_collections: The updated output collections. :param update_log_collections: The updated log collections. :param new_contents: The new contents. :param update_contents: The updated contents. :param new_processing: The new processing. :param messages: Messages. :param message_bulk_size: The message bulk size. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ work = transform['transform_metadata']['work'] if input_collections: for coll in input_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if output_collections: for coll in output_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if log_collections: for coll in log_collections: coll_id = orm_collections.add_collection(**coll, session=session) work.set_collection_id(coll, coll_id) if update_input_collections: orm_collections.update_collections(update_input_collections, session=session) if update_output_collections: orm_collections.update_collections(update_output_collections, session=session) if update_log_collections: orm_collections.update_collections(update_log_collections, session=session) if new_contents: orm_contents.add_contents(new_contents, session=session) if update_contents: orm_contents.add_contents(update_contents, session=session) processing_id = None if new_processing: processing_id = orm_processings.add_processing(**new_processing, session=session) """ if output_contents: orm_contents.add_contents(output_contents, session=session) if messages: if not type(messages) in [list, tuple]: messages = [messages] for message in messages: orm_messages.add_message(msg_type=message['msg_type'], status=message['status'], source=message['source'], transform_id=message['transform_id'], num_contents=message['num_contents'], msg_content=message['msg_content'], bulk_size=message_bulk_size, session=session) if to_cancel_processing: to_cancel_params = {'status': ProcessingStatus.Cancel} for to_cancel_id in to_cancel_processing: orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session) processing_id = None if processing: processing_id = orm_processings.add_processing(**processing, session=session) """ if transform: """ if processing_id is not None: if not transform['transform_metadata']: transform['transform_metadata'] = {'processing_id': processing_id} else: transform['transform_metadata']['processing_id'] = processing_id """ if processing_id: work.set_processing_id(new_processing, processing_id) parameters = { 'status': transform['status'], 'locking': transform['locking'], 'transform_metadata': transform['transform_metadata'] } orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=parameters, session=session)
def add_transform_outputs(transform, input_collection, output_collection, input_contents, output_contents, processing, to_cancel_processing=None, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collection: The input collection. :param output_collection: The output collection. :param input_contents: The input contents. :param output_contents: The corresponding output contents. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ if output_contents: orm_contents.add_contents(output_contents, session=session) if input_contents: update_input_contents = [] for input_content in input_contents: update_input_content = { 'content_id': input_content['content_id'], 'status': ContentStatus.Mapped, 'path': None } update_input_contents.append(update_input_content) if update_input_contents: orm_contents.update_contents(update_input_contents, with_content_id=True, session=session) if output_collection: # TODO, the status and new_files should be updated orm_collections.update_collection( output_collection['coll_id'], {'status': CollectionStatus.Processing}, session=session) if to_cancel_processing: to_cancel_params = {'status': ProcessingStatus.Cancel} for to_cancel_id in to_cancel_processing: orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session) processing_id = None if processing: processing_id = orm_processings.add_processing(**processing, session=session) if transform: if processing_id is not None: if not transform['transform_metadata']: transform['transform_metadata'] = { 'processing_id': processing_id } else: transform['transform_metadata'][ 'processing_id'] = processing_id parameters = { 'status': transform['status'], 'locking': transform['locking'], 'transform_metadata': transform['transform_metadata'] } orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=parameters, session=session)
def add_transform_outputs(transform, transform_parameters, input_collections=None, output_collections=None, log_collections=None, update_input_collections=None, update_output_collections=None, update_log_collections=None, new_contents=None, update_contents=None, new_processing=None, update_processing=None, messages=None, update_messages=None, message_bulk_size=10000, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collections: The new input collections. :param output_collections: The new output collections. :param log_collections: The new log collections. :param update_input_collections: The updated input collections. :param update_output_collections: The updated output collections. :param update_log_collections: The updated log collections. :param new_contents: The new contents. :param update_contents: The updated contents. :param new_processing: The new processing. :param messages: Messages. :param message_bulk_size: The message bulk size. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ work = transform['transform_metadata']['work'] if input_collections: for coll in input_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if output_collections: for coll in output_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if log_collections: for coll in log_collections: collection = coll['collection'] del coll['collection'] coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id if update_input_collections: update_input_colls = [ coll.collection for coll in update_input_collections ] orm_collections.update_collections(update_input_colls, session=session) if update_output_collections: update_output_colls = [ coll.collection for coll in update_output_collections ] orm_collections.update_collections(update_output_colls, session=session) if update_log_collections: update_log_colls = [coll.collection for coll in update_log_collections] orm_collections.update_collections(update_log_colls, session=session) if new_contents: orm_contents.add_contents(new_contents, session=session) if update_contents: orm_contents.update_contents(update_contents, session=session) processing_id = None if new_processing: # print(new_processing) processing_id = orm_processings.add_processing(**new_processing, session=session) if update_processing: for proc_id in update_processing: orm_processings.update_processing( processing_id=proc_id, parameters=update_processing[proc_id], session=session) if messages: if not type(messages) in [list, tuple]: messages = [messages] # for message in messages: # orm_messages.add_message(msg_type=message['msg_type'], # status=message['status'], # source=message['source'], # request_id=message['request_id'], # workload_id=message['workload_id'], # transform_id=message['transform_id'], # num_contents=message['num_contents'], # msg_content=message['msg_content'], # bulk_size=message_bulk_size, # session=session) orm_messages.add_messages(messages, session=session) if update_messages: orm_messages.update_messages(update_messages, session=session) if transform: if processing_id: # work.set_processing_id(new_processing, processing_id) work.set_processing_id( new_processing['processing_metadata']['processing'], processing_id) work.refresh_work() orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=transform_parameters, session=session)