def update_processing_with_collection_contents(updated_processing, new_processing=None, updated_collection=None, updated_files=None, new_files=None, coll_msg_content=None, file_msg_content=None, transform_updates=None, message_bulk_size=1000, session=None): """ Update processing with collection, contents, file messages and collection messages. :param updated_processing: dict with processing id and parameters. :param updated_collection: dict with collection id and parameters. :param updated_files: list of content files. :param coll_msg_content: message with collection info. :param file_msg_content: message with files info. """ if updated_files: orm_contents.update_contents(updated_files, session=session) if new_files: orm_contents.add_contents(contents=new_files, session=session) if file_msg_content: if not type(file_msg_content) in [list, tuple]: file_msg_content = [file_msg_content] for file_msg_con in file_msg_content: orm_messages.add_message(msg_type=file_msg_con['msg_type'], status=file_msg_con['status'], source=file_msg_con['source'], transform_id=file_msg_con['transform_id'], num_contents=file_msg_con['num_contents'], msg_content=file_msg_con['msg_content'], bulk_size=message_bulk_size, session=session) if updated_collection: orm_collections.update_collection( coll_id=updated_collection['coll_id'], parameters=updated_collection['parameters'], session=session) if coll_msg_content: orm_messages.add_message(msg_type=coll_msg_content['msg_type'], status=coll_msg_content['status'], source=coll_msg_content['source'], transform_id=coll_msg_content['transform_id'], num_contents=coll_msg_content['num_contents'], msg_content=coll_msg_content['msg_content'], session=session) if updated_processing: orm_processings.update_processing( processing_id=updated_processing['processing_id'], parameters=updated_processing['parameters'], session=session) if new_processing: orm_processings.add_processing(**new_processing, session=session) if transform_updates: orm_transforms.update_transform( transform_id=transform_updates['transform_id'], parameters=transform_updates['parameters'], session=session)
def get_collections_by_status(status, relation_type=CollectionRelationType.Input, time_period=None, locking=False, bulk_size=None, session=None): """ Get collections by status, relation_type and time_period or raise a NoObject exception. :param status: The collection status. :param relation_type: The relation_type of the collection to the transform. :param time_period: time period in seconds since last update. :param locking: Whether to retrieve unlocked files and lock them. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ colls = orm_collections.get_collections_by_status( status=status, relation_type=relation_type, bulk_size=bulk_size, time_period=time_period, locking=locking, session=session) if locking: parameters = {'locking': CollectionLocking.Locking} for coll in colls: orm_collections.update_collection(coll_id=coll['coll_id'], parameters=parameters, session=session) return colls
def update_collection(coll_id, parameters, msg=None, session=None): """ update a collection. :param coll_id: the collection id. :param parameters: A dictionary of parameters. :param msg: messages. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. """ orm_collections.update_collection(coll_id=coll_id, parameters=parameters, session=session) if msg: orm_messages.add_message(msg_type=msg['msg_type'], status=msg['status'], source=msg['source'], transform_id=msg['transform_id'], num_contents=msg['num_contents'], msg_content=msg['msg_content'], session=session)
def test_create_and_check_for_transform_collection_content_orm(self): """ Transform/Collection/Content (ORM): Test the creation, query, and cancel of a Transform/Collection/Content """ trans_properties = get_transform_properties() coll_properties = get_collection_properties() content_properties = get_content_properties() trans_id = add_transform(**trans_properties) coll_properties['transform_id'] = trans_id coll_id = add_collection(**coll_properties) coll_id_1 = get_collection_id( transform_id=trans_id, relation_type=coll_properties['relation_type']) assert_equal(coll_id, coll_id_1) coll = get_collection(coll_id=coll_id) for key in coll_properties: assert_equal(coll[key], coll_properties[key]) update_collection(coll_id, {'status': CollectionStatus.Closed}) coll = get_collection(coll_id=coll_id) assert_equal(coll['status'], CollectionStatus.Closed) content_properties['coll_id'] = coll_id content_id = add_content(**content_properties) content_1 = get_content(coll_id=coll_id, scope=content_properties['scope'], name=content_properties['name'], content_type=ContentType.File) content_id_1 = content_1['content_id'] assert_equal(content_id, content_id_1) content = get_content(content_id=content_id) update_content(content_id=content_id, parameters={'status': ContentStatus.Lost}) content = get_content(content_id=content_id) assert_equal(content['status'], ContentStatus.Lost) delete_content(content_id=content_id) c = get_content(content_id=content_id) assert_equal(c, None) delete_collection(coll_id=coll_id) coll = get_collection(coll_id=coll_id) assert_equal(coll, None) delete_transform(transform_id=trans_id) t = get_transform(transform_id=trans_id) assert_equal(t, None)
def update_collection(coll_id, parameters, session=None): """ update a collection. :param coll_id: the collection id. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. """ orm_collections.update_collection(coll_id=coll_id, parameters=parameters, session=session)
def add_transform_outputs(transform, input_collection, output_collection, input_contents, output_contents, processing, to_cancel_processing=None, session=None): """ For input contents, add corresponding output contents. :param transform: the transform. :param input_collection: The input collection. :param output_collection: The output collection. :param input_contents: The input contents. :param output_contents: The corresponding output contents. :param session: The database session in use. :raises DatabaseException: If there is a database error. """ if output_contents: orm_contents.add_contents(output_contents, session=session) if input_contents: update_input_contents = [] for input_content in input_contents: update_input_content = { 'content_id': input_content['content_id'], 'status': ContentStatus.Mapped, 'path': None } update_input_contents.append(update_input_content) if update_input_contents: orm_contents.update_contents(update_input_contents, with_content_id=True, session=session) if output_collection: # TODO, the status and new_files should be updated orm_collections.update_collection( output_collection['coll_id'], {'status': CollectionStatus.Processing}, session=session) if to_cancel_processing: to_cancel_params = {'status': ProcessingStatus.Cancel} for to_cancel_id in to_cancel_processing: orm_processings.update_processing(processing_id=to_cancel_id, parameters=to_cancel_params, session=session) processing_id = None if processing: processing_id = orm_processings.add_processing(**processing, session=session) if transform: if processing_id is not None: if not transform['transform_metadata']: transform['transform_metadata'] = { 'processing_id': processing_id } else: transform['transform_metadata'][ 'processing_id'] = processing_id parameters = { 'status': transform['status'], 'locking': transform['locking'], 'transform_metadata': transform['transform_metadata'] } orm_transforms.update_transform(transform_id=transform['transform_id'], parameters=parameters, session=session)