def get_collection(coll_id=None, transform_id=None, relation_type=None, session=None): """ Get a collection or raise a NoObject exception. :param coll_id: The id of the collection. :param transform_id: The transform id related to this collection. :param relation_type: The relation between this collection and its transform, such as Input, Output, Log and so on. :param session: The database session in use. :raises NoObject: If no request is founded. :returns: Collection. """ try: if coll_id: coll_select = """select * from atlas_idds.collections where coll_id=:coll_id """ stmt = text(coll_select) result = session.execute(stmt, {'coll_id': coll_id}) else: coll_select = """select * from atlas_idds.collections where transform_id=:transform_id and relation_type=:relation_type """ stmt = text(coll_select) result = session.execute(stmt, {'transform_id': transform_id, 'relation_type': relation_type}) collection = result.fetchone() if collection is None: raise exceptions.NoObject('collection(coll_id: %s, transform_id: %s, relation_type: %s) cannot be found' % (coll_id, transform_id, relation_type)) collection = row2dict(collection) if collection['coll_type'] is not None: collection['coll_type'] = CollectionType(collection['coll_type']) if collection['relation_type'] is not None: collection['relation_type'] = CollectionRelationType(collection['relation_type']) if collection['status'] is not None: collection['status'] = CollectionStatus(collection['status']) if collection['locking'] is not None: collection['locking'] = CollectionLocking(collection['locking']) if collection['coll_metadata']: collection['coll_metadata'] = json.loads(collection['coll_metadata']) return collection except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('collection(coll_id: %s, transform_id: %s, relation_type: %s) cannot be found: %s' % (coll_id, transform_id, relation_type, error)) except Exception as error: raise error
def get_collections(scope, name, transform_ids=None, session=None): """ Get collections by request id or raise a NoObject exception. :param scope: collection scope. :param name: collection name, can be wildcard. :param transform_id: list of transform id related to this collection. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if transform_ids: select = """select * from atlas_idds.collections where scope=:scope and name like :name and transform_id in :transform_ids""" stmt = text(select) stmt = stmt.bindparams(bindparam('transform_ids', expanding=True)) result = session.execute(stmt, {'scope': scope, 'name': '%' + name + '%', 'transform_ids': transform_ids}) else: select = """select * from atlas_idds.collections where scope=:scope and name like :name""" stmt = text(select) result = session.execute(stmt, {'scope': scope, 'name': '%' + name + '%'}) collections = result.fetchall() ret = [] for collection in collections: collection = row2dict(collection) if collection['coll_type'] is not None: collection['coll_type'] = CollectionType(collection['coll_type']) if collection['relation_type'] is not None: collection['relation_type'] = CollectionRelationType(collection['relation_type']) if collection['status'] is not None: collection['status'] = CollectionStatus(collection['status']) if collection['locking'] is not None: collection['locking'] = CollectionLocking(collection['locking']) if collection['coll_metadata']: collection['coll_metadata'] = json.loads(collection['coll_metadata']) ret.append(collection) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No collection with scope(%s), name(%s), transform_ids(%s): %s' % (scope, name, transform_ids, error)) except Exception as error: raise error
def convert_request_to_dict(request): request = row2dict(request) if request['errors']: request['errors'] = json.loads(request['errors']) if request['request_metadata']: request['request_metadata'] = json.loads(request['request_metadata']) if request['request_type'] is not None: request['request_type'] = RequestType(request['request_type']) if request['status'] is not None: request['status'] = RequestStatus(request['status']) if request['locking'] is not None: request['locking'] = RequestLocking(request['locking']) if request['processing_metadata']: request['processing_metadata'] = json.loads( request['processing_metadata']) return request
def get_processing(processing_id, session=None): """ Get processing or raise a NoObject exception. :param processing_id: Processing id. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processing. """ try: select = """select * from atlas_idds.processings where processing_id=:processing_id""" stmt = text(select) result = session.execute(stmt, {'processing_id': processing_id}) processing = result.fetchone() if processing is None: raise exceptions.NoObject( 'Processing(processing_id: %s) cannot be found' % (processing_id)) processing = row2dict(processing) if processing['granularity_type'] is not None: processing['granularity_type'] = GranularityType( processing['granularity_type']) if processing['status'] is not None: processing['status'] = ProcessingStatus(processing['status']) if processing['locking'] is not None: processing['locking'] = ProcessingLocking(processing['locking']) if processing['processing_metadata']: processing['processing_metadata'] = json.loads( processing['processing_metadata']) if processing['output_metadata']: processing['output_metadata'] = json.loads( processing['output_metadata']) return processing except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Processing(processing_id: %s) cannot be found: %s' % (processing_id, error)) except Exception as error: raise error
def get_processings_by_transform_id(transform_id=None, session=None): """ Get processings or raise a NoObject exception. :param tranform_id: Transform id. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processings. """ try: select = """select * from atlas_idds.processings where transform_id=:transform_id""" stmt = text(select) result = session.execute(stmt, {'transform_id': transform_id}) processings = result.fetchall() ret = [] for processing in processings: processing = row2dict(processing) if processing['granularity_type'] is not None: processing['granularity_type'] = GranularityType( processing['granularity_type']) if processing['status'] is not None: processing['status'] = ProcessingStatus(processing['status']) if processing['locking'] is not None: processing['locking'] = ProcessingLocking( processing['locking']) if processing['processing_metadata']: processing['processing_metadata'] = json.loads( processing['processing_metadata']) if processing['output_metadata']: processing['output_metadata'] = json.loads( processing['output_metadata']) ret.append(processing) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Processings(transform_id: %s) cannot be found: %s' % (transform_id, error)) except Exception as error: raise error
def get_collections_by_transform_ids(transform_ids=None, session=None): """ Get collections by transform id or raise a NoObject exception. :param transform_id: list of transform ids related to this collection. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if transform_ids: select = """select * from atlas_idds.collections where transform_id in :transform_ids""" stmt = text(select) stmt = stmt.bindparams(bindparam('transform_ids', expanding=True)) result = session.execute(stmt, {'transform_ids': transform_ids}) else: raise exceptions.WrongParamterException("Transform_ids are None.") collections = result.fetchall() ret = [] for collection in collections: collection = row2dict(collection) if collection['coll_type'] is not None: collection['coll_type'] = CollectionType(collection['coll_type']) if collection['relation_type'] is not None: collection['relation_type'] = CollectionRelationType(collection['relation_type']) if collection['status'] is not None: collection['status'] = CollectionStatus(collection['status']) if collection['locking'] is not None: collection['locking'] = CollectionLocking(collection['locking']) if collection['coll_metadata']: collection['coll_metadata'] = json.loads(collection['coll_metadata']) ret.append(collection) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No collections with transform_ids(%s): %s' % (transform_ids, error)) except Exception as error: raise error
def get_transform(transform_id, session=None): """ Get transform or raise a NoObject exception. :param transform_id: Transform id. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: Transform. """ try: select = """select * from atlas_idds.transforms where transform_id=:transform_id""" stmt = text(select) result = session.execute(stmt, {'transform_id': transform_id}) transform = result.fetchone() if transform is None: raise exceptions.NoObject( 'Transform(transform_id: %s) cannot be found' % (transform_id)) transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) return transform except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Transform(transform_id: %s) cannot be found: %s' % (transform_id, error)) except Exception as error: raise error
def get_transforms(request_id, session=None): """ Get transforms or raise a NoObject exception. :param request_id: Request id. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: list of transform. """ try: select = """select * from atlas_idds.transforms where transform_id in (select transform_id from atlas_idds.req2transforms where request_id=:request_id) """ stmt = text(select) result = session.execute(stmt, {'request_id': request_id}) transforms = result.fetchall() new_transforms = [] for transform in transforms: transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) new_transforms.append(transform) return new_transforms except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No transforms attached with request id (%s): %s' % (request_id, error)) except Exception as error: raise error
def get_collections_by_status(status, relation_type=CollectionRelationType.Input, time_period=None, locking=False, bulk_size=None, session=None): """ Get collections by status, relation_type and time_period or raise a NoObject exception. :param status: The collection status. :param relation_type: The relation_type of the collection to the transform. :param time_period: time period in seconds since last update. :param locking: Wheter to retrieve unlocked files. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if status is None: raise exceptions.WrongParameterException("status should not be None") if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, CollectionStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.collections where status in :status""" params = {'status': status} if relation_type is not None: if isinstance(relation_type, CollectionRelationType): relation_type = relation_type.value select = select + " and relation_type=:relation_type" params['relation_type'] = relation_type if time_period is not None: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=time_period) if locking: select = select + " and locking=:locking" params['locking'] = CollectionLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by coll_id asc" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) collections = result.fetchall() ret = [] for collection in collections: collection = row2dict(collection) if collection['coll_type'] is not None: collection['coll_type'] = CollectionType(collection['coll_type']) if collection['relation_type'] is not None: collection['relation_type'] = CollectionRelationType(collection['relation_type']) if collection['status'] is not None: collection['status'] = CollectionStatus(collection['status']) if collection['locking'] is not None: collection['locking'] = CollectionLocking(collection['locking']) if collection['coll_metadata']: collection['coll_metadata'] = json.loads(collection['coll_metadata']) ret.append(collection) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No collections with status(%s), relation_type(%s), time_period(%s): %s' % (status, relation_type, time_period, error)) except Exception as error: raise error
def get_processings_by_status(status, period=None, locking=False, bulk_size=None, session=None): """ Get processing or raise a NoObject exception. :param status: Processing status of list of processing status. :param period: Time period in seconds. :param locking: Whether to retrieve only unlocked items. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processings. """ try: if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, ProcessingStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.processings where status in :status""" params = {'status': status} if period: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=period) if locking: select = select + " and locking=:locking" params['locking'] = ProcessingLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by processing_id asc" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) processings = result.fetchall() if processings is None: raise exceptions.NoObject('Processing(status: %s, period: %s) cannot be found' % (status, period)) new_processings = [] for processing in processings: processing = row2dict(processing) if processing['granularity_type'] is not None: processing['granularity_type'] = GranularityType(processing['granularity_type']) if processing['status'] is not None: processing['status'] = ProcessingStatus(processing['status']) if processing['processing_metadata']: processing['processing_metadata'] = json.loads(processing['processing_metadata']) if processing['output_metadata']: processing['output_metadata'] = json.loads(processing['output_metadata']) new_processings.append(processing) return new_processings except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No processing attached with status (%s): %s' % (status, error)) except Exception as error: raise error
def get_contents(scope=None, name=None, coll_id=None, status=None, session=None): """ Get content or raise a NoObject exception. :param scope: The scope of the content data. :param name: The name of the content data. :param coll_id: Collection id. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of contents. """ try: if status is not None: if not isinstance(status, (tuple, list)): status = [status] new_status = [] for st in status: if isinstance(st, ContentStatus): new_status.append(st.value) else: new_status.append(st) status = new_status if scope and name: if coll_id: if status is not None: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name like :name and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': '%' + name + '%', 'status': status }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name like :name""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': '%' + name + '%' }) else: if status is not None: select = """select * from atlas_idds.contents where scope=:scope and name like :name and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, { 'scope': scope, 'name': '%' + name + '%', 'status': status }) else: select = """select * from atlas_idds.contents where scope=:scope and name like :name""" stmt = text(select) result = session.execute(stmt, { 'scope': scope, 'name': '%' + name + '%' }) else: if coll_id: if status is not None: select = """select * from atlas_idds.contents where coll_id=:coll_id and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, { 'coll_id': coll_id, 'status': status }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id""" stmt = text(select) result = session.execute(stmt, {'coll_id': coll_id}) else: if status is not None: select = """select * from atlas_idds.contents where status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, {'status': status}) else: raise exceptions.WrongParameterException( "Both (scope:%s and name:%s) and coll_id:%s status:%s are not fully provided" % (scope, name, coll_id, status)) contents = result.fetchall() rets = [] for content in contents: content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) rets.append(content) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No record can be found with (scope=%s, name=%s, coll_id=%s): %s' % (scope, name, coll_id, error)) except Exception as error: raise error
def get_match_contents(coll_id, scope, name, content_type=None, min_id=None, max_id=None, session=None): """ Get contents which matches the query or raise a NoObject exception. :param coll_id: collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of Content ids. """ try: if content_type is not None and isinstance(content_type, ContentType): content_type = content_type.value if content_type is not None: if content_type == ContentType.File.value: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type and min_id<=:min_id and max_id>=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type, 'min_id': min_id, 'max_id': max_id }) else: if min_id is None or max_id is None: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name""" stmt = text(select) result = session.execute(stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and min_id<=:min_id and max_id>=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'min_id': min_id, 'max_id': max_id }) contents = result.fetchall() rets = [] for content in contents: content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) rets.append(content) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No match contents for (coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s): %s' % (coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_content(content_id=None, coll_id=None, scope=None, name=None, content_type=None, min_id=None, max_id=None, session=None): """ Get content or raise a NoObject exception. :param content_id: Content id. :param coll_id: Collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: Content. """ try: if not content_id: content_id = get_content_id(coll_id=coll_id, scope=scope, name=name, content_type=content_type, min_id=min_id, max_id=max_id, session=session) select = """select * from atlas_idds.contents where content_id=:content_id""" stmt = text(select) result = session.execute(stmt, {'content_id': content_id}) content = result.fetchone() if content is None: raise exceptions.NoObject( 'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found' % (content_id, coll_id, scope, name, content_type, min_id, max_id)) content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) return content except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s' % (content_id, coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_transforms_by_status(status, period=None, locking=False, bulk_size=None, session=None): """ Get transforms or raise a NoObject exception. :param status: Transform status or list of transform status. :param period: Time period in seconds. :param locking: Whether to retrieved unlocked items. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: list of transform. """ try: if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, TransformStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.transforms where status in :status""" params = {'status': status} if period: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=period) if locking: select = select + " and locking=:locking" params['locking'] = TransformLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by transform_id" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) transforms = result.fetchall() new_transforms = [] for transform in transforms: transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) new_transforms.append(transform) return new_transforms except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No transforms attached with status (%s): %s' % (status, error)) except Exception as error: raise error
def get_transforms_with_input_collection(transform_type, transform_tag, coll_scope, coll_name, session=None): """ Get transforms or raise a NoObject exception. :param transform_type: Transform type. :param transform_tag: Transform tag. :param coll_scope: The collection scope. :param coll_name: The collection name. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: Transform. """ try: if isinstance(transform_type, TransformType): transform_type = transform_type.value select = """select t.transform_id, t.transform_type, t.transform_tag, t.priority, t.status, t.locking, t.retries, t.created_at, t.started_at, t.finished_at, t.expired_at, t.transform_metadata from atlas_idds.transforms t join atlas_idds.collections c on t.transform_type=:transform_type and t.transform_tag=:transform_tag and t.transform_id=c.transform_id and c.scope=:scope and c.name=:name and c.relation_type=:relation_type """ stmt = text(select) result = session.execute( stmt, { 'transform_type': transform_type, 'transform_tag': transform_tag, 'relation_type': CollectionRelationType.Input.value, 'scope': coll_scope, 'name': coll_name }) transforms = result.fetchall() ret = [] for transform in transforms: transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) ret.append(transform) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Transform(transform_type: %s, transform_tag: %s, coll_scope: %s, coll_name: %s) cannot be found: %s' % (transform_type, transform_tag, coll_scope, coll_name, error)) except Exception as error: raise error