def get_contents_by_transform(transform_id, to_json=False, session=None): """ Get content or raise a NoObject exception. :param transform_id: transform id. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of contents. """ try: query = session.query(models.Content) query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENT_ID_UQ)", 'oracle') query = query.filter(models.Content.transform_id == transform_id) query = query.order_by(asc(models.Content.map_id)) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No record can be found with (transform_id=%s): %s' % (transform_id, error)) except Exception as error: raise error
def cancel_request(request_id=None, workload_id=None, session=None): """ cancel an request. :param request_id: The id of the request. :param workload_id: The workload_id of the request. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. """ try: if not request_id and workload_id: request_id = get_request_id_by_workload_id(workload_id) req_update = "update atlas_idds.requests set status=:status where request_id=:request_id" req_stmt = text(req_update) session.execute(req_stmt, { 'status': RequestStatus.Cancel.value, 'request_id': request_id }) except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('Request %s cannot be found: %s' % (request_id, error))
def get_processings_by_transform_id(transform_id=None, to_json=False, session=None): """ Get processings or raise a NoObject exception. :param tranform_id: Transform id. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processings. """ try: query = session.query(models.Processing)\ .filter_by(transform_id=transform_id) query = query.order_by(asc(models.Processing.processing_id)) ret = query.all() if not ret: return [] else: items = [] for t in ret: if to_json: items.append(t.to_dict_json()) else: items.append(t.to_dict()) return items except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Processings(transform_id: %s) cannot be found: %s' % (transform_id, error)) except Exception as error: raise error
def get_processings_by_status(status, period=None, locking=False, bulk_size=None, submitter=None, to_json=False, session=None): """ Get processing or raise a NoObject exception. :param status: Processing status of list of processing status. :param period: Time period in seconds. :param locking: Whether to retrieve only unlocked items. :param bulk_size: bulk size limitation. :param submitter: The submitter name. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processings. """ try: if not isinstance(status, (list, tuple)): status = [status] if len(status) == 1: status = [status[0], status[0]] query = session.query(models.Processing)\ .filter(models.Processing.status.in_(status))\ .filter(models.Processing.next_poll_at < datetime.datetime.utcnow()) if period: query = query.filter( models.Processing.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=period)) if locking: query = query.filter( models.Processing.locking == ProcessingLocking.Idle) if submitter: query = query.filter(models.Processing.submitter == submitter) query = query.order_by(asc(models.Processing.updated_at)) if bulk_size: query = query.limit(bulk_size) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No processing attached with status (%s): %s' % (status, error)) except Exception as error: raise error
def get_collections_by_status(status, relation_type=CollectionRelationType.Input, time_period=None, locking=False, bulk_size=None, session=None): """ Get collections by status, relation_type and time_period or raise a NoObject exception. :param status: The collection status. :param relation_type: The relation_type of the collection to the transform. :param time_period: time period in seconds since last update. :param locking: Wheter to retrieve unlocked files. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if status is None: raise exceptions.WrongParameterException("status should not be None") if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, CollectionStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.collections where status in :status""" params = {'status': status} if relation_type is not None: if isinstance(relation_type, CollectionRelationType): relation_type = relation_type.value select = select + " and relation_type=:relation_type" params['relation_type'] = relation_type if time_period is not None: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=time_period) if locking: select = select + " and locking=:locking" params['locking'] = CollectionLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by coll_id asc" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) collections = result.fetchall() ret = [] for collection in collections: collection = row2dict(collection) if collection['coll_type'] is not None: collection['coll_type'] = CollectionType(collection['coll_type']) if collection['relation_type'] is not None: collection['relation_type'] = CollectionRelationType(collection['relation_type']) if collection['status'] is not None: collection['status'] = CollectionStatus(collection['status']) if collection['locking'] is not None: collection['locking'] = CollectionLocking(collection['locking']) if collection['coll_metadata']: collection['coll_metadata'] = json.loads(collection['coll_metadata']) ret.append(collection) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No collections with status(%s), relation_type(%s), time_period(%s): %s' % (status, relation_type, time_period, error)) except Exception as error: raise error
def get_requests_by_status_type(status, request_type=None, time_period=None, locking=False, bulk_size=None, session=None): """ Get requests. :param status: list of status of the request data. :param request_type: The type of the request data. :param locking: Wheter to lock requests to avoid others get the same request. :param bulk_size: Size limitation per retrieve. :raises NoObject: If no request are founded. :returns: list of Request. """ try: if status is None: raise exceptions.WrongParameterException( "status should not be None") if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, RequestStatus): st = st.value new_status.append(st) status = new_status req_select = """select request_id, scope, name, requester, request_type, transform_tag, priority, status, locking, workload_id, created_at, updated_at, accessed_at, expired_at, errors, request_metadata, processing_metadata from atlas_idds.requests where status in :status """ req_params = {'status': status} if request_type is not None: req_select = req_select + " and request_type=:request_type" req_params['request_type'] = request_type if time_period is not None: req_select = req_select + " and updated_at < :updated_at" req_params['updated_at'] = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=time_period) if locking: req_select = req_select + " and locking=:locking" req_params['locking'] = RequestLocking.Idle.value if bulk_size: req_select = req_select + " and rownum < %s + 1 order by priority desc, request_id asc" % bulk_size else: req_select = req_select + " order by priority desc" req_stmt = text(req_select) req_stmt = req_stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(req_stmt, req_params) requests = result.fetchall() return [convert_request_to_dict(req) for req in requests] except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No requests with status: %s, request_type: %s, time_period: %s, locking: %s, %s' % (status, request_type, time_period, locking, error))
def get_processings_by_status(status, period=None, locking=False, bulk_size=None, session=None): """ Get processing or raise a NoObject exception. :param status: Processing status of list of processing status. :param period: Time period in seconds. :param locking: Whether to retrieve only unlocked items. :param session: The database session in use. :raises NoObject: If no processing is founded. :returns: Processings. """ try: if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, ProcessingStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.processings where status in :status""" params = {'status': status} if period: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=period) if locking: select = select + " and locking=:locking" params['locking'] = ProcessingLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by processing_id asc" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) processings = result.fetchall() if processings is None: raise exceptions.NoObject('Processing(status: %s, period: %s) cannot be found' % (status, period)) new_processings = [] for processing in processings: processing = row2dict(processing) if processing['granularity_type'] is not None: processing['granularity_type'] = GranularityType(processing['granularity_type']) if processing['status'] is not None: processing['status'] = ProcessingStatus(processing['status']) if processing['processing_metadata']: processing['processing_metadata'] = json.loads(processing['processing_metadata']) if processing['output_metadata']: processing['output_metadata'] = json.loads(processing['output_metadata']) new_processings.append(processing) return new_processings except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('No processing attached with status (%s): %s' % (status, error)) except Exception as error: raise error
def get_collections_by_status(status, relation_type=CollectionRelationType.Input, time_period=None, locking=False, bulk_size=None, to_json=False, session=None): """ Get collections by status, relation_type and time_period or raise a NoObject exception. :param status: The collection status. :param relation_type: The relation_type of the collection to the transform. :param time_period: time period in seconds since last update. :param locking: Wheter to retrieve unlocked files. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if not isinstance(status, (list, tuple)): status = [status] if len(status) == 1: status = [status[0], status[0]] query = session.query(models.Collection)\ .filter(models.Collection.status.in_(status))\ .filter(models.Collection.next_poll_at < datetime.datetime.utcnow()) if relation_type is not None: query = query.filter( models.Collection.relation_type == relation_type) if time_period: query = query.filter( models.Collection.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=time_period)) if locking: query = query.filter( models.Collection.locking == CollectionLocking.Idle) query = query.order_by(asc(models.Collection.updated_at)) if bulk_size: query = query.limit(bulk_size) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No collections with status(%s), relation_type(%s), time_period(%s): %s' % (status, relation_type, time_period, error)) except Exception as error: raise error
def get_content(content_id=None, coll_id=None, scope=None, name=None, content_type=None, min_id=0, max_id=0, to_json=False, session=None): """ Get contentor raise a NoObject exception. :param content_id: content id. :param coll_id: collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: Content. """ try: if content_id: query = session.query(models.Content)\ .filter(models.Content.content_id == content_id) else: if content_type in [ContentType.File, ContentType.File.value]: query = session.query(models.Content)\ .with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle')\ .filter(models.Content.coll_id == coll_id)\ .filter(models.Content.scope == scope)\ .filter(models.Content.name == name)\ .filter(models.Content.content_type == content_type) else: query = session.query(models.Content)\ .with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle')\ .filter(models.Content.coll_id == coll_id)\ .filter(models.Content.scope == scope)\ .filter(models.Content.name == name)\ .filter(models.Content.min_id == min_id)\ .filter(models.Content.max_id == max_id) if content_type: query = query.filter( models.Content.content_type == content_type) ret = query.first() if not ret: return None else: if to_json: return ret.to_dict_json() else: return ret.to_dict() except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s' % (content_id, coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_content(content_id=None, coll_id=None, scope=None, name=None, content_type=None, min_id=None, max_id=None, session=None): """ Get content or raise a NoObject exception. :param content_id: Content id. :param coll_id: Collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: Content. """ try: if not content_id: content_id = get_content_id(coll_id=coll_id, scope=scope, name=name, content_type=content_type, min_id=min_id, max_id=max_id, session=session) select = """select * from atlas_idds.contents where content_id=:content_id""" stmt = text(select) result = session.execute(stmt, {'content_id': content_id}) content = result.fetchone() if content is None: raise exceptions.NoObject( 'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found' % (content_id, coll_id, scope, name, content_type, min_id, max_id)) content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) return content except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s' % (content_id, coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_content_id(coll_id, scope, name, content_type=None, min_id=None, max_id=None, session=None): """ Get content id or raise a NoObject exception. :param coll_id: collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: Content id. """ try: if content_type is not None and isinstance(content_type, ContentType): content_type = content_type.value if content_type is None: select = """select content_id from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and min_id=:min_id and max_id=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'min_id': min_id, 'max_id': max_id }) else: if content_type == ContentType.File.value: select = """select content_id from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type }) else: select = """select content_id from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type and min_id=:min_id and max_id=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type, 'min_id': min_id, 'max_id': max_id }) content_id = result.fetchone() if content_id is None: raise exceptions.NoObject( 'content(coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found' % (coll_id, scope, name, content_type, min_id, max_id)) content_id = content_id[0] return content_id except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'content(coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s' % (coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_transforms_by_status(status, period=None, locking=False, bulk_size=None, session=None): """ Get transforms or raise a NoObject exception. :param status: Transform status or list of transform status. :param period: Time period in seconds. :param locking: Whether to retrieved unlocked items. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: list of transform. """ try: if not isinstance(status, (list, tuple)): status = [status] new_status = [] for st in status: if isinstance(st, TransformStatus): st = st.value new_status.append(st) status = new_status select = """select * from atlas_idds.transforms where status in :status""" params = {'status': status} if period: select = select + " and updated_at < :updated_at" params['updated_at'] = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=period) if locking: select = select + " and locking=:locking" params['locking'] = TransformLocking.Idle.value if bulk_size: select = select + " and rownum < %s + 1 order by transform_id" % bulk_size stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, params) transforms = result.fetchall() new_transforms = [] for transform in transforms: transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) new_transforms.append(transform) return new_transforms except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No transforms attached with status (%s): %s' % (status, error)) except Exception as error: raise error
def get_transforms_with_input_collection(transform_type, transform_tag, coll_scope, coll_name, session=None): """ Get transforms or raise a NoObject exception. :param transform_type: Transform type. :param transform_tag: Transform tag. :param coll_scope: The collection scope. :param coll_name: The collection name. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: Transform. """ try: if isinstance(transform_type, TransformType): transform_type = transform_type.value select = """select t.transform_id, t.transform_type, t.transform_tag, t.priority, t.status, t.locking, t.retries, t.created_at, t.started_at, t.finished_at, t.expired_at, t.transform_metadata from atlas_idds.transforms t join atlas_idds.collections c on t.transform_type=:transform_type and t.transform_tag=:transform_tag and t.transform_id=c.transform_id and c.scope=:scope and c.name=:name and c.relation_type=:relation_type """ stmt = text(select) result = session.execute( stmt, { 'transform_type': transform_type, 'transform_tag': transform_tag, 'relation_type': CollectionRelationType.Input.value, 'scope': coll_scope, 'name': coll_name }) transforms = result.fetchall() ret = [] for transform in transforms: transform = row2dict(transform) if transform['transform_type']: transform['transform_type'] = TransformType( transform['transform_type']) if transform['status'] is not None: transform['status'] = TransformStatus(transform['status']) if transform['locking'] is not None: transform['locking'] = TransformLocking(transform['locking']) if transform['transform_metadata']: transform['transform_metadata'] = json.loads( transform['transform_metadata']) ret.append(transform) return ret except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'Transform(transform_type: %s, transform_tag: %s, coll_scope: %s, coll_name: %s) cannot be found: %s' % (transform_type, transform_tag, coll_scope, coll_name, error)) except Exception as error: raise error
def get_collections(scope=None, name=None, request_id=None, workload_id=None, transform_id=None, relation_type=None, to_json=False, session=None): """ Get collections by request id or raise a NoObject exception. :param scope: collection scope. :param name: collection name, can be wildcard. :param request_id: The request id. :param workload_id: The workload id. :param transform_id: list of transform id related to this collection. :param relation_type: The relation type between this collection and the transform: Input, Ouput and Log. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no collections are founded. :returns: list of Collections. """ try: if transform_id and type(transform_id) not in (list, tuple): transform_id = [transform_id] query = session.query(models.Collection) if scope: query = query.filter(models.Collection.scope == scope) if name: query = query.filter( models.Collection.name.like(name.replace('*', '%'))) if request_id: query = query.filter(models.Collection.request_id == request_id) if workload_id: query = query.filter(models.Collection.workload_id == workload_id) if transform_id: query = query.filter( models.Collection.transform_id.in_(transform_id)) if relation_type is not None: query = query.filter( models.Collection.relation_type == relation_type) query = query.order_by(asc(models.Collection.updated_at)) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No collection with scope(%s), name(%s), transform_id(%s): %s, relation_type: %s' % (scope, name, transform_id, relation_type, error)) except Exception as error: raise error
def get_requests(request_id=None, workload_id=None, with_detail=False, with_metadata=False, with_processing=False, to_json=False, session=None): """ Get a request or raise a NoObject exception. :param workload_id: The workload id of the request. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no request is founded. :returns: Request. """ try: if not with_detail and not with_processing: if with_metadata: query = session.query(models.Request)\ .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle') if request_id: query = query.filter( models.Request.request_id == request_id) if workload_id: query = query.filter( models.Request.workload_id == workload_id) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: t_dict = t.to_dict_json() else: t_dict = t.to_dict() rets.append(t_dict) return rets else: query = session.query(models.Request.request_id, models.Request.scope, models.Request.name, models.Request.requester, models.Request.request_type, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, models.Request.status, models.Request.substatus, models.Request.locking, models.Request.created_at, models.Request.updated_at, models.Request.next_poll_at, models.Request.accessed_at, models.Request.expired_at, models.Request.errors)\ .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle') if request_id: query = query.filter( models.Request.request_id == request_id) if workload_id: query = query.filter( models.Request.workload_id == workload_id) tmp = query.all() rets = [] if tmp: for t in tmp: t2 = dict(zip(t.keys(), t)) rets.append(t2) return rets elif with_processing: subquery = session.query( models.Processing.processing_id, models.Processing.transform_id, models.Processing.status.label("processing_status"), models.Processing.created_at.label("processing_created_at"), models.Processing.updated_at.label("processing_updated_at"), models.Processing.finished_at.label("processing_finished_at")) subquery = subquery.subquery() if with_metadata: query = session.query( models.Request.request_id, models.Request.scope, models.Request.name, models.Request.requester, models.Request.request_type, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, models.Request.status, models.Request.substatus, models.Request.locking, models.Request.created_at, models.Request.updated_at, models.Request.next_poll_at, models.Request.accessed_at, models.Request.expired_at, models.Request.errors, models.Request._request_metadata.label('request_metadata'), models.Request._processing_metadata.label( 'processing_metadata'), models.Transform.transform_id, models.Transform.workload_id.label( "transform_workload_id"), models.Transform.status.label("transform_status"), subquery.c.processing_id, subquery.c.processing_status, subquery.c.processing_created_at, subquery.c.processing_updated_at, subquery.c.processing_finished_at) else: query = session.query( models.Request.request_id, models.Request.scope, models.Request.name, models.Request.requester, models.Request.request_type, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, models.Request.status, models.Request.substatus, models.Request.locking, models.Request.created_at, models.Request.updated_at, models.Request.next_poll_at, models.Request.accessed_at, models.Request.expired_at, models.Request.errors, models.Transform.transform_id, models.Transform.workload_id.label( "transform_workload_id"), models.Transform.status.label("transform_status"), subquery.c.processing_id, subquery.c.processing_status, subquery.c.processing_created_at, subquery.c.processing_updated_at, subquery.c.processing_finished_at) if request_id: query = query.filter(models.Request.request_id == request_id) if workload_id: query = query.filter(models.Request.workload_id == workload_id) query = query.outerjoin( models.Transform, and_(models.Request.request_id == models.Transform.request_id)) query = query.outerjoin( subquery, and_(subquery.c.transform_id == models.Transform.transform_id)) query = query.order_by(asc(models.Request.request_id)) tmp = query.all() rets = [] if tmp: for t in tmp: # t2 = dict(t) t2 = dict(zip(t.keys(), t)) if 'request_metadata' in t2 and t2[ 'request_metadata'] and 'workflow' in t2[ 'request_metadata']: workflow = t2['request_metadata']['workflow'] workflow_data = None if 'processing_metadata' in t2 and t2[ 'processing_metadata'] and 'workflow_data' in t2[ 'processing_metadata']: workflow_data = t2['processing_metadata'][ 'workflow_data'] if workflow is not None and workflow_data is not None: workflow.metadata = workflow_data t2['request_metadata']['workflow'] = workflow rets.append(t2) return rets elif with_detail: subquery1 = session.query( models.Collection.coll_id, models.Collection.transform_id, models.Collection.scope.label("input_coll_scope"), models.Collection.name.label("input_coll_name"), models.Collection.status.label("input_coll_status"), models.Collection.bytes.label("input_coll_bytes"), models.Collection.total_files.label("input_total_files"), models.Collection.processed_files.label( "input_processed_files"), models.Collection.processing_files.label( "input_processing_files")).filter( models.Collection.relation_type == 0) subquery1 = subquery1.subquery() subquery2 = session.query( models.Collection.coll_id, models.Collection.transform_id, models.Collection.scope.label("output_coll_scope"), models.Collection.name.label("output_coll_name"), models.Collection.status.label("output_coll_status"), models.Collection.bytes.label("output_coll_bytes"), models.Collection.total_files.label("output_total_files"), models.Collection.processed_files.label( "output_processed_files"), models.Collection.processing_files.label( "output_processing_files")).filter( models.Collection.relation_type == 1) subquery2 = subquery2.subquery() if with_metadata: query = session.query( models.Request.request_id, models.Request.scope, models.Request.name, models.Request.requester, models.Request.request_type, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, models.Request.status, models.Request.substatus, models.Request.locking, models.Request.created_at, models.Request.updated_at, models.Request.next_poll_at, models.Request.accessed_at, models.Request.expired_at, models.Request.errors, models.Request._request_metadata.label('request_metadata'), models.Request._processing_metadata.label( 'processing_metadata'), models.Transform.transform_id, models.Transform.transform_type, models.Transform.workload_id.label( "transform_workload_id"), models.Transform.status.label("transform_status"), models.Transform.created_at.label("transform_created_at"), models.Transform.updated_at.label("transform_updated_at"), models.Transform.finished_at.label( "transform_finished_at"), subquery1.c.input_coll_scope, subquery1.c.input_coll_name, subquery1.c.input_coll_status, subquery1.c.input_coll_bytes, subquery1.c.input_total_files, subquery1.c.input_processed_files, subquery1.c.input_processing_files, subquery2.c.output_coll_scope, subquery2.c.output_coll_name, subquery2.c.output_coll_status, subquery2.c.output_coll_bytes, subquery2.c.output_total_files, subquery2.c.output_processed_files, subquery2.c.output_processing_files) else: query = session.query( models.Request.request_id, models.Request.scope, models.Request.name, models.Request.requester, models.Request.request_type, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, models.Request.status, models.Request.substatus, models.Request.locking, models.Request.created_at, models.Request.updated_at, models.Request.next_poll_at, models.Request.accessed_at, models.Request.expired_at, models.Request.errors, models.Transform.transform_id, models.Transform.transform_type, models.Transform.workload_id.label( "transform_workload_id"), models.Transform.status.label("transform_status"), models.Transform.created_at.label("transform_created_at"), models.Transform.updated_at.label("transform_updated_at"), models.Transform.finished_at.label( "transform_finished_at"), subquery1.c.input_coll_scope, subquery1.c.input_coll_name, subquery1.c.input_coll_status, subquery1.c.input_coll_bytes, subquery1.c.input_total_files, subquery1.c.input_processed_files, subquery1.c.input_processing_files, subquery2.c.output_coll_scope, subquery2.c.output_coll_name, subquery2.c.output_coll_status, subquery2.c.output_coll_bytes, subquery2.c.output_total_files, subquery2.c.output_processed_files, subquery2.c.output_processing_files) if request_id: query = query.filter(models.Request.request_id == request_id) if workload_id: query = query.filter(models.Request.workload_id == workload_id) query = query.outerjoin( models.Transform, and_(models.Request.request_id == models.Transform.request_id)) query = query.outerjoin( subquery1, and_( subquery1.c.transform_id == models.Transform.transform_id)) query = query.outerjoin( subquery2, and_( subquery2.c.transform_id == models.Transform.transform_id)) query = query.order_by(asc(models.Request.request_id)) tmp = query.all() rets = [] if tmp: for t in tmp: # t2 = dict(t) t2 = dict(zip(t.keys(), t)) if 'request_metadata' in t2 and t2[ 'request_metadata'] and 'workflow' in t2[ 'request_metadata']: workflow = t2['request_metadata']['workflow'] workflow_data = None if 'processing_metadata' in t2 and t2[ 'processing_metadata'] and 'workflow_data' in t2[ 'processing_metadata']: workflow_data = t2['processing_metadata'][ 'workflow_data'] if workflow is not None and workflow_data is not None: workflow.metadata = workflow_data t2['request_metadata']['workflow'] = workflow rets.append(t2) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'request workload_id: %s cannot be found: %s' % (workload_id, error))
def get_match_contents(coll_id, scope, name, content_type=None, min_id=None, max_id=None, session=None): """ Get contents which matches the query or raise a NoObject exception. :param coll_id: collection id. :param scope: The scope of the request data. :param name: The name of the request data. :param min_id: The minimal id of the content. :param max_id: The maximal id of the content. :param content_type: The type of the content. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of Content ids. """ try: if content_type is not None and isinstance(content_type, ContentType): content_type = content_type.value if content_type is not None: if content_type == ContentType.File.value: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and content_type=:content_type and min_id<=:min_id and max_id>=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'content_type': content_type, 'min_id': min_id, 'max_id': max_id }) else: if min_id is None or max_id is None: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name""" stmt = text(select) result = session.execute(stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name=:name and min_id<=:min_id and max_id>=:max_id""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': name, 'min_id': min_id, 'max_id': max_id }) contents = result.fetchall() rets = [] for content in contents: content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) rets.append(content) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No match contents for (coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s): %s' % (coll_id, scope, name, content_type, min_id, max_id, error)) except Exception as error: raise error
def get_requests_by_status_type(status, request_type=None, time_period=None, locking=False, bulk_size=None, to_json=False, session=None): """ Get requests. :param status: list of status of the request data. :param request_type: The type of the request data. :param locking: Wheter to lock requests to avoid others get the same request. :param bulk_size: Size limitation per retrieve. :param to_json: return json format. :raises NoObject: If no request are founded. :returns: list of Request. """ try: if status is None: raise exceptions.WrongParameterException( "status should not be None") if not isinstance(status, (list, tuple)): status = [status] if len(status) == 1: status = [status[0], status[0]] query = session.query(models.Request)\ .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle')\ .filter(models.Request.status.in_(status))\ .filter(models.Request.next_poll_at < datetime.datetime.utcnow()) if request_type is not None: query = query.filter(models.Request.request_type == request_type) if time_period is not None: query = query.filter( models.Request.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=time_period)) if locking: query = query.filter(models.Request.locking == RequestLocking.Idle) query = query.order_by(asc(models.Request.updated_at))\ .order_by(desc(models.Request.priority)) if bulk_size: query = query.limit(bulk_size) tmp = query.all() rets = [] if tmp: for req in tmp: if to_json: rets.append(req.to_dict_json()) else: rets.append(req.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No requests with status: %s, request_type: %s, time_period: %s, locking: %s, %s' % (status, request_type, time_period, locking, error))
def get_contents(scope=None, name=None, coll_id=None, status=None, session=None): """ Get content or raise a NoObject exception. :param scope: The scope of the content data. :param name: The name of the content data. :param coll_id: Collection id. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of contents. """ try: if status is not None: if not isinstance(status, (tuple, list)): status = [status] new_status = [] for st in status: if isinstance(st, ContentStatus): new_status.append(st.value) else: new_status.append(st) status = new_status if scope and name: if coll_id: if status is not None: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name like :name and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': '%' + name + '%', 'status': status }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id and scope=:scope and name like :name""" stmt = text(select) result = session.execute( stmt, { 'coll_id': coll_id, 'scope': scope, 'name': '%' + name + '%' }) else: if status is not None: select = """select * from atlas_idds.contents where scope=:scope and name like :name and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, { 'scope': scope, 'name': '%' + name + '%', 'status': status }) else: select = """select * from atlas_idds.contents where scope=:scope and name like :name""" stmt = text(select) result = session.execute(stmt, { 'scope': scope, 'name': '%' + name + '%' }) else: if coll_id: if status is not None: select = """select * from atlas_idds.contents where coll_id=:coll_id and status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, { 'coll_id': coll_id, 'status': status }) else: select = """select * from atlas_idds.contents where coll_id=:coll_id""" stmt = text(select) result = session.execute(stmt, {'coll_id': coll_id}) else: if status is not None: select = """select * from atlas_idds.contents where status in :status""" stmt = text(select) stmt = stmt.bindparams(bindparam('status', expanding=True)) result = session.execute(stmt, {'status': status}) else: raise exceptions.WrongParameterException( "Both (scope:%s and name:%s) and coll_id:%s status:%s are not fully provided" % (scope, name, coll_id, status)) contents = result.fetchall() rets = [] for content in contents: content = row2dict(content) if content['content_type'] is not None: content['content_type'] = ContentType(content['content_type']) if content['status'] is not None: content['status'] = ContentStatus(content['status']) if content['content_metadata']: content['content_metadata'] = json.loads( content['content_metadata']) rets.append(content) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No record can be found with (scope=%s, name=%s, coll_id=%s): %s' % (scope, name, coll_id, error)) except Exception as error: raise error
def get_contents(scope=None, name=None, coll_id=None, status=None, to_json=False, session=None): """ Get content or raise a NoObject exception. :param scope: The scope of the content data. :param name: The name of the content data. :param coll_id: list of Collection ids. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no content is founded. :returns: list of contents. """ try: if status is not None: if not isinstance(status, (tuple, list)): status = [status] if len(status) == 1: status = [status[0], status[0]] if coll_id is not None: if not isinstance(coll_id, (tuple, list)): coll_id = [coll_id] if len(coll_id) == 1: coll_id = [coll_id[0], coll_id[0]] query = session.query(models.Content) query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle') if coll_id: query = query.filter(models.Content.coll_id.in_(coll_id)) if scope: query = query.filter(models.Content.scope == scope) if name: query = query.filter( models.Content.name.like(name.replace('*', '%'))) if status is not None: query = query.filter(models.Content.status.in_(status)) tmp = query.all() rets = [] if tmp: for t in tmp: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No record can be found with (scope=%s, name=%s, coll_id=%s): %s' % (scope, name, coll_id, error)) except Exception as error: raise error
def get_transforms_by_status(status, period=None, transform_ids=[], locking=False, locking_for_update=False, bulk_size=None, to_json=False, by_substatus=False, only_return_id=False, session=None): """ Get transforms or raise a NoObject exception. :param status: Transform status or list of transform status. :param period: Time period in seconds. :param locking: Whether to retrieved unlocked items. :param to_json: return json format. :param session: The database session in use. :raises NoObject: If no transform is founded. :returns: list of transform. """ try: if status: if not isinstance(status, (list, tuple)): status = [status] if len(status) == 1: status = [status[0], status[0]] if only_return_id: query = session.query(models.Transform.transform_id) else: query = session.query(models.Transform) if status: if by_substatus: query = query.filter(models.Transform.substatus.in_(status)) else: query = query.filter(models.Transform.status.in_(status)) query = query.filter( models.Transform.next_poll_at <= datetime.datetime.utcnow()) if transform_ids: query = query.filter( models.Transform.transform_id.in_(transform_ids)) # if period: # query = query.filter(models.Transform.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=period)) if locking: query = query.filter( models.Transform.locking == TransformLocking.Idle) if locking_for_update: query = query.with_for_update(skip_locked=True) else: query = query.order_by(asc(models.Transform.updated_at)).order_by( desc(models.Transform.priority)) if bulk_size: query = query.limit(bulk_size) tmp = query.all() rets = [] if tmp: for t in tmp: if only_return_id: rets.append(t[0]) else: if to_json: rets.append(t.to_dict_json()) else: rets.append(t.to_dict()) return rets except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'No transforms attached with status (%s): %s' % (status, error)) except Exception as error: raise error