def get_contents_by_transform(transform_id, to_json=False, session=None):
"""
Get content or raise a NoObject exception.
:param transform_id: transform id.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: list of contents.
"""
try:
query = session.query(models.Content)
query = query.with_hint(models.Content,
"INDEX(CONTENTS CONTENT_ID_UQ)", 'oracle')
query = query.filter(models.Content.transform_id == transform_id)
query = query.order_by(asc(models.Content.map_id))
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No record can be found with (transform_id=%s): %s' %
(transform_id, error))
except Exception as error:
raise error
def cancel_request(request_id=None, workload_id=None, session=None):
"""
cancel an request.
:param request_id: The id of the request.
:param workload_id: The workload_id of the request.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
"""
try:
if not request_id and workload_id:
request_id = get_request_id_by_workload_id(workload_id)
req_update = "update atlas_idds.requests set status=:status where request_id=:request_id"
req_stmt = text(req_update)
session.execute(req_stmt, {
'status': RequestStatus.Cancel.value,
'request_id': request_id
})
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('Request %s cannot be found: %s' %
(request_id, error))
def get_processings_by_transform_id(transform_id=None,
to_json=False,
session=None):
"""
Get processings or raise a NoObject exception.
:param tranform_id: Transform id.
:param session: The database session in use.
:raises NoObject: If no processing is founded.
:returns: Processings.
"""
try:
query = session.query(models.Processing)\
.filter_by(transform_id=transform_id)
query = query.order_by(asc(models.Processing.processing_id))
ret = query.all()
if not ret:
return []
else:
items = []
for t in ret:
if to_json:
items.append(t.to_dict_json())
else:
items.append(t.to_dict())
return items
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'Processings(transform_id: %s) cannot be found: %s' %
(transform_id, error))
except Exception as error:
raise error
def get_processings_by_status(status,
period=None,
locking=False,
bulk_size=None,
submitter=None,
to_json=False,
session=None):
"""
Get processing or raise a NoObject exception.
:param status: Processing status of list of processing status.
:param period: Time period in seconds.
:param locking: Whether to retrieve only unlocked items.
:param bulk_size: bulk size limitation.
:param submitter: The submitter name.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no processing is founded.
:returns: Processings.
"""
try:
if not isinstance(status, (list, tuple)):
status = [status]
if len(status) == 1:
status = [status[0], status[0]]
query = session.query(models.Processing)\
.filter(models.Processing.status.in_(status))\
.filter(models.Processing.next_poll_at < datetime.datetime.utcnow())
if period:
query = query.filter(
models.Processing.updated_at < datetime.datetime.utcnow() -
datetime.timedelta(seconds=period))
if locking:
query = query.filter(
models.Processing.locking == ProcessingLocking.Idle)
if submitter:
query = query.filter(models.Processing.submitter == submitter)
query = query.order_by(asc(models.Processing.updated_at))
if bulk_size:
query = query.limit(bulk_size)
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No processing attached with status (%s): %s' % (status, error))
except Exception as error:
raise error
def get_collections_by_status(status, relation_type=CollectionRelationType.Input, time_period=None,
locking=False, bulk_size=None, session=None):
"""
Get collections by status, relation_type and time_period or raise a NoObject exception.
:param status: The collection status.
:param relation_type: The relation_type of the collection to the transform.
:param time_period: time period in seconds since last update.
:param locking: Wheter to retrieve unlocked files.
:param session: The database session in use.
:raises NoObject: If no collections are founded.
:returns: list of Collections.
"""
try:
if status is None:
raise exceptions.WrongParameterException("status should not be None")
if not isinstance(status, (list, tuple)):
status = [status]
new_status = []
for st in status:
if isinstance(st, CollectionStatus):
st = st.value
new_status.append(st)
status = new_status
select = """select * from atlas_idds.collections where status in :status"""
params = {'status': status}
if relation_type is not None:
if isinstance(relation_type, CollectionRelationType):
relation_type = relation_type.value
select = select + " and relation_type=:relation_type"
params['relation_type'] = relation_type
if time_period is not None:
select = select + " and updated_at < :updated_at"
params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=time_period)
if locking:
select = select + " and locking=:locking"
params['locking'] = CollectionLocking.Idle.value
if bulk_size:
select = select + " and rownum < %s + 1 order by coll_id asc" % bulk_size
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, params)
collections = result.fetchall()
ret = []
for collection in collections:
collection = row2dict(collection)
if collection['coll_type'] is not None:
collection['coll_type'] = CollectionType(collection['coll_type'])
if collection['relation_type'] is not None:
collection['relation_type'] = CollectionRelationType(collection['relation_type'])
if collection['status'] is not None:
collection['status'] = CollectionStatus(collection['status'])
if collection['locking'] is not None:
collection['locking'] = CollectionLocking(collection['locking'])
if collection['coll_metadata']:
collection['coll_metadata'] = json.loads(collection['coll_metadata'])
ret.append(collection)
return ret
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('No collections with status(%s), relation_type(%s), time_period(%s): %s' %
(status, relation_type, time_period, error))
except Exception as error:
raise error
def get_requests_by_status_type(status,
request_type=None,
time_period=None,
locking=False,
bulk_size=None,
session=None):
"""
Get requests.
:param status: list of status of the request data.
:param request_type: The type of the request data.
:param locking: Wheter to lock requests to avoid others get the same request.
:param bulk_size: Size limitation per retrieve.
:raises NoObject: If no request are founded.
:returns: list of Request.
"""
try:
if status is None:
raise exceptions.WrongParameterException(
"status should not be None")
if not isinstance(status, (list, tuple)):
status = [status]
new_status = []
for st in status:
if isinstance(st, RequestStatus):
st = st.value
new_status.append(st)
status = new_status
req_select = """select request_id, scope, name, requester, request_type, transform_tag, priority,
status, locking, workload_id, created_at, updated_at, accessed_at, expired_at,
errors, request_metadata, processing_metadata
from atlas_idds.requests where status in :status
"""
req_params = {'status': status}
if request_type is not None:
req_select = req_select + " and request_type=:request_type"
req_params['request_type'] = request_type
if time_period is not None:
req_select = req_select + " and updated_at < :updated_at"
req_params['updated_at'] = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=time_period)
if locking:
req_select = req_select + " and locking=:locking"
req_params['locking'] = RequestLocking.Idle.value
if bulk_size:
req_select = req_select + " and rownum < %s + 1 order by priority desc, request_id asc" % bulk_size
else:
req_select = req_select + " order by priority desc"
req_stmt = text(req_select)
req_stmt = req_stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(req_stmt, req_params)
requests = result.fetchall()
return [convert_request_to_dict(req) for req in requests]
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No requests with status: %s, request_type: %s, time_period: %s, locking: %s, %s'
% (status, request_type, time_period, locking, error))
def get_processings_by_status(status, period=None, locking=False, bulk_size=None, session=None):
"""
Get processing or raise a NoObject exception.
:param status: Processing status of list of processing status.
:param period: Time period in seconds.
:param locking: Whether to retrieve only unlocked items.
:param session: The database session in use.
:raises NoObject: If no processing is founded.
:returns: Processings.
"""
try:
if not isinstance(status, (list, tuple)):
status = [status]
new_status = []
for st in status:
if isinstance(st, ProcessingStatus):
st = st.value
new_status.append(st)
status = new_status
select = """select * from atlas_idds.processings where status in :status"""
params = {'status': status}
if period:
select = select + " and updated_at < :updated_at"
params['updated_at'] = datetime.datetime.utcnow() - datetime.timedelta(seconds=period)
if locking:
select = select + " and locking=:locking"
params['locking'] = ProcessingLocking.Idle.value
if bulk_size:
select = select + " and rownum < %s + 1 order by processing_id asc" % bulk_size
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, params)
processings = result.fetchall()
if processings is None:
raise exceptions.NoObject('Processing(status: %s, period: %s) cannot be found' %
(status, period))
new_processings = []
for processing in processings:
processing = row2dict(processing)
if processing['granularity_type'] is not None:
processing['granularity_type'] = GranularityType(processing['granularity_type'])
if processing['status'] is not None:
processing['status'] = ProcessingStatus(processing['status'])
if processing['processing_metadata']:
processing['processing_metadata'] = json.loads(processing['processing_metadata'])
if processing['output_metadata']:
processing['output_metadata'] = json.loads(processing['output_metadata'])
new_processings.append(processing)
return new_processings
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('No processing attached with status (%s): %s' % (status, error))
except Exception as error:
raise error
def get_collections_by_status(status,
relation_type=CollectionRelationType.Input,
time_period=None,
locking=False,
bulk_size=None,
to_json=False,
session=None):
"""
Get collections by status, relation_type and time_period or raise a NoObject exception.
:param status: The collection status.
:param relation_type: The relation_type of the collection to the transform.
:param time_period: time period in seconds since last update.
:param locking: Wheter to retrieve unlocked files.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no collections are founded.
:returns: list of Collections.
"""
try:
if not isinstance(status, (list, tuple)):
status = [status]
if len(status) == 1:
status = [status[0], status[0]]
query = session.query(models.Collection)\
.filter(models.Collection.status.in_(status))\
.filter(models.Collection.next_poll_at < datetime.datetime.utcnow())
if relation_type is not None:
query = query.filter(
models.Collection.relation_type == relation_type)
if time_period:
query = query.filter(
models.Collection.updated_at < datetime.datetime.utcnow() -
datetime.timedelta(seconds=time_period))
if locking:
query = query.filter(
models.Collection.locking == CollectionLocking.Idle)
query = query.order_by(asc(models.Collection.updated_at))
if bulk_size:
query = query.limit(bulk_size)
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No collections with status(%s), relation_type(%s), time_period(%s): %s'
% (status, relation_type, time_period, error))
except Exception as error:
raise error
def get_content(content_id=None,
coll_id=None,
scope=None,
name=None,
content_type=None,
min_id=0,
max_id=0,
to_json=False,
session=None):
"""
Get contentor raise a NoObject exception.
:param content_id: content id.
:param coll_id: collection id.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: Content.
"""
try:
if content_id:
query = session.query(models.Content)\
.filter(models.Content.content_id == content_id)
else:
if content_type in [ContentType.File, ContentType.File.value]:
query = session.query(models.Content)\
.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle')\
.filter(models.Content.coll_id == coll_id)\
.filter(models.Content.scope == scope)\
.filter(models.Content.name == name)\
.filter(models.Content.content_type == content_type)
else:
query = session.query(models.Content)\
.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle')\
.filter(models.Content.coll_id == coll_id)\
.filter(models.Content.scope == scope)\
.filter(models.Content.name == name)\
.filter(models.Content.min_id == min_id)\
.filter(models.Content.max_id == max_id)
if content_type:
query = query.filter(
models.Content.content_type == content_type)
ret = query.first()
if not ret:
return None
else:
if to_json:
return ret.to_dict_json()
else:
return ret.to_dict()
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s'
% (content_id, coll_id, scope, name, content_type, min_id, max_id,
error))
except Exception as error:
raise error
def get_content(content_id=None,
coll_id=None,
scope=None,
name=None,
content_type=None,
min_id=None,
max_id=None,
session=None):
"""
Get content or raise a NoObject exception.
:param content_id: Content id.
:param coll_id: Collection id.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: Content.
"""
try:
if not content_id:
content_id = get_content_id(coll_id=coll_id,
scope=scope,
name=name,
content_type=content_type,
min_id=min_id,
max_id=max_id,
session=session)
select = """select * from atlas_idds.contents where content_id=:content_id"""
stmt = text(select)
result = session.execute(stmt, {'content_id': content_id})
content = result.fetchone()
if content is None:
raise exceptions.NoObject(
'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found'
% (content_id, coll_id, scope, name, content_type, min_id,
max_id))
content = row2dict(content)
if content['content_type'] is not None:
content['content_type'] = ContentType(content['content_type'])
if content['status'] is not None:
content['status'] = ContentStatus(content['status'])
if content['content_metadata']:
content['content_metadata'] = json.loads(
content['content_metadata'])
return content
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'content(content_id: %s, coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s'
% (content_id, coll_id, scope, name, content_type, min_id, max_id,
error))
except Exception as error:
raise error
def get_content_id(coll_id,
scope,
name,
content_type=None,
min_id=None,
max_id=None,
session=None):
"""
Get content id or raise a NoObject exception.
:param coll_id: collection id.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: Content id.
"""
try:
if content_type is not None and isinstance(content_type, ContentType):
content_type = content_type.value
if content_type is None:
select = """select content_id from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and min_id=:min_id and max_id=:max_id"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'min_id': min_id,
'max_id': max_id
})
else:
if content_type == ContentType.File.value:
select = """select content_id from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and content_type=:content_type"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'content_type': content_type
})
else:
select = """select content_id from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and content_type=:content_type and min_id=:min_id and
max_id=:max_id"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'content_type': content_type,
'min_id': min_id,
'max_id': max_id
})
content_id = result.fetchone()
if content_id is None:
raise exceptions.NoObject(
'content(coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found'
% (coll_id, scope, name, content_type, min_id, max_id))
content_id = content_id[0]
return content_id
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'content(coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s) cannot be found: %s'
% (coll_id, scope, name, content_type, min_id, max_id, error))
except Exception as error:
raise error
def get_transforms_by_status(status,
period=None,
locking=False,
bulk_size=None,
session=None):
"""
Get transforms or raise a NoObject exception.
:param status: Transform status or list of transform status.
:param period: Time period in seconds.
:param locking: Whether to retrieved unlocked items.
:param session: The database session in use.
:raises NoObject: If no transform is founded.
:returns: list of transform.
"""
try:
if not isinstance(status, (list, tuple)):
status = [status]
new_status = []
for st in status:
if isinstance(st, TransformStatus):
st = st.value
new_status.append(st)
status = new_status
select = """select * from atlas_idds.transforms where status in :status"""
params = {'status': status}
if period:
select = select + " and updated_at < :updated_at"
params['updated_at'] = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=period)
if locking:
select = select + " and locking=:locking"
params['locking'] = TransformLocking.Idle.value
if bulk_size:
select = select + " and rownum < %s + 1 order by transform_id" % bulk_size
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, params)
transforms = result.fetchall()
new_transforms = []
for transform in transforms:
transform = row2dict(transform)
if transform['transform_type']:
transform['transform_type'] = TransformType(
transform['transform_type'])
if transform['status'] is not None:
transform['status'] = TransformStatus(transform['status'])
if transform['locking'] is not None:
transform['locking'] = TransformLocking(transform['locking'])
if transform['transform_metadata']:
transform['transform_metadata'] = json.loads(
transform['transform_metadata'])
new_transforms.append(transform)
return new_transforms
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No transforms attached with status (%s): %s' % (status, error))
except Exception as error:
raise error
def get_transforms_with_input_collection(transform_type,
transform_tag,
coll_scope,
coll_name,
session=None):
"""
Get transforms or raise a NoObject exception.
:param transform_type: Transform type.
:param transform_tag: Transform tag.
:param coll_scope: The collection scope.
:param coll_name: The collection name.
:param session: The database session in use.
:raises NoObject: If no transform is founded.
:returns: Transform.
"""
try:
if isinstance(transform_type, TransformType):
transform_type = transform_type.value
select = """select t.transform_id, t.transform_type, t.transform_tag, t.priority, t.status,
t.locking, t.retries, t.created_at, t.started_at, t.finished_at, t.expired_at,
t.transform_metadata
from atlas_idds.transforms t join atlas_idds.collections c
on t.transform_type=:transform_type and t.transform_tag=:transform_tag
and t.transform_id=c.transform_id and c.scope=:scope and c.name=:name
and c.relation_type=:relation_type
"""
stmt = text(select)
result = session.execute(
stmt, {
'transform_type': transform_type,
'transform_tag': transform_tag,
'relation_type': CollectionRelationType.Input.value,
'scope': coll_scope,
'name': coll_name
})
transforms = result.fetchall()
ret = []
for transform in transforms:
transform = row2dict(transform)
if transform['transform_type']:
transform['transform_type'] = TransformType(
transform['transform_type'])
if transform['status'] is not None:
transform['status'] = TransformStatus(transform['status'])
if transform['locking'] is not None:
transform['locking'] = TransformLocking(transform['locking'])
if transform['transform_metadata']:
transform['transform_metadata'] = json.loads(
transform['transform_metadata'])
ret.append(transform)
return ret
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'Transform(transform_type: %s, transform_tag: %s, coll_scope: %s, coll_name: %s) cannot be found: %s'
% (transform_type, transform_tag, coll_scope, coll_name, error))
except Exception as error:
raise error
def get_collections(scope=None,
name=None,
request_id=None,
workload_id=None,
transform_id=None,
relation_type=None,
to_json=False,
session=None):
"""
Get collections by request id or raise a NoObject exception.
:param scope: collection scope.
:param name: collection name, can be wildcard.
:param request_id: The request id.
:param workload_id: The workload id.
:param transform_id: list of transform id related to this collection.
:param relation_type: The relation type between this collection and the transform: Input, Ouput and Log.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no collections are founded.
:returns: list of Collections.
"""
try:
if transform_id and type(transform_id) not in (list, tuple):
transform_id = [transform_id]
query = session.query(models.Collection)
if scope:
query = query.filter(models.Collection.scope == scope)
if name:
query = query.filter(
models.Collection.name.like(name.replace('*', '%')))
if request_id:
query = query.filter(models.Collection.request_id == request_id)
if workload_id:
query = query.filter(models.Collection.workload_id == workload_id)
if transform_id:
query = query.filter(
models.Collection.transform_id.in_(transform_id))
if relation_type is not None:
query = query.filter(
models.Collection.relation_type == relation_type)
query = query.order_by(asc(models.Collection.updated_at))
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No collection with scope(%s), name(%s), transform_id(%s): %s, relation_type: %s'
% (scope, name, transform_id, relation_type, error))
except Exception as error:
raise error
def get_requests(request_id=None,
workload_id=None,
with_detail=False,
with_metadata=False,
with_processing=False,
to_json=False,
session=None):
"""
Get a request or raise a NoObject exception.
:param workload_id: The workload id of the request.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: Request.
"""
try:
if not with_detail and not with_processing:
if with_metadata:
query = session.query(models.Request)\
.with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle')
if request_id:
query = query.filter(
models.Request.request_id == request_id)
if workload_id:
query = query.filter(
models.Request.workload_id == workload_id)
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
t_dict = t.to_dict_json()
else:
t_dict = t.to_dict()
rets.append(t_dict)
return rets
else:
query = session.query(models.Request.request_id,
models.Request.scope,
models.Request.name,
models.Request.requester,
models.Request.request_type,
models.Request.transform_tag,
models.Request.workload_id,
models.Request.priority,
models.Request.status,
models.Request.substatus,
models.Request.locking,
models.Request.created_at,
models.Request.updated_at,
models.Request.next_poll_at,
models.Request.accessed_at,
models.Request.expired_at,
models.Request.errors)\
.with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle')
if request_id:
query = query.filter(
models.Request.request_id == request_id)
if workload_id:
query = query.filter(
models.Request.workload_id == workload_id)
tmp = query.all()
rets = []
if tmp:
for t in tmp:
t2 = dict(zip(t.keys(), t))
rets.append(t2)
return rets
elif with_processing:
subquery = session.query(
models.Processing.processing_id,
models.Processing.transform_id,
models.Processing.status.label("processing_status"),
models.Processing.created_at.label("processing_created_at"),
models.Processing.updated_at.label("processing_updated_at"),
models.Processing.finished_at.label("processing_finished_at"))
subquery = subquery.subquery()
if with_metadata:
query = session.query(
models.Request.request_id, models.Request.scope,
models.Request.name, models.Request.requester,
models.Request.request_type, models.Request.transform_tag,
models.Request.workload_id, models.Request.priority,
models.Request.status, models.Request.substatus,
models.Request.locking, models.Request.created_at,
models.Request.updated_at, models.Request.next_poll_at,
models.Request.accessed_at, models.Request.expired_at,
models.Request.errors,
models.Request._request_metadata.label('request_metadata'),
models.Request._processing_metadata.label(
'processing_metadata'), models.Transform.transform_id,
models.Transform.workload_id.label(
"transform_workload_id"),
models.Transform.status.label("transform_status"),
subquery.c.processing_id, subquery.c.processing_status,
subquery.c.processing_created_at,
subquery.c.processing_updated_at,
subquery.c.processing_finished_at)
else:
query = session.query(
models.Request.request_id, models.Request.scope,
models.Request.name, models.Request.requester,
models.Request.request_type, models.Request.transform_tag,
models.Request.workload_id, models.Request.priority,
models.Request.status, models.Request.substatus,
models.Request.locking, models.Request.created_at,
models.Request.updated_at, models.Request.next_poll_at,
models.Request.accessed_at, models.Request.expired_at,
models.Request.errors, models.Transform.transform_id,
models.Transform.workload_id.label(
"transform_workload_id"),
models.Transform.status.label("transform_status"),
subquery.c.processing_id, subquery.c.processing_status,
subquery.c.processing_created_at,
subquery.c.processing_updated_at,
subquery.c.processing_finished_at)
if request_id:
query = query.filter(models.Request.request_id == request_id)
if workload_id:
query = query.filter(models.Request.workload_id == workload_id)
query = query.outerjoin(
models.Transform,
and_(models.Request.request_id == models.Transform.request_id))
query = query.outerjoin(
subquery,
and_(subquery.c.transform_id == models.Transform.transform_id))
query = query.order_by(asc(models.Request.request_id))
tmp = query.all()
rets = []
if tmp:
for t in tmp:
# t2 = dict(t)
t2 = dict(zip(t.keys(), t))
if 'request_metadata' in t2 and t2[
'request_metadata'] and 'workflow' in t2[
'request_metadata']:
workflow = t2['request_metadata']['workflow']
workflow_data = None
if 'processing_metadata' in t2 and t2[
'processing_metadata'] and 'workflow_data' in t2[
'processing_metadata']:
workflow_data = t2['processing_metadata'][
'workflow_data']
if workflow is not None and workflow_data is not None:
workflow.metadata = workflow_data
t2['request_metadata']['workflow'] = workflow
rets.append(t2)
return rets
elif with_detail:
subquery1 = session.query(
models.Collection.coll_id, models.Collection.transform_id,
models.Collection.scope.label("input_coll_scope"),
models.Collection.name.label("input_coll_name"),
models.Collection.status.label("input_coll_status"),
models.Collection.bytes.label("input_coll_bytes"),
models.Collection.total_files.label("input_total_files"),
models.Collection.processed_files.label(
"input_processed_files"),
models.Collection.processing_files.label(
"input_processing_files")).filter(
models.Collection.relation_type == 0)
subquery1 = subquery1.subquery()
subquery2 = session.query(
models.Collection.coll_id, models.Collection.transform_id,
models.Collection.scope.label("output_coll_scope"),
models.Collection.name.label("output_coll_name"),
models.Collection.status.label("output_coll_status"),
models.Collection.bytes.label("output_coll_bytes"),
models.Collection.total_files.label("output_total_files"),
models.Collection.processed_files.label(
"output_processed_files"),
models.Collection.processing_files.label(
"output_processing_files")).filter(
models.Collection.relation_type == 1)
subquery2 = subquery2.subquery()
if with_metadata:
query = session.query(
models.Request.request_id, models.Request.scope,
models.Request.name, models.Request.requester,
models.Request.request_type, models.Request.transform_tag,
models.Request.workload_id, models.Request.priority,
models.Request.status, models.Request.substatus,
models.Request.locking, models.Request.created_at,
models.Request.updated_at, models.Request.next_poll_at,
models.Request.accessed_at, models.Request.expired_at,
models.Request.errors,
models.Request._request_metadata.label('request_metadata'),
models.Request._processing_metadata.label(
'processing_metadata'), models.Transform.transform_id,
models.Transform.transform_type,
models.Transform.workload_id.label(
"transform_workload_id"),
models.Transform.status.label("transform_status"),
models.Transform.created_at.label("transform_created_at"),
models.Transform.updated_at.label("transform_updated_at"),
models.Transform.finished_at.label(
"transform_finished_at"), subquery1.c.input_coll_scope,
subquery1.c.input_coll_name, subquery1.c.input_coll_status,
subquery1.c.input_coll_bytes,
subquery1.c.input_total_files,
subquery1.c.input_processed_files,
subquery1.c.input_processing_files,
subquery2.c.output_coll_scope,
subquery2.c.output_coll_name,
subquery2.c.output_coll_status,
subquery2.c.output_coll_bytes,
subquery2.c.output_total_files,
subquery2.c.output_processed_files,
subquery2.c.output_processing_files)
else:
query = session.query(
models.Request.request_id, models.Request.scope,
models.Request.name, models.Request.requester,
models.Request.request_type, models.Request.transform_tag,
models.Request.workload_id, models.Request.priority,
models.Request.status, models.Request.substatus,
models.Request.locking, models.Request.created_at,
models.Request.updated_at, models.Request.next_poll_at,
models.Request.accessed_at, models.Request.expired_at,
models.Request.errors, models.Transform.transform_id,
models.Transform.transform_type,
models.Transform.workload_id.label(
"transform_workload_id"),
models.Transform.status.label("transform_status"),
models.Transform.created_at.label("transform_created_at"),
models.Transform.updated_at.label("transform_updated_at"),
models.Transform.finished_at.label(
"transform_finished_at"), subquery1.c.input_coll_scope,
subquery1.c.input_coll_name, subquery1.c.input_coll_status,
subquery1.c.input_coll_bytes,
subquery1.c.input_total_files,
subquery1.c.input_processed_files,
subquery1.c.input_processing_files,
subquery2.c.output_coll_scope,
subquery2.c.output_coll_name,
subquery2.c.output_coll_status,
subquery2.c.output_coll_bytes,
subquery2.c.output_total_files,
subquery2.c.output_processed_files,
subquery2.c.output_processing_files)
if request_id:
query = query.filter(models.Request.request_id == request_id)
if workload_id:
query = query.filter(models.Request.workload_id == workload_id)
query = query.outerjoin(
models.Transform,
and_(models.Request.request_id == models.Transform.request_id))
query = query.outerjoin(
subquery1,
and_(
subquery1.c.transform_id == models.Transform.transform_id))
query = query.outerjoin(
subquery2,
and_(
subquery2.c.transform_id == models.Transform.transform_id))
query = query.order_by(asc(models.Request.request_id))
tmp = query.all()
rets = []
if tmp:
for t in tmp:
# t2 = dict(t)
t2 = dict(zip(t.keys(), t))
if 'request_metadata' in t2 and t2[
'request_metadata'] and 'workflow' in t2[
'request_metadata']:
workflow = t2['request_metadata']['workflow']
workflow_data = None
if 'processing_metadata' in t2 and t2[
'processing_metadata'] and 'workflow_data' in t2[
'processing_metadata']:
workflow_data = t2['processing_metadata'][
'workflow_data']
if workflow is not None and workflow_data is not None:
workflow.metadata = workflow_data
t2['request_metadata']['workflow'] = workflow
rets.append(t2)
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'request workload_id: %s cannot be found: %s' %
(workload_id, error))
def get_match_contents(coll_id,
scope,
name,
content_type=None,
min_id=None,
max_id=None,
session=None):
"""
Get contents which matches the query or raise a NoObject exception.
:param coll_id: collection id.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: list of Content ids.
"""
try:
if content_type is not None and isinstance(content_type, ContentType):
content_type = content_type.value
if content_type is not None:
if content_type == ContentType.File.value:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and content_type=:content_type"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'content_type': content_type
})
else:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and content_type=:content_type
and min_id<=:min_id and max_id>=:max_id"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'content_type': content_type,
'min_id': min_id,
'max_id': max_id
})
else:
if min_id is None or max_id is None:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name"""
stmt = text(select)
result = session.execute(stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name
})
else:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name=:name and min_id<=:min_id and max_id>=:max_id"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'min_id': min_id,
'max_id': max_id
})
contents = result.fetchall()
rets = []
for content in contents:
content = row2dict(content)
if content['content_type'] is not None:
content['content_type'] = ContentType(content['content_type'])
if content['status'] is not None:
content['status'] = ContentStatus(content['status'])
if content['content_metadata']:
content['content_metadata'] = json.loads(
content['content_metadata'])
rets.append(content)
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No match contents for (coll_id: %s, scope: %s, name: %s, content_type: %s, min_id: %s, max_id: %s): %s'
% (coll_id, scope, name, content_type, min_id, max_id, error))
except Exception as error:
raise error
def get_requests_by_status_type(status,
request_type=None,
time_period=None,
locking=False,
bulk_size=None,
to_json=False,
session=None):
"""
Get requests.
:param status: list of status of the request data.
:param request_type: The type of the request data.
:param locking: Wheter to lock requests to avoid others get the same request.
:param bulk_size: Size limitation per retrieve.
:param to_json: return json format.
:raises NoObject: If no request are founded.
:returns: list of Request.
"""
try:
if status is None:
raise exceptions.WrongParameterException(
"status should not be None")
if not isinstance(status, (list, tuple)):
status = [status]
if len(status) == 1:
status = [status[0], status[0]]
query = session.query(models.Request)\
.with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle')\
.filter(models.Request.status.in_(status))\
.filter(models.Request.next_poll_at < datetime.datetime.utcnow())
if request_type is not None:
query = query.filter(models.Request.request_type == request_type)
if time_period is not None:
query = query.filter(
models.Request.updated_at < datetime.datetime.utcnow() -
datetime.timedelta(seconds=time_period))
if locking:
query = query.filter(models.Request.locking == RequestLocking.Idle)
query = query.order_by(asc(models.Request.updated_at))\
.order_by(desc(models.Request.priority))
if bulk_size:
query = query.limit(bulk_size)
tmp = query.all()
rets = []
if tmp:
for req in tmp:
if to_json:
rets.append(req.to_dict_json())
else:
rets.append(req.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No requests with status: %s, request_type: %s, time_period: %s, locking: %s, %s'
% (status, request_type, time_period, locking, error))
def get_contents(scope=None,
name=None,
coll_id=None,
status=None,
session=None):
"""
Get content or raise a NoObject exception.
:param scope: The scope of the content data.
:param name: The name of the content data.
:param coll_id: Collection id.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: list of contents.
"""
try:
if status is not None:
if not isinstance(status, (tuple, list)):
status = [status]
new_status = []
for st in status:
if isinstance(st, ContentStatus):
new_status.append(st.value)
else:
new_status.append(st)
status = new_status
if scope and name:
if coll_id:
if status is not None:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name like :name and status in :status"""
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': '%' + name + '%',
'status': status
})
else:
select = """select * from atlas_idds.contents where coll_id=:coll_id and
scope=:scope and name like :name"""
stmt = text(select)
result = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': '%' + name + '%'
})
else:
if status is not None:
select = """select * from atlas_idds.contents where scope=:scope and name like :name and status in :status"""
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, {
'scope': scope,
'name': '%' + name + '%',
'status': status
})
else:
select = """select * from atlas_idds.contents where scope=:scope and name like :name"""
stmt = text(select)
result = session.execute(stmt, {
'scope': scope,
'name': '%' + name + '%'
})
else:
if coll_id:
if status is not None:
select = """select * from atlas_idds.contents where coll_id=:coll_id and status in :status"""
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, {
'coll_id': coll_id,
'status': status
})
else:
select = """select * from atlas_idds.contents where coll_id=:coll_id"""
stmt = text(select)
result = session.execute(stmt, {'coll_id': coll_id})
else:
if status is not None:
select = """select * from atlas_idds.contents where status in :status"""
stmt = text(select)
stmt = stmt.bindparams(bindparam('status', expanding=True))
result = session.execute(stmt, {'status': status})
else:
raise exceptions.WrongParameterException(
"Both (scope:%s and name:%s) and coll_id:%s status:%s are not fully provided"
% (scope, name, coll_id, status))
contents = result.fetchall()
rets = []
for content in contents:
content = row2dict(content)
if content['content_type'] is not None:
content['content_type'] = ContentType(content['content_type'])
if content['status'] is not None:
content['status'] = ContentStatus(content['status'])
if content['content_metadata']:
content['content_metadata'] = json.loads(
content['content_metadata'])
rets.append(content)
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No record can be found with (scope=%s, name=%s, coll_id=%s): %s' %
(scope, name, coll_id, error))
except Exception as error:
raise error
def get_contents(scope=None,
name=None,
coll_id=None,
status=None,
to_json=False,
session=None):
"""
Get content or raise a NoObject exception.
:param scope: The scope of the content data.
:param name: The name of the content data.
:param coll_id: list of Collection ids.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:returns: list of contents.
"""
try:
if status is not None:
if not isinstance(status, (tuple, list)):
status = [status]
if len(status) == 1:
status = [status[0], status[0]]
if coll_id is not None:
if not isinstance(coll_id, (tuple, list)):
coll_id = [coll_id]
if len(coll_id) == 1:
coll_id = [coll_id[0], coll_id[0]]
query = session.query(models.Content)
query = query.with_hint(models.Content,
"INDEX(CONTENTS CONTENTS_ID_NAME_IDX)",
'oracle')
if coll_id:
query = query.filter(models.Content.coll_id.in_(coll_id))
if scope:
query = query.filter(models.Content.scope == scope)
if name:
query = query.filter(
models.Content.name.like(name.replace('*', '%')))
if status is not None:
query = query.filter(models.Content.status.in_(status))
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No record can be found with (scope=%s, name=%s, coll_id=%s): %s' %
(scope, name, coll_id, error))
except Exception as error:
raise error
def get_transforms_by_status(status,
period=None,
transform_ids=[],
locking=False,
locking_for_update=False,
bulk_size=None,
to_json=False,
by_substatus=False,
only_return_id=False,
session=None):
"""
Get transforms or raise a NoObject exception.
:param status: Transform status or list of transform status.
:param period: Time period in seconds.
:param locking: Whether to retrieved unlocked items.
:param to_json: return json format.
:param session: The database session in use.
:raises NoObject: If no transform is founded.
:returns: list of transform.
"""
try:
if status:
if not isinstance(status, (list, tuple)):
status = [status]
if len(status) == 1:
status = [status[0], status[0]]
if only_return_id:
query = session.query(models.Transform.transform_id)
else:
query = session.query(models.Transform)
if status:
if by_substatus:
query = query.filter(models.Transform.substatus.in_(status))
else:
query = query.filter(models.Transform.status.in_(status))
query = query.filter(
models.Transform.next_poll_at <= datetime.datetime.utcnow())
if transform_ids:
query = query.filter(
models.Transform.transform_id.in_(transform_ids))
# if period:
# query = query.filter(models.Transform.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=period))
if locking:
query = query.filter(
models.Transform.locking == TransformLocking.Idle)
if locking_for_update:
query = query.with_for_update(skip_locked=True)
else:
query = query.order_by(asc(models.Transform.updated_at)).order_by(
desc(models.Transform.priority))
if bulk_size:
query = query.limit(bulk_size)
tmp = query.all()
rets = []
if tmp:
for t in tmp:
if only_return_id:
rets.append(t[0])
else:
if to_json:
rets.append(t.to_dict_json())
else:
rets.append(t.to_dict())
return rets
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'No transforms attached with status (%s): %s' % (status, error))
except Exception as error:
raise error
