def update_messages(messages, bulk_size=1000, session=None):
try:
session.bulk_update_mappings(models.Message, messages)
except TypeError as e:
raise exceptions.DatabaseException('Invalid JSON for msg_content: %s' %
str(e))
except DatabaseError as e:
if re.match('.*ORA-12899.*', e.args[0]) \
or re.match('.*1406.*', e.args[0]):
raise exceptions.DatabaseException(
'Could not persist message, msg_content too large: %s' %
str(e))
else:
raise exceptions.DatabaseException(
'Could not persist message: %s' % str(e))
def add_processing(request_id, workload_id, transform_id, status=ProcessingStatus.New,
locking=ProcessingLocking.Idle, submitter=None, substatus=ProcessingStatus.New,
granularity=None, granularity_type=GranularityType.File, expired_at=None, processing_metadata=None,
output_metadata=None, session=None):
"""
Add a processing.
:param request_id: The request id.
:param workload_id: The workload id.
:param transform_id: Transform id.
:param status: processing status.
:param locking: processing locking.
:param submitter: submitter name.
:param granularity: Granularity size.
:param granularity_type: Granularity type.
:param expired_at: The datetime when it expires.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If a processing with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: processing id.
"""
try:
new_processing = create_processing(request_id=request_id, workload_id=workload_id, transform_id=transform_id,
status=status, substatus=substatus, locking=locking, submitter=submitter,
granularity=granularity, granularity_type=granularity_type, expired_at=expired_at,
processing_metadata=processing_metadata, output_metadata=output_metadata)
new_processing.save(session=session)
proc_id = new_processing.processing_id
return proc_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Processing already exists!: %s' % (error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def retrieve_messages(bulk_size=1000, msg_type=None, status=None, source=None, session=None):
"""
Retrieve up to $bulk messages.
:param bulk: Number of messages as an integer.
:param msg_type: Return only specified msg_type.
:param status: The status about the message
:param source: The source where the message is from.
:param session: The database session.
:returns messages: List of dictionaries
"""
messages = []
try:
query = session.query(models.Message)
if msg_type is not None:
query = query.filter_by(msg_type=msg_type)
if status is not None:
query = query.filter_by(status=status)
if source is not None:
query = query.filter_by(source=source)
if bulk_size:
query = query.order_by(models.Message.created_at).limit(bulk_size)
# query = query.with_for_update(nowait=True)
tmp = query.all()
if tmp:
for t in tmp:
messages.append(t.to_dict())
return messages
except IntegrityError as e:
raise exceptions.DatabaseException(e.args)
def add_workprogresses(workprogresses, bulk_size=1000, session=None):
"""
Add workprogresses.
:param workprogresses: dict of workprogress.
:param session: session.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: workprogress ids.
"""
sub_params = [
workprogresses[i:i + bulk_size]
for i in range(0, len(workprogresses), bulk_size)
]
try:
for sub_param in sub_params:
session.bulk_insert_mappings(models.Workprogress, sub_param)
wp_ids = [None for _ in range(len(workprogresses))]
return wp_ids
except IntegrityError as error:
raise exceptions.DuplicatedObject('Duplicated objects: %s' % (error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_request(scope=None,
name=None,
requester=None,
request_type=None,
username=None,
userdn=None,
transform_tag=None,
status=RequestStatus.New,
locking=RequestLocking.Idle,
priority=0,
lifetime=None,
workload_id=None,
request_metadata=None,
processing_metadata=None,
session=None):
"""
Add a request.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param requestr: The requester, such as panda, user and so on.
:param request_type: The type of the request, such as ESS, DAOD.
:param transform_tag: Transform tag, such as ATLAS AMI tag.
:param status: The request status as integer.
:param locking: The request locking as integer.
:param priority: The priority as integer.
:param lifetime: The life time as umber of days.
:param workload_id: The external workload id.
:param request_metadata: The metadata as json.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If an request with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: request id.
"""
try:
new_request = create_request(scope=scope,
name=name,
requester=requester,
request_type=request_type,
username=username,
userdn=userdn,
transform_tag=transform_tag,
status=status,
locking=locking,
priority=priority,
workload_id=workload_id,
lifetime=lifetime,
request_metadata=request_metadata,
processing_metadata=processing_metadata)
new_request.save(session=session)
request_id = new_request.request_id
return request_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Request %s:%s already exists!: %s' %
(scope, name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_transform(request_id,
workload_id,
transform_type,
transform_tag=None,
priority=0,
status=TransformStatus.New,
substatus=TransformStatus.New,
locking=TransformLocking.Idle,
retries=0,
expired_at=None,
transform_metadata=None,
workprogress_id=None,
session=None):
"""
Add a transform.
:param request_id: The request id.
:param workload_id: The workload id.
:param transform_type: Transform type.
:param transform_tag: Transform tag.
:param priority: priority.
:param status: Transform status.
:param locking: Transform locking.
:param retries: The number of retries.
:param expired_at: The datetime when it expires.
:param transform_metadata: The metadata as json.
:raises DuplicatedObject: If a transform with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: transform id.
"""
try:
new_transform = create_transform(request_id=request_id,
workload_id=workload_id,
transform_type=transform_type,
transform_tag=transform_tag,
priority=priority,
status=status,
substatus=substatus,
locking=locking,
retries=retries,
expired_at=expired_at,
transform_metadata=transform_metadata)
new_transform.save(session=session)
transform_id = new_transform.transform_id
if workprogress_id:
new_wp2transform = models.Workprogress2transform(
workprogress_id=workprogress_id, transform_id=transform_id)
new_wp2transform.save(session=session)
return transform_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Transform already exists!: %s' %
(error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_collection(scope,
name,
coll_type=CollectionType.Dataset,
transform_id=None,
relation_type=CollectionRelationType.Input,
bytes=0,
status=CollectionStatus.New,
locking=CollectionLocking.Idle,
total_files=0,
retries=0,
expired_at=None,
coll_metadata=None,
session=None):
"""
Add a collection.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param coll_type: The type of dataset as dataset or container.
:param transform_id: The transform id related to this collection.
:param relation_type: The relation between this collection and its transform,
such as Input, Output, Log and so on.
:param size: The size of the collection.
:param status: The status.
:param locking: The locking.
:param total_files: Number of total files.
:param retries: Number of retries.
:param expired_at: The datetime when it expires.
:param coll_metadata: The metadata as json.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection id.
"""
try:
new_coll = create_collection(scope=scope,
name=name,
coll_type=coll_type,
transform_id=transform_id,
relation_type=relation_type,
bytes=bytes,
status=status,
locking=locking,
total_files=total_files,
retries=retries,
expired_at=expired_at,
coll_metadata=coll_metadata)
new_coll.save(session=session)
coll_id = new_coll.coll_id
return coll_id
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'Collection scope:name(%s:%s) with transform_id(%s) already exists!: %s'
% (scope, name, transform_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_message(msg_type, status, source, transform_id, num_contents, msg_content, bulk_size=None, session=None):
"""
Add a message to be submitted asynchronously to a message broker.
:param msg_type: The type of the msg as a number, e.g., finished_stagein.
:param status: The status about the message
:param source: The source where the message is from.
:param transform_id: The transform id.
:param num_contents: Number of items in msg_content.
:param msg_content: The message msg_content as JSON.
:param session: The database session.
"""
try:
num_contents_list = []
msg_content_list = []
if bulk_size and num_contents > bulk_size:
if 'files' in msg_content:
files = msg_content['files']
chunks = [files[i:i + bulk_size] for i in range(0, len(files), bulk_size)]
for chunk in chunks:
new_msg_content = copy.deepcopy(msg_content)
new_msg_content['files'] = chunk
new_num_contents = len(chunk)
num_contents_list.append(new_num_contents)
msg_content_list.append(new_msg_content)
else:
num_contents_list.append(num_contents)
msg_content_list.append(msg_content)
for msg_content, num_contents in zip(msg_content_list, num_contents_list):
new_message = models.Message(msg_type=msg_type, status=status, transform_id=transform_id,
source=source, num_contents=num_contents,
locking=0, msg_content=msg_content)
new_message.save(session=session)
except TypeError as e:
raise exceptions.DatabaseException('Invalid JSON for msg_content: %s' % str(e))
except DatabaseError as e:
if re.match('.*ORA-12899.*', e.args[0]) \
or re.match('.*1406.*', e.args[0]):
raise exceptions.DatabaseException('Could not persist message, msg_content too large: %s' % str(e))
else:
raise exceptions.DatabaseException('Could not persist message: %s' % str(e))
def add_contents(contents, bulk_size=10000, session=None):
"""
Add contents.
:param contents: dict of contents.
:param session: session.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
default_params = {
'request_id': None,
'workload_id': None,
'transform_id': None,
'coll_id': None,
'map_id': None,
'scope': None,
'name': None,
'min_id': 0,
'max_id': 0,
'content_type': ContentType.File,
'status': ContentStatus.New,
'locking': ContentLocking.Idle,
'content_relation_type': ContentRelationType.Input,
'bytes': 0,
'md5': None,
'adler32': None,
'processing_id': None,
'storage_id': None,
'retries': 0,
'path': None,
'expired_at': datetime.datetime.utcnow() + datetime.timedelta(days=30),
'content_metadata': None
}
for content in contents:
for key in default_params:
if key not in content:
content[key] = default_params[key]
sub_params = [
contents[i:i + bulk_size] for i in range(0, len(contents), bulk_size)
]
try:
for sub_param in sub_params:
session.bulk_insert_mappings(models.Content, sub_param)
content_ids = [None for _ in range(len(contents))]
return content_ids
except IntegrityError as error:
raise exceptions.DuplicatedObject('Duplicated objects: %s' % (error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def update_messages(messages, session=None):
"""
Update all messages status with the given IDs.
:param messages: The messages to be updated as a list of dictionaries.
"""
try:
for msg in messages:
session.query(models.Message).filter_by(msg_id=msg['msg_id']).update({'status': msg['status']}, synchronize_session=False)
except IntegrityError as e:
raise exceptions.DatabaseException(e.args)
def add_processing(transform_id, status=ProcessingStatus.New, locking=ProcessingLocking.Idle, submitter=None,
granularity=None, granularity_type=None, expired_at=None, processing_metadata=None,
output_metadata=None, session=None):
"""
Add a processing.
:param transform_id: Transform id.
:param status: processing status.
:param locking: processing locking.
:param submitter: submitter name.
:param granularity: Granularity size.
:param granularity_type: Granularity type.
:param expired_at: The datetime when it expires.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If a processing with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: processing id.
"""
if isinstance(granularity_type, GranularityType):
granularity_type = granularity_type.value
if isinstance(status, ProcessingStatus):
status = status.value
if isinstance(locking, ProcessingLocking):
locking = locking.value
if processing_metadata:
processing_metadata = json.dumps(processing_metadata)
if output_metadata:
output_metadata = json.dumps(output_metadata)
insert = """insert into atlas_idds.processings(transform_id, status, locking, submitter, granularity_type,
granularity, created_at, updated_at, expired_at, processing_metadata,
output_metadata)
values(:transform_id, :status, :locking, :submitter, :granularity_type, :granularity, :created_at,
:updated_at, :expired_at, :processing_metadata, :output_metadata) returning processing_id into :processing_id
"""
stmt = text(insert)
stmt = stmt.bindparams(outparam("processing_id", type_=BigInteger().with_variant(Integer, "sqlite")))
try:
processing_id = None
ret = session.execute(stmt, {'transform_id': transform_id, 'status': status, 'locking': locking,
'submitter': submitter, 'granularity_type': granularity_type, 'granularity': granularity,
'created_at': datetime.datetime.utcnow(), 'updated_at': datetime.datetime.utcnow(),
'expired_at': expired_at, 'processing_metadata': processing_metadata,
'output_metadata': output_metadata, 'processing_id': processing_id})
processing_id = ret.out_parameters['processing_id'][0]
return processing_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Processing already exists!: %s' % (error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_workprogress(request_id,
scope,
name,
priority=0,
status=WorkprogressStatus.New,
locking=WorkprogressLocking.Idle,
expired_at=None,
errors=None,
workprogress_metadata=None,
processing_metadata=None,
session=None):
"""
Add a workprogress.
:param request_id: The request id.
:param scope: The scope.
:param name: The name.
:param status: The status as integer.
:param locking: The locking as integer.
:param priority: The priority as integer.
:param expired_at: The datetime when the workprogress will be expired at.
:param errors: The errors as a json.
:param workprogress_metadata: The metadata as json.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If a workprogress with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: workprogress id.
"""
try:
new_wp = create_workprogress(
request_id=request_id,
scope=scope,
name=name,
priority=priority,
status=status,
locking=locking,
expired_at=expired_at,
workprogress_metadata=workprogress_metadata,
processing_metadata=processing_metadata)
new_wp.save(session=session)
wp_id = new_wp.workprogress_id
return wp_id
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'workprogress %s already exists!: %s' % (new_wp, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_content(transform_id, coll_id, map_id, scope, name, min_id=0, max_id=0, content_type=ContentType.File, status=ContentStatus.New,
bytes=0, md5=None, adler32=None, processing_id=None, storage_id=None, retries=0,
locking=ContentLocking.Idle, path=None, expired_at=None, content_metadata=None, session=None):
"""
Add a content.
:param transform_id: transform id.
:param coll_id: collection id.
:param map_id: The id to map inputs to outputs.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param status: content status.
:param bytes: The size of the content.
:param md5: md5 checksum.
:param alder32: adler32 checksum.
:param processing_id: The processing id.
:param storage_id: The storage id.
:param retries: The number of retries.
:param path: The content path.
:param expired_at: The datetime when it expires.
:param content_metadata: The metadata as json.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
try:
new_content = create_content(transform_id=transform_id, coll_id=coll_id, map_id=map_id,
scope=scope, name=name, min_id=min_id, max_id=max_id,
content_type=content_type, status=status, bytes=bytes, md5=md5,
adler32=adler32, processing_id=processing_id, storage_id=storage_id,
retries=retries, path=path, expired_at=expired_at,
content_metadata=content_metadata)
new_content.save(session=session)
content_id = new_content.content_id
return content_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Content transform_id:map_id(%s:%s) already exists!: %s' %
(transform_id, map_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_wp2transform(workprogress_id, transform_id, session=None):
"""
Add the relation between workprogress_id and transform_id
:param workprogress_id: Workprogress id.
:param transform_id: Transform id.
:param session: The database session in use.
"""
try:
new_wp2transform = models.Workprogress2transform(
workprogress_id=workprogress_id, transform_id=transform_id)
new_wp2transform.save(session=session)
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'Workprogress2Transform already exists!(%s:%s): %s' %
(workprogress_id, transform_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def delete_messages(messages, session=None):
"""
Delete all messages with the given IDs.
:param messages: The messages to delete as a list of dictionaries.
"""
message_condition = []
for message in messages:
message_condition.append(models.Message.msg_id == message['msg_id'])
try:
if message_condition:
session.query(models.Message).\
with_hint(models.Message, "index(messages MESSAGES_PK)", 'oracle').\
filter(or_(*message_condition)).\
delete(synchronize_session=False)
except IntegrityError as e:
raise exceptions.DatabaseException(e.args)
def add_req2transform(request_id, transform_id, session=None):
"""
Add the relation between request_id and transform_id
:param request_id: Request id.
:param transform_id: Transform id.
:param session: The database session in use.
"""
try:
new_req2transform = models.Req2transform(request_id=request_id,
transform_id=transform_id)
new_req2transform.save(session=session)
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'Request2Transform already exists!(%s:%s): %s' %
(request_id, transform_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def retrieve_health_items(session=None):
"""
Retrieve health items.
:param session: The database session.
:returns healths: List of dictionaries
"""
items = []
try:
query = session.query(models.Health)
tmp = query.all()
if tmp:
for t in tmp:
items.append(t.to_dict())
return items
except IntegrityError as e:
raise exceptions.DatabaseException(e.args)
def add_req2transform(request_id, transform_id, session=None):
"""
Add the relation between request_id and transform_id
:param request_id: Request id.
:param transform_id: Transform id.
:param session: The database session in use.
"""
try:
insert_req2transforms = """insert into atlas_idds.req2transforms(request_id, transform_id)
values(:request_id, :transform_id)
"""
stmt = text(insert_req2transforms)
session.execute(stmt, {
'request_id': request_id,
'transform_id': transform_id
})
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'Request2Transform already exists!(%s:%s): %s' %
(request_id, transform_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_health_item(agent,
hostname,
pid,
thread_id,
thread_name,
payload,
session=None):
"""
Add a health item.
:param agent: The agent name.
:param hostname: The hostname.
:param pid: The pid.
:param thread_id: The thread id.
:param thread_name: The thread name.
:param payload: The payload.
:param session: The database session.
"""
try:
counts = session.query(models.Health)\
.filter(models.Health.agent == agent)\
.filter(models.Health.hostname == hostname)\
.filter(models.Health.pid == pid)\
.filter(models.Health.thread_id == thread_id)\
.update({'updated_at': datetime.datetime.utcnow()})
if not counts:
new_h = models.Health(agent=agent,
hostname=hostname,
pid=pid,
thread_id=thread_id,
thread_name=thread_name,
payload=payload)
new_h.save(session=session)
except DatabaseError as e:
raise exceptions.DatabaseException('Could not persist message: %s' %
str(e))
def add_collection(scope, name, coll_type=CollectionType.Dataset, transform_id=None,
relation_type=CollectionRelationType.Input, bytes=0, status=CollectionStatus.New,
locking=CollectionLocking.Idle, total_files=0, retries=0, expired_at=None,
coll_metadata=None, session=None):
"""
Add a collection.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param coll_type: The type of dataset as dataset or container.
:param transform_id: The transform id related to this collection.
:param relation_type: The relation between this collection and its transform,
such as Input, Output, Log and so on.
:param size: The size of the collection.
:param status: The status.
:param locking: The locking.
:param total_files: Number of total files.
:param retries: Number of retries.
:param expired_at: The datetime when it expires.
:param coll_metadata: The metadata as json.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection id.
"""
if isinstance(coll_type, CollectionType):
coll_type = coll_type.value
if isinstance(status, CollectionStatus):
status = status.value
if isinstance(locking, CollectionLocking):
locking = locking.value
if isinstance(relation_type, CollectionRelationType):
relation_type = relation_type.value
if coll_metadata:
coll_metadata = json.dumps(coll_metadata)
insert_coll_sql = """insert into atlas_idds.collections(scope, name, coll_type, transform_id,
relation_type, bytes, status, locking, total_files,
retries, created_at, updated_at, expired_at,
coll_metadata)
values(:scope, :name, :coll_type, :transform_id, :relation_type, :bytes,
:status, :locking, :total_files, :retries, :created_at, :updated_at, :expired_at,
:coll_metadata) returning coll_id into :coll_id
"""
stmt = text(insert_coll_sql)
stmt = stmt.bindparams(outparam("coll_id", type_=BigInteger().with_variant(Integer, "sqlite")))
try:
coll_id = None
ret = session.execute(stmt, {'scope': scope, 'name': name, 'coll_type': coll_type,
'transform_id': transform_id, 'relation_type': relation_type, 'bytes': bytes,
'status': status, 'locking': locking, 'total_files': total_files, 'retries': retries,
'created_at': datetime.datetime.utcnow(), 'updated_at': datetime.datetime.utcnow(),
'expired_at': expired_at, 'coll_metadata': coll_metadata, 'coll_id': coll_id})
coll_id = ret.out_parameters['coll_id'][0]
return coll_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Collection scope:name(%s:%s) with transform_id(%s) already exists!: %s' %
(scope, name, transform_id, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_message(msg_type,
status,
source,
request_id,
workload_id,
transform_id,
num_contents,
msg_content,
bulk_size=None,
processing_id=None,
destination=MessageDestination.Outside,
session=None):
"""
Add a message to be submitted asynchronously to a message broker.
:param msg_type: The type of the msg as a number, e.g., finished_stagein.
:param status: The status about the message
:param source: The source where the message is from.
:param request_id: The request id.
:param workload_id: The workload id.
:param transform_id: The transform id.
:param num_contents: Number of items in msg_content.
:param msg_content: The message msg_content as JSON.
:param session: The database session.
"""
try:
num_contents_list = []
msg_content_list = []
if bulk_size and num_contents > bulk_size:
if 'files' in msg_content:
files = msg_content['files']
chunks = [
files[i:i + bulk_size]
for i in range(0, len(files), bulk_size)
]
for chunk in chunks:
new_msg_content = copy.deepcopy(msg_content)
new_msg_content['files'] = chunk
new_num_contents = len(chunk)
num_contents_list.append(new_num_contents)
msg_content_list.append(new_msg_content)
else:
num_contents_list.append(num_contents)
msg_content_list.append(msg_content)
else:
num_contents_list.append(num_contents)
msg_content_list.append(msg_content)
msgs = []
for msg_content, num_contents in zip(msg_content_list,
num_contents_list):
new_message = {
'msg_type': msg_type,
'status': status,
'request_id': request_id,
'workload_id': workload_id,
'transform_id': transform_id,
'source': source,
'num_contents': num_contents,
'destination': destination,
'processing_id': processing_id,
'locking': 0,
'msg_content': msg_content
}
msgs.append(new_message)
session.bulk_insert_mappings(models.Message, msgs)
except TypeError as e:
raise exceptions.DatabaseException('Invalid JSON for msg_content: %s' %
str(e))
except DatabaseError as e:
if re.match('.*ORA-12899.*', e.args[0]) \
or re.match('.*1406.*', e.args[0]):
raise exceptions.DatabaseException(
'Could not persist message, msg_content too large: %s' %
str(e))
else:
raise exceptions.DatabaseException(
'Could not persist message: %s' % str(e))
def retrieve_messages(bulk_size=1000,
msg_type=None,
status=None,
source=None,
destination=None,
request_id=None,
workload_id=None,
transform_id=None,
processing_id=None,
session=None):
"""
Retrieve up to $bulk messages.
:param bulk: Number of messages as an integer.
:param msg_type: Return only specified msg_type.
:param status: The status about the message
:param source: The source where the message is from.
:param session: The database session.
:returns messages: List of dictionaries
"""
messages = []
try:
query = session.query(models.Message)
if request_id is not None:
query = query.with_hint(models.Message,
"INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)",
'oracle')
elif transform_id:
query = query.with_hint(models.Message,
"INDEX(MESSAGES MESSAGES_TYPE_ST_TF_IDX)",
'oracle')
elif processing_id is not None:
query = query.with_hint(models.Message,
"INDEX(MESSAGES MESSAGES_TYPE_ST_PR_IDX)",
'oracle')
else:
query = query.with_hint(models.Message,
"INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)",
'oracle')
if msg_type is not None:
query = query.filter_by(msg_type=msg_type)
if status is not None:
query = query.filter_by(status=status)
if source is not None:
query = query.filter_by(source=source)
if destination is not None:
query = query.filter_by(destination=destination)
if request_id is not None:
query = query.filter_by(request_id=request_id)
if workload_id is not None:
query = query.filter_by(workload_id=workload_id)
if transform_id is not None:
query = query.filter_by(transform_id=transform_id)
if processing_id is not None:
query = query.filter_by(processing_id=processing_id)
if bulk_size:
query = query.order_by(models.Message.created_at).limit(bulk_size)
# query = query.with_for_update(nowait=True)
tmp = query.all()
if tmp:
for t in tmp:
messages.append(t.to_dict())
return messages
except IntegrityError as e:
raise exceptions.DatabaseException(e.args)
def add_transform(transform_type,
transform_tag=None,
priority=0,
status=TransformStatus.New,
locking=TransformLocking.Idle,
retries=0,
expired_at=None,
transform_metadata=None,
request_id=None,
session=None):
"""
Add a transform.
:param transform_type: Transform type.
:param transform_tag: Transform tag.
:param priority: priority.
:param status: Transform status.
:param locking: Transform locking.
:param retries: The number of retries.
:param expired_at: The datetime when it expires.
:param transform_metadata: The metadata as json.
:raises DuplicatedObject: If a transform with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
if isinstance(transform_type, TransformType):
transform_type = transform_type.value
if isinstance(status, TransformStatus):
status = status.value
if isinstance(locking, TransformLocking):
locking = locking.value
if transform_metadata:
transform_metadata = json.dumps(transform_metadata)
insert = """insert into atlas_idds.transforms(transform_type, transform_tag, priority, status, locking, retries,
created_at, expired_at, transform_metadata)
values(:transform_type, :transform_tag, :priority, :status, :locking, :retries, :created_at,
:expired_at, :transform_metadata) returning transform_id into :transform_id
"""
stmt = text(insert)
stmt = stmt.bindparams(
outparam("transform_id",
type_=BigInteger().with_variant(Integer, "sqlite")))
try:
transform_id = None
ret = session.execute(
stmt, {
'transform_type': transform_type,
'transform_tag': transform_tag,
'priority': priority,
'status': status,
'locking': locking,
'retries': retries,
'created_at': datetime.datetime.utcnow(),
'updated_at': datetime.datetime.utcnow(),
'expired_at': expired_at,
'transform_metadata': transform_metadata,
'transform_id': transform_id
})
transform_id = ret.out_parameters['transform_id'][0]
if request_id:
insert_req2transforms = """insert into atlas_idds.req2transforms(request_id, transform_id)
values(:request_id, :transform_id)
"""
stmt = text(insert_req2transforms)
session.execute(stmt, {
'request_id': request_id,
'transform_id': transform_id
})
return transform_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Transform already exists!: %s' %
(error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_contents(contents, returning_id=False, bulk_size=100, session=None):
"""
Add contents.
:param contents: dict of contents.
:param returning_id: whether to return id.
:param session: session.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
default_params = {
'coll_id': None,
'scope': None,
'name': None,
'min_id': None,
'max_id': None,
'content_type': ContentType.File,
'status': ContentStatus.New,
'bytes': 0,
'md5': None,
'adler32': None,
'processing_id': None,
'storage_id': None,
'retries': 0,
'path': None,
'expired_at': datetime.datetime.utcnow() + datetime.timedelta(days=30),
'content_metadata': None
}
if returning_id:
insert_coll_sql = """insert into atlas_idds.contents(coll_id, scope, name, min_id, max_id, content_type,
status, bytes, md5, adler32, processing_id,
storage_id, retries, path, expired_at,
content_metadata)
values(:coll_id, :scope, :name, :min_id, :max_id, :content_type, :status, :bytes,
:md5, :adler32, :processing_id, :storage_id, :retries, :path, :expired_at,
:content_metadata) RETURNING content_id into :content_id
"""
stmt = text(insert_coll_sql)
stmt = stmt.bindparams(
outparam("content_id",
type_=BigInteger().with_variant(Integer, "sqlite")))
else:
insert_coll_sql = """insert into atlas_idds.contents(coll_id, scope, name, min_id, max_id, content_type,
status, bytes, md5, adler32, processing_id,
storage_id, retries, path, expired_at,
content_metadata)
values(:coll_id, :scope, :name, :min_id, :max_id, :content_type, :status, :bytes,
:md5, :adler32, :processing_id, :storage_id, :retries, :path, :expired_at,
:content_metadata)
"""
stmt = text(insert_coll_sql)
params = []
for content in contents:
param = {}
for key in default_params:
if key in content:
param[key] = content[key]
else:
param[key] = default_params[key]
if isinstance(param['content_type'], ContentType):
param['content_type'] = param['content_type'].value
if isinstance(param['status'], ContentStatus):
param['status'] = param['status'].value
if param['content_metadata']:
param['content_metadata'] = json.dumps(param['content_metadata'])
params.append(param)
sub_params = [
params[i:i + bulk_size] for i in range(0, len(params), bulk_size)
]
try:
content_ids = None
if returning_id:
content_ids = []
for sub_param in sub_params:
content_id = None
sub_param['content_id'] = content_id
ret = session.execute(stmt, sub_param)
content_ids.extend(ret.out_parameters['content_id'])
else:
for sub_param in sub_params:
ret = session.execute(stmt, sub_param)
content_ids = [None for _ in range(len(params))]
return content_ids
except IntegrityError as error:
raise exceptions.DuplicatedObject('Duplicated objects: %s' % (error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_content(coll_id,
scope,
name,
min_id,
max_id,
content_type=ContentType.File,
status=ContentStatus.New,
bytes=0,
md5=None,
adler32=None,
processing_id=None,
storage_id=None,
retries=0,
path=None,
expired_at=None,
content_metadata=None,
returning_id=False,
session=None):
"""
Add a content.
:param coll_id: collection id.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param min_id: The minimal id of the content.
:param max_id: The maximal id of the content.
:param content_type: The type of the content.
:param status: content status.
:param bytes: The size of the content.
:param md5: md5 checksum.
:param alder32: adler32 checksum.
:param processing_id: The processing id.
:param storage_id: The storage id.
:param retries: The number of retries.
:param path: The content path.
:param expired_at: The datetime when it expires.
:param content_metadata: The metadata as json.
:raises DuplicatedObject: If a collection with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
if isinstance(content_type, ContentType):
content_type = content_type.value
if isinstance(status, ContentStatus):
status = status.value
if content_metadata:
content_metadata = json.dumps(content_metadata)
if returning_id:
insert_coll_sql = """insert into atlas_idds.contents(coll_id, scope, name, min_id, max_id, content_type,
status, bytes, md5, adler32, processing_id,
storage_id, retries, path, expired_at,
content_metadata)
values(:coll_id, :scope, :name, :min_id, :max_id, :content_type, :status, :bytes,
:md5, :adler32, :processing_id, :storage_id, :retries, :path, :expired_at,
:content_metadata) RETURNING content_id into :content_id
"""
stmt = text(insert_coll_sql)
stmt = stmt.bindparams(
outparam("content_id",
type_=BigInteger().with_variant(Integer, "sqlite")))
else:
insert_coll_sql = """insert into atlas_idds.contents(coll_id, scope, name, min_id, max_id, content_type,
status, bytes, md5, adler32, processing_id,
storage_id, retries, path, expired_at,
content_metadata)
values(:coll_id, :scope, :name, :min_id, :max_id, :content_type, :status, :bytes,
:md5, :adler32, :processing_id, :storage_id, :retries, :path, :expired_at,
:content_metadata)
"""
stmt = text(insert_coll_sql)
try:
content_id = None
if returning_id:
ret = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'min_id': min_id,
'max_id': max_id,
'content_type': content_type,
'status': status,
'bytes': bytes,
'md5': md5,
'adler32': adler32,
'processing_id': processing_id,
'storage_id': storage_id,
'retries': retries,
'path': path,
'created_at': datetime.datetime.utcnow(),
'updated_at': datetime.datetime.utcnow(),
'expired_at': expired_at,
'content_metadata': content_metadata,
'content_id': content_id
})
content_id = ret.out_parameters['content_id'][0]
else:
ret = session.execute(
stmt, {
'coll_id': coll_id,
'scope': scope,
'name': name,
'min_id': min_id,
'max_id': max_id,
'content_type': content_type,
'status': status,
'bytes': bytes,
'md5': md5,
'adler32': adler32,
'processing_id': processing_id,
'storage_id': storage_id,
'retries': retries,
'path': path,
'created_at': datetime.datetime.utcnow(),
'updated_at': datetime.datetime.utcnow(),
'expired_at': expired_at,
'content_metadata': content_metadata
})
return content_id
except IntegrityError as error:
raise exceptions.DuplicatedObject(
'Content coll_id:scope:name(%s:%s:%s) already exists!: %s' %
(coll_id, scope, name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
def add_request(scope,
name,
requester=None,
request_type=None,
transform_tag=None,
status=RequestStatus.New,
locking=RequestLocking.Idle,
priority=0,
lifetime=30,
workload_id=None,
request_metadata=None,
processing_metadata=None,
session=None):
"""
Add a request.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param requestr: The requester, such as panda, user and so on.
:param request_type: The type of the request, such as ESS, DAOD.
:param transform_tag: Transform tag, such as ATLAS AMI tag.
:param status: The request status as integer.
:param locking: The request locking as integer.
:param priority: The priority as integer.
:param lifetime: The life time as umber of days.
:param workload_id: The external workload id.
:param request_metadata: The metadata as json.
:param processing_metadata: The metadata as json.
:raises DuplicatedObject: If an request with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: request id.
"""
if isinstance(request_type, RequestType):
request_type = request_type.value
if isinstance(status, RequestStatus):
status = status.value
if isinstance(locking, RequestLocking):
locking = locking.value
if request_metadata:
request_metadata = json.dumps(request_metadata)
if processing_metadata:
processing_metadata = json.dumps(processing_metadata)
insert_request_sql = """insert into atlas_idds.requests(scope, name, requester, request_type, transform_tag, priority,
status, locking, workload_id, created_at, updated_at, expired_at, request_metadata,
processing_metadata)
values(:scope, :name, :requester, :request_type, :transform_tag, :priority, :status,
:locking, :workload_id, :created_at, :updated_at, :expired_at,
:request_metadata, :processing_metadata) RETURNING request_id into :request_id
"""
stmt = text(insert_request_sql)
stmt = stmt.bindparams(
outparam("request_id",
type_=BigInteger().with_variant(Integer, "sqlite")))
try:
request_id = None
ret = session.execute(
stmt, {
"scope":
scope,
"name":
name,
"requester":
requester,
"request_type":
request_type,
"transform_tag":
transform_tag,
"priority":
priority,
'status':
status,
'locking':
locking,
'workload_id':
workload_id,
'created_at':
datetime.datetime.utcnow(),
'updated_at':
datetime.datetime.utcnow(),
'expired_at':
datetime.datetime.utcnow() + datetime.timedelta(days=lifetime),
'request_metadata':
request_metadata,
'processing_metadata':
processing_metadata,
'request_id':
request_id
})
request_id = ret.out_parameters['request_id'][0]
return request_id
except IntegrityError as error:
raise exceptions.DuplicatedObject('Request %s:%s already exists!: %s' %
(scope, name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error)
