def test_create_and_check_for_request_orm(self):
""" Request (ORM): Test the creation, query, and cancel of a Request """
properties = get_request_properties()
request_id = add_request(**properties)
request = get_request(request_id=request_id)
assert_equal(request_id, request['request_id'])
for key in properties:
if key in ['lifetime']:
continue
assert_equal(request[key], properties[key])
request_id1 = add_request(**properties)
delete_requests(request_id=request_id1)
update_request(request_id, parameters={'status': RequestStatus.Failed})
request = get_request(request_id=request_id)
assert_equal(request['status'], RequestStatus.Failed)
req = get_request(request_id=999999)
assert_equal(req, None)
delete_requests(request_id=request_id)
req = get_request(request_id=request_id)
assert_equal(req, None)
def get_requests_by_status_type(status,
request_type=None,
time_period=None,
locking=False,
bulk_size=None,
session=None):
"""
Get requests by status and type
:param status: list of status of the request data.
:param request_type: The type of the request data.
:param time_period: Delay of seconds before last update.
:param locking: Wheter to lock requests to avoid others get the same request.
:param bulk_size: Size limitation per retrieve.
:returns: list of Request.
"""
reqs = orm_requests.get_requests_by_status_type(status,
request_type,
time_period,
locking=locking,
bulk_size=bulk_size,
session=session)
if locking:
parameters = {'locking': RequestLocking.Locking}
for req in reqs:
orm_requests.update_request(request_id=req['request_id'],
parameters=parameters,
session=session)
return reqs
def update_request(request_id, parameters, session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
"""
return orm_requests.update_request(request_id, parameters, session=session)
def update_request_with_transforms(request_id,
parameters,
transforms_to_add,
transforms_to_extend,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param transforms_to_add: list of transforms
:param transforms_to_extend: list of transforms
"""
for transform in transforms_to_add:
if 'collections' not in transform or len(
transform['collections']) == 0:
msg = "Transform must have collections, such as input collection, output collection and log collection"
raise exceptions.WrongParameterException(msg)
collections = transform['collections']
del transform['collections']
transform_id = orm_transforms.add_transform(**transform,
session=session)
input_coll_ids = []
log_coll_ids = []
for collection in collections['input_collections']:
collection['transform_id'] = transform_id
input_coll_id = orm_collections.add_collection(**collection,
session=session)
input_coll_ids.append(input_coll_id)
for collection in collections['log_collections']:
collection['transform_id'] = transform_id
log_coll_id = orm_collections.add_collection(**collection,
session=session)
log_coll_ids.append(log_coll_id)
for collection in collections['output_collections']:
collection['transform_id'] = transform_id
workload_id = transform['transform_metadata'][
'workload_id'] if 'workload_id' in transform[
'transform_metadata'] else None
collection['coll_metadata'] = {
'transform_id': transform_id,
'workload_id': workload_id,
'input_collections': input_coll_ids,
'log_collections': log_coll_ids
}
orm_collections.add_collection(**collection, session=session)
for transform in transforms_to_extend:
transform_id = transform['transform_id']
del transform['transform_id']
# orm_transforms.add_req2transform(request_id, transform_id, session=session)
orm_transforms.update_transform(transform_id,
parameters=transform,
session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def get_requests_with_messaging(locking=False, bulk_size=None, session=None):
msgs = core_messages.retrieve_request_messages(request_id=None,
bulk_size=bulk_size,
session=session)
if msgs:
req_ids = [msg['request_id'] for msg in msgs]
if locking:
req2s = orm_requests.get_requests_by_status_type(
status=None,
request_ids=req_ids,
locking=locking,
locking_for_update=True,
bulk_size=None,
session=session)
if req2s:
reqs = []
for req_id in req_ids:
if len(reqs) >= bulk_size:
break
for req in req2s:
if req['request_id'] == req_id:
reqs.append(req)
break
else:
reqs = []
parameters = {'locking': RequestLocking.Locking}
for req in reqs:
orm_requests.update_request(request_id=req['request_id'],
parameters=parameters,
session=session)
return reqs
else:
reqs = orm_requests.get_requests_by_status_type(
status=None,
request_ids=req_ids,
locking=locking,
locking_for_update=locking,
bulk_size=bulk_size,
session=session)
return reqs
else:
return []
def update_request_with_workprogresses(request_id, parameters, new_workprogresses, session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param new_workprogresses: list of new workprogresses.
"""
if new_workprogresses:
orm_workprogresses.add_workprogresses(new_workprogresses, session=session)
return orm_requests.update_request(request_id, parameters, session=session)
def get_requests_by_status_type(status,
request_type=None,
time_period=None,
locking=False,
bulk_size=None,
to_json=False,
by_substatus=False,
with_messaging=False,
session=None):
"""
Get requests by status and type
:param status: list of status of the request data.
:param request_type: The type of the request data.
:param time_period: Delay of seconds before last update.
:param locking: Wheter to lock requests to avoid others get the same request.
:param bulk_size: Size limitation per retrieve.
:param to_json: return json format.
:returns: list of Request.
"""
if with_messaging:
reqs = get_requests_with_messaging(locking=locking,
bulk_size=bulk_size,
session=session)
if reqs:
return reqs
if locking:
if bulk_size:
# order by cannot work together with locking. So first select 2 * bulk_size without locking with order by.
# then select with locking.
req_ids = orm_requests.get_requests_by_status_type(
status,
request_type,
time_period,
locking=locking,
bulk_size=bulk_size * 2,
locking_for_update=False,
to_json=False,
by_substatus=by_substatus,
only_return_id=True,
session=session)
if req_ids:
req2s = orm_requests.get_requests_by_status_type(
status,
request_type,
time_period,
request_ids=req_ids,
locking=locking,
locking_for_update=True,
bulk_size=None,
to_json=to_json,
by_substatus=by_substatus,
session=session)
if req2s:
# reqs = req2s[:bulk_size]
# order requests
reqs = []
for req_id in req_ids:
if len(reqs) >= bulk_size:
break
for req in req2s:
if req['request_id'] == req_id:
reqs.append(req)
break
# reqs = reqs[:bulk_size]
else:
reqs = []
else:
reqs = []
else:
reqs = orm_requests.get_requests_by_status_type(
status,
request_type,
time_period,
locking=locking,
locking_for_update=locking,
bulk_size=bulk_size,
to_json=to_json,
by_substatus=by_substatus,
session=session)
parameters = {'locking': RequestLocking.Locking}
for req in reqs:
orm_requests.update_request(request_id=req['request_id'],
parameters=parameters,
session=session)
else:
reqs = orm_requests.get_requests_by_status_type(
status,
request_type,
time_period,
locking=locking,
bulk_size=bulk_size,
to_json=to_json,
by_substatus=by_substatus,
session=session)
return reqs
def update_request_with_transforms(request_id,
parameters,
new_transforms=None,
update_transforms=None,
new_messages=None,
update_messages=None,
session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param new_transforms: list of transforms
:param update_transforms: list of transforms
"""
if new_transforms:
for tf in new_transforms:
# tf_id = orm_transforms.add_transform(**tf, session=session)
# original_work = tf['transform_metadata']['original_work']
# del tf['transform_metadata']['original_work']
workflow = tf['transform_metadata']['workflow']
del tf['transform_metadata']['workflow']
work = tf['transform_metadata']['work']
tf_copy = copy.deepcopy(tf)
tf_id = orm_transforms.add_transform(**tf_copy, session=session)
tf['transform_id'] = tf_id
# work = tf['transform_metadata']['work']
# original_work.set_work_id(tf_id, transforming=True)
# original_work.set_status(WorkStatus.New)
work.set_work_id(tf_id, transforming=True)
work.set_status(WorkStatus.New)
workflow.refresh_works()
collections = generate_collections(tf)
for coll in collections:
collection = coll['collection']
del coll['collection']
coll['transform_id'] = tf_id
coll_id = orm_collections.add_collection(**coll,
session=session)
# work.set_collection_id(coll, coll_id)
collection.coll_id = coll_id
# update transform to record the coll_id
work.refresh_work()
orm_transforms.update_transform(
transform_id=tf_id,
parameters={'transform_metadata': tf['transform_metadata']},
session=session)
if update_transforms:
for tr_id in update_transforms:
orm_transforms.update_transform(
transform_id=tr_id,
parameters=update_transforms[tr_id],
session=session)
if new_messages:
orm_messages.add_messages(new_messages, session=session)
if update_messages:
orm_messages.update_messages(update_messages, session=session)
return orm_requests.update_request(request_id, parameters, session=session)
