def test_create_and_check_for_request_orm(self): """ Request (ORM): Test the creation, query, and cancel of a Request """ properties = get_request_properties() request_id = add_request(**properties) request = get_request(request_id=request_id) assert_equal(request_id, request['request_id']) for key in properties: if key in ['lifetime']: continue assert_equal(request[key], properties[key]) request_id1 = add_request(**properties) delete_requests(request_id=request_id1) update_request(request_id, parameters={'status': RequestStatus.Failed}) request = get_request(request_id=request_id) assert_equal(request['status'], RequestStatus.Failed) req = get_request(request_id=999999) assert_equal(req, None) delete_requests(request_id=request_id) req = get_request(request_id=request_id) assert_equal(req, None)
def get_requests_by_status_type(status, request_type=None, time_period=None, locking=False, bulk_size=None, session=None): """ Get requests by status and type :param status: list of status of the request data. :param request_type: The type of the request data. :param time_period: Delay of seconds before last update. :param locking: Wheter to lock requests to avoid others get the same request. :param bulk_size: Size limitation per retrieve. :returns: list of Request. """ reqs = orm_requests.get_requests_by_status_type(status, request_type, time_period, locking=locking, bulk_size=bulk_size, session=session) if locking: parameters = {'locking': RequestLocking.Locking} for req in reqs: orm_requests.update_request(request_id=req['request_id'], parameters=parameters, session=session) return reqs
def update_request(request_id, parameters, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. """ return orm_requests.update_request(request_id, parameters, session=session)
def update_request_with_transforms(request_id, parameters, transforms_to_add, transforms_to_extend, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param transforms_to_add: list of transforms :param transforms_to_extend: list of transforms """ for transform in transforms_to_add: if 'collections' not in transform or len( transform['collections']) == 0: msg = "Transform must have collections, such as input collection, output collection and log collection" raise exceptions.WrongParameterException(msg) collections = transform['collections'] del transform['collections'] transform_id = orm_transforms.add_transform(**transform, session=session) input_coll_ids = [] log_coll_ids = [] for collection in collections['input_collections']: collection['transform_id'] = transform_id input_coll_id = orm_collections.add_collection(**collection, session=session) input_coll_ids.append(input_coll_id) for collection in collections['log_collections']: collection['transform_id'] = transform_id log_coll_id = orm_collections.add_collection(**collection, session=session) log_coll_ids.append(log_coll_id) for collection in collections['output_collections']: collection['transform_id'] = transform_id workload_id = transform['transform_metadata'][ 'workload_id'] if 'workload_id' in transform[ 'transform_metadata'] else None collection['coll_metadata'] = { 'transform_id': transform_id, 'workload_id': workload_id, 'input_collections': input_coll_ids, 'log_collections': log_coll_ids } orm_collections.add_collection(**collection, session=session) for transform in transforms_to_extend: transform_id = transform['transform_id'] del transform['transform_id'] # orm_transforms.add_req2transform(request_id, transform_id, session=session) orm_transforms.update_transform(transform_id, parameters=transform, session=session) return orm_requests.update_request(request_id, parameters, session=session)
def get_requests_with_messaging(locking=False, bulk_size=None, session=None): msgs = core_messages.retrieve_request_messages(request_id=None, bulk_size=bulk_size, session=session) if msgs: req_ids = [msg['request_id'] for msg in msgs] if locking: req2s = orm_requests.get_requests_by_status_type( status=None, request_ids=req_ids, locking=locking, locking_for_update=True, bulk_size=None, session=session) if req2s: reqs = [] for req_id in req_ids: if len(reqs) >= bulk_size: break for req in req2s: if req['request_id'] == req_id: reqs.append(req) break else: reqs = [] parameters = {'locking': RequestLocking.Locking} for req in reqs: orm_requests.update_request(request_id=req['request_id'], parameters=parameters, session=session) return reqs else: reqs = orm_requests.get_requests_by_status_type( status=None, request_ids=req_ids, locking=locking, locking_for_update=locking, bulk_size=bulk_size, session=session) return reqs else: return []
def update_request_with_workprogresses(request_id, parameters, new_workprogresses, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param new_workprogresses: list of new workprogresses. """ if new_workprogresses: orm_workprogresses.add_workprogresses(new_workprogresses, session=session) return orm_requests.update_request(request_id, parameters, session=session)
def get_requests_by_status_type(status, request_type=None, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, with_messaging=False, session=None): """ Get requests by status and type :param status: list of status of the request data. :param request_type: The type of the request data. :param time_period: Delay of seconds before last update. :param locking: Wheter to lock requests to avoid others get the same request. :param bulk_size: Size limitation per retrieve. :param to_json: return json format. :returns: list of Request. """ if with_messaging: reqs = get_requests_with_messaging(locking=locking, bulk_size=bulk_size, session=session) if reqs: return reqs if locking: if bulk_size: # order by cannot work together with locking. So first select 2 * bulk_size without locking with order by. # then select with locking. req_ids = orm_requests.get_requests_by_status_type( status, request_type, time_period, locking=locking, bulk_size=bulk_size * 2, locking_for_update=False, to_json=False, by_substatus=by_substatus, only_return_id=True, session=session) if req_ids: req2s = orm_requests.get_requests_by_status_type( status, request_type, time_period, request_ids=req_ids, locking=locking, locking_for_update=True, bulk_size=None, to_json=to_json, by_substatus=by_substatus, session=session) if req2s: # reqs = req2s[:bulk_size] # order requests reqs = [] for req_id in req_ids: if len(reqs) >= bulk_size: break for req in req2s: if req['request_id'] == req_id: reqs.append(req) break # reqs = reqs[:bulk_size] else: reqs = [] else: reqs = [] else: reqs = orm_requests.get_requests_by_status_type( status, request_type, time_period, locking=locking, locking_for_update=locking, bulk_size=bulk_size, to_json=to_json, by_substatus=by_substatus, session=session) parameters = {'locking': RequestLocking.Locking} for req in reqs: orm_requests.update_request(request_id=req['request_id'], parameters=parameters, session=session) else: reqs = orm_requests.get_requests_by_status_type( status, request_type, time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, by_substatus=by_substatus, session=session) return reqs
def update_request_with_transforms(request_id, parameters, new_transforms=None, update_transforms=None, new_messages=None, update_messages=None, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param new_transforms: list of transforms :param update_transforms: list of transforms """ if new_transforms: for tf in new_transforms: # tf_id = orm_transforms.add_transform(**tf, session=session) # original_work = tf['transform_metadata']['original_work'] # del tf['transform_metadata']['original_work'] workflow = tf['transform_metadata']['workflow'] del tf['transform_metadata']['workflow'] work = tf['transform_metadata']['work'] tf_copy = copy.deepcopy(tf) tf_id = orm_transforms.add_transform(**tf_copy, session=session) tf['transform_id'] = tf_id # work = tf['transform_metadata']['work'] # original_work.set_work_id(tf_id, transforming=True) # original_work.set_status(WorkStatus.New) work.set_work_id(tf_id, transforming=True) work.set_status(WorkStatus.New) workflow.refresh_works() collections = generate_collections(tf) for coll in collections: collection = coll['collection'] del coll['collection'] coll['transform_id'] = tf_id coll_id = orm_collections.add_collection(**coll, session=session) # work.set_collection_id(coll, coll_id) collection.coll_id = coll_id # update transform to record the coll_id work.refresh_work() orm_transforms.update_transform( transform_id=tf_id, parameters={'transform_metadata': tf['transform_metadata']}, session=session) if update_transforms: for tr_id in update_transforms: orm_transforms.update_transform( transform_id=tr_id, parameters=update_transforms[tr_id], session=session) if new_messages: orm_messages.add_messages(new_messages, session=session) if update_messages: orm_messages.update_messages(update_messages, session=session) return orm_requests.update_request(request_id, parameters, session=session)