def get_tasks(self):
"""
Get tasks to process
"""
requests = get_requests(status=RequestStatus.ASSIGNED,
edge_name=self.resource_name)
self.logger.info("Main thread get %s tasks" % len(requests))
for req in requests:
self.tasks.put(req)
update_request(request_id=req.request_id,
parameters={'status': RequestStatus.PRECACHING})
def get_tasks(self):
"""
Get tasks to process
"""
requests = get_requests(status=RequestStatus.NEW)
self.logger.info("Main thread get %s tasks" % len(requests))
for req in requests:
req.errors = None
self.tasks.put(req)
update_request(request_id=req.request_id, parameters={'status': RequestStatus.BROKERING})
if requests:
self.get_resources()
def GET(self):
"""
Get requests.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: A list containing requests.
"""
header('Content-Type', 'application/x-json-stream')
params = input()
try:
status = None
edge_name = None
edge_id = None
if 'status' in params:
status = params['status']
if 'edge_name' in params:
edge_name = params['edge_name']
if 'edge_id' in params:
edge_id = int(params['edge_id'])
reqs = get_requests(status=status,
edge_name=edge_name,
edge_id=edge_id)
except exceptions.NoObject as error:
raise self.generate_http_response(HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.ESSException as error:
raise self.generate_http_response(HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
raise self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
raise self.generate_http_response(HTTP_STATUS_CODE.OK,
data=[req.to_dict() for req in reqs])
def prepare_split_request_task(self):
"""
Prepare split request
"""
requests = get_requests(status=RequestStatus.PRECACHED,
edge_name=self.resource_name)
if requests:
self.logger.info("Main thread get %s split requests" %
len(requests))
for req in requests:
self.logger.info("Prepare to_split files for request %s" %
req.request_id)
update_request(request_id=req.request_id,
parameters={'status': RequestStatus.TOSPLITTING})
self.prepare_to_split_files(req)
update_request(request_id=req.request_id,
parameters={'status': RequestStatus.SPLITTING})
def assign_local_requests(self):
reqs = get_requests(edge_name=self.resource_name,
status=RequestStatus.ASSIGNING)
for req in reqs:
req.status = RequestStatus.ASSIGNED
update_request(req.request_id, {'status': req.status})
def finish_local_requests(self):
reqs = get_requests(edge_name=self.resource_name,
status=RequestStatus.SPLITTING)
for req in reqs:
if req.granularity_type == GranularityType.FILE:
statistics = get_contents_statistics(
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'],
content_type=ContentType.FILE)
items = {}
for item in statistics:
items[item.status] = items.counter
if len(items.keys()) == 1 and items.keys(
)[0] == ContentStatus.AVAILABLE and items.values()[0] > 0:
self.logger.info(
'All files are available for request(%s): %s' %
(req.request_id, items))
# To sync content info to the head service
self.sync_contents(collection_scope=req.scope,
collection_name=req.name,
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'])
req.status = RequestStatus.AVAILABLE
self.logger.info("Updating request %s to status %s" %
(req.request_id, req.status))
update_request(req.request_id, {'status': req.status})
if self.send_messaging:
msg = {
'event_type': 'REQUEST_DONE',
'payload': {
'scope': req.scope,
'name': req.name,
'metadata': req.request_metadata
},
'created_at':
date_to_str(datetime.datetime.utcnow())
}
self.messaging_queue.put(msg)
else:
self.logger.info(
'Not all files are available for request(%s): %s' %
(req.request_id, items))
if req.granularity_type == GranularityType.PARTIAL:
statistics = get_contents_statistics(
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'],
content_type=ContentType.PARTIAL)
items = {}
for item in statistics:
items[item.status] = item.counter
if len(items.keys()) == 1 and items.keys(
)[0] == ContentStatus.AVAILABLE and items.values()[0] > 0:
self.logger.info(
'All partial files are available for request(%s): %s' %
(req.request_id, items))
# To sync content info to the head service
self.sync_contents(collection_scope=req.scope,
collection_name=req.name,
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'])
req.status = RequestStatus.AVAILABLE
self.logger.info("Updating request %s to status %s" %
(req.request_id, req.status))
update_request(req.request_id, {'status': req.status})
if self.send_messaging:
msg = {
'event_type': 'REQUEST_DONE',
'payload': {
'scope': req.scope,
'name': req.name,
'metadata': req.request_meta
},
'created_at':
date_to_str(datetime.datetime.utcnow())
}
self.messaging_queue.put(msg)
else:
self.logger.info(
'Not all partial files are available for request(%s): %s'
% (req.request_id, items))
