def get_tasks(self): """ Get tasks to process """ requests = get_requests(status=RequestStatus.ASSIGNED, edge_name=self.resource_name) self.logger.info("Main thread get %s tasks" % len(requests)) for req in requests: self.tasks.put(req) update_request(request_id=req.request_id, parameters={'status': RequestStatus.PRECACHING})
def get_tasks(self): """ Get tasks to process """ requests = get_requests(status=RequestStatus.NEW) self.logger.info("Main thread get %s tasks" % len(requests)) for req in requests: req.errors = None self.tasks.put(req) update_request(request_id=req.request_id, parameters={'status': RequestStatus.BROKERING}) if requests: self.get_resources()
def GET(self): """ Get requests. HTTP Success: 200 OK HTTP Error: 404 Not Found 500 InternalError :returns: A list containing requests. """ header('Content-Type', 'application/x-json-stream') params = input() try: status = None edge_name = None edge_id = None if 'status' in params: status = params['status'] if 'edge_name' in params: edge_name = params['edge_name'] if 'edge_id' in params: edge_id = int(params['edge_id']) reqs = get_requests(status=status, edge_name=edge_name, edge_id=edge_id) except exceptions.NoObject as error: raise self.generate_http_response(HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.ESSException as error: raise self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(format_exc()) raise self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) raise self.generate_http_response(HTTP_STATUS_CODE.OK, data=[req.to_dict() for req in reqs])
def prepare_split_request_task(self): """ Prepare split request """ requests = get_requests(status=RequestStatus.PRECACHED, edge_name=self.resource_name) if requests: self.logger.info("Main thread get %s split requests" % len(requests)) for req in requests: self.logger.info("Prepare to_split files for request %s" % req.request_id) update_request(request_id=req.request_id, parameters={'status': RequestStatus.TOSPLITTING}) self.prepare_to_split_files(req) update_request(request_id=req.request_id, parameters={'status': RequestStatus.SPLITTING})
def assign_local_requests(self): reqs = get_requests(edge_name=self.resource_name, status=RequestStatus.ASSIGNING) for req in reqs: req.status = RequestStatus.ASSIGNED update_request(req.request_id, {'status': req.status})
def finish_local_requests(self): reqs = get_requests(edge_name=self.resource_name, status=RequestStatus.SPLITTING) for req in reqs: if req.granularity_type == GranularityType.FILE: statistics = get_contents_statistics( edge_name=self.resource_name, edge_id=req.edge_id, coll_id=req.processing_meta['coll_id'], content_type=ContentType.FILE) items = {} for item in statistics: items[item.status] = items.counter if len(items.keys()) == 1 and items.keys( )[0] == ContentStatus.AVAILABLE and items.values()[0] > 0: self.logger.info( 'All files are available for request(%s): %s' % (req.request_id, items)) # To sync content info to the head service self.sync_contents(collection_scope=req.scope, collection_name=req.name, edge_name=self.resource_name, edge_id=req.edge_id, coll_id=req.processing_meta['coll_id']) req.status = RequestStatus.AVAILABLE self.logger.info("Updating request %s to status %s" % (req.request_id, req.status)) update_request(req.request_id, {'status': req.status}) if self.send_messaging: msg = { 'event_type': 'REQUEST_DONE', 'payload': { 'scope': req.scope, 'name': req.name, 'metadata': req.request_metadata }, 'created_at': date_to_str(datetime.datetime.utcnow()) } self.messaging_queue.put(msg) else: self.logger.info( 'Not all files are available for request(%s): %s' % (req.request_id, items)) if req.granularity_type == GranularityType.PARTIAL: statistics = get_contents_statistics( edge_name=self.resource_name, edge_id=req.edge_id, coll_id=req.processing_meta['coll_id'], content_type=ContentType.PARTIAL) items = {} for item in statistics: items[item.status] = item.counter if len(items.keys()) == 1 and items.keys( )[0] == ContentStatus.AVAILABLE and items.values()[0] > 0: self.logger.info( 'All partial files are available for request(%s): %s' % (req.request_id, items)) # To sync content info to the head service self.sync_contents(collection_scope=req.scope, collection_name=req.name, edge_name=self.resource_name, edge_id=req.edge_id, coll_id=req.processing_meta['coll_id']) req.status = RequestStatus.AVAILABLE self.logger.info("Updating request %s to status %s" % (req.request_id, req.status)) update_request(req.request_id, {'status': req.status}) if self.send_messaging: msg = { 'event_type': 'REQUEST_DONE', 'payload': { 'scope': req.scope, 'name': req.name, 'metadata': req.request_meta }, 'created_at': date_to_str(datetime.datetime.utcnow()) } self.messaging_queue.put(msg) else: self.logger.info( 'Not all partial files are available for request(%s): %s' % (req.request_id, items))