def broker_request(self, req, collection):
"""
Broker the req to one edge service which will serve this request.
"""
edges_canditates = []
for edge in self.edges:
free_space = edge.total_space - edge.used_space - edge.reserved_space
if free_space > collection.coll_size:
edges_canditates.append(edge)
if not edges_canditates:
self.logger.info("No edges available with enough space for this request.")
req.status = RequestStatus.WAITING
req.errors = {'message': 'No edges available with enough space for this request'}
raise NoSuitableEdges('No edges available with enough space for this request')
if 'requestbroker' in self.plugins:
try:
edge = self.plugins['requestbroker'].broker_request(req, collection, edges_canditates)
except Exception as error:
self.logger.error("Broker plugin throws an exception: %s" % (str(error), traceback.format_exc()))
raise DaemonPluginError("Broker plugin throws an exception: %s" % (str(error), traceback.format_exc()))
else:
self.logger.critical("No available request broker plugins")
raise NoPluginException("No available request broker plugins")
return edge
def get_request_data_info(self, req):
"""
Based on the request's scope and name, find the requested dataset and the size of requested dataset.
"""
try:
collection = get_collection(req.scope, req.name)
except NoObject as error:
self.logger.error("Request dataset(%s:%s) is not registered in ESS, will look for it for data management system" %
(req.scope, req.name))
if 'datafinder' in self.plugins:
try:
data = self.plugins['datafinder'].find_dataset(req.scope, req.name)
collection = {'scope': req.scope,
'name': req.name,
'collection_type': data.get('collection_type', CollectionType.DATASET),
'coll_size': data.get('size', 0),
'global_status': CollectionStatus.NEW,
'total_files': data.get('total_files', 0)}
except Exception as error:
self.logger.critical("Request dataset(%s:%s) cannot be found from data management system: %s, %s" %
(req.scope, req.name, error, traceback.format_exc()))
raise DaemonPluginError("Request dataset cannot be found from data management system")
collection_id = add_collection(**collection)
collection = get_collection(req.scope, req.name, coll_id=collection_id)
else:
self.logger.critical("No available data finder plugins")
raise NoPluginException("No available data finder plugins")
return collection
def is_stagers_alive(self):
if 'stager' in self.plugins:
try:
self.plugins['stager'].is_alive()
except Exception as error:
self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc()))
raise DaemonPluginError("Stager plugin throws an exception: %s" % (error))
return False
def stop_stagers(self):
if 'stager' in self.plugins:
try:
self.logger.info("Stopping stager plugin %s" % self.plugins['stager'])
self.plugins['stager'].stop()
except Exception as error:
self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc()))
raise DaemonPluginError("Stager plugin throws an exception: %s" % (error))
def start_stagers(self):
if 'stager' in self.plugins:
try:
self.logger.info("Starting stager plugin %s" % self.plugins['stager'])
self.plugins['stager'].set_queues(self.request_queue, self.finished_queue)
self.plugins['stager'].start()
self.logger.info("Stager plugin %s started" % self.plugins['stager'])
except Exception as error:
self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc()))
raise DaemonPluginError("Stager plugin throws an exception: %s" % (error))
else:
self.logger.critical("No available stager plugins")
raise NoPluginException("No available stager plugins")
def pre_cache(self, scope, name):
if 'precache' in self.plugins:
try:
files = self.plugins['precache'].pre_cache(scope, name)
return files
except Exception as error:
self.logger.error(
"Precacher plugin throws an exception: %s, %s" %
(error, traceback.format_exc()))
raise DaemonPluginError(
"Precacher plugin throws an exception: %s" % (error))
else:
self.logger.critical("No available pre-cache plugins")
raise NoPluginException("No available pre-cache plugins")
def start_stagers(self):
if 'stager' in self.plugins:
try:
self.logger.info("Starting stager plugin %s" %
self.plugins['stager'])
self.plugins['stager'].set_output_queue(self.output_queue)
self.plugins['stager'].start()
self.logger.info("Stager plugin %s started" %
self.plugins['stager'])
except Exception as error:
self.logger.error("Stager plugin throws an exception: %s, %s" %
(error, traceback.format_exc()))
raise DaemonPluginError(
"Stager plugin throws an exception: %s" % (error))
def stop_splitter_process(self):
if 'splitter' in self.plugins:
try:
self.logger.info("Stopping splitter plugin %s" %
self.plugins['splitter'])
self.plugins['splitter'].stop()
except Exception as error:
self.logger.error(
"Splitter plugin throws an exception: %s, %s" %
(error, traceback.format_exc()))
raise DaemonPluginError(
"Splitter plugin throws an exception: %s" % (error))
else:
self.logger.critical("No available splitter plugins")
raise NoPluginException("No available splitter plugins")
def stage_out_outputs(self, outputs):
"""
Stage out outputs
"""
if 'stager' in self.plugins:
try:
self.logger.info("Staging out outputs: %s" % outputs)
self.plugins['stager'].stage_out_outputs(outputs)
except Exception as error:
self.logger.error("Stager plugin throws an exception: %s, %s" %
(error, traceback.format_exc()))
raise DaemonPluginError(
"Stager plugin throws an exception: %s" % (error))
else:
self.logger.info(
"No stager plugin, will directly register the outputs")
for output in outputs:
self.output_queue.put(output)