def broker_request(self, req, collection): """ Broker the req to one edge service which will serve this request. """ edges_canditates = [] for edge in self.edges: free_space = edge.total_space - edge.used_space - edge.reserved_space if free_space > collection.coll_size: edges_canditates.append(edge) if not edges_canditates: self.logger.info("No edges available with enough space for this request.") req.status = RequestStatus.WAITING req.errors = {'message': 'No edges available with enough space for this request'} raise NoSuitableEdges('No edges available with enough space for this request') if 'requestbroker' in self.plugins: try: edge = self.plugins['requestbroker'].broker_request(req, collection, edges_canditates) except Exception as error: self.logger.error("Broker plugin throws an exception: %s" % (str(error), traceback.format_exc())) raise DaemonPluginError("Broker plugin throws an exception: %s" % (str(error), traceback.format_exc())) else: self.logger.critical("No available request broker plugins") raise NoPluginException("No available request broker plugins") return edge
def get_request_data_info(self, req): """ Based on the request's scope and name, find the requested dataset and the size of requested dataset. """ try: collection = get_collection(req.scope, req.name) except NoObject as error: self.logger.error("Request dataset(%s:%s) is not registered in ESS, will look for it for data management system" % (req.scope, req.name)) if 'datafinder' in self.plugins: try: data = self.plugins['datafinder'].find_dataset(req.scope, req.name) collection = {'scope': req.scope, 'name': req.name, 'collection_type': data.get('collection_type', CollectionType.DATASET), 'coll_size': data.get('size', 0), 'global_status': CollectionStatus.NEW, 'total_files': data.get('total_files', 0)} except Exception as error: self.logger.critical("Request dataset(%s:%s) cannot be found from data management system: %s, %s" % (req.scope, req.name, error, traceback.format_exc())) raise DaemonPluginError("Request dataset cannot be found from data management system") collection_id = add_collection(**collection) collection = get_collection(req.scope, req.name, coll_id=collection_id) else: self.logger.critical("No available data finder plugins") raise NoPluginException("No available data finder plugins") return collection
def is_stagers_alive(self): if 'stager' in self.plugins: try: self.plugins['stager'].is_alive() except Exception as error: self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError("Stager plugin throws an exception: %s" % (error)) return False
def stop_stagers(self): if 'stager' in self.plugins: try: self.logger.info("Stopping stager plugin %s" % self.plugins['stager']) self.plugins['stager'].stop() except Exception as error: self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError("Stager plugin throws an exception: %s" % (error))
def start_stagers(self): if 'stager' in self.plugins: try: self.logger.info("Starting stager plugin %s" % self.plugins['stager']) self.plugins['stager'].set_queues(self.request_queue, self.finished_queue) self.plugins['stager'].start() self.logger.info("Stager plugin %s started" % self.plugins['stager']) except Exception as error: self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError("Stager plugin throws an exception: %s" % (error)) else: self.logger.critical("No available stager plugins") raise NoPluginException("No available stager plugins")
def pre_cache(self, scope, name): if 'precache' in self.plugins: try: files = self.plugins['precache'].pre_cache(scope, name) return files except Exception as error: self.logger.error( "Precacher plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError( "Precacher plugin throws an exception: %s" % (error)) else: self.logger.critical("No available pre-cache plugins") raise NoPluginException("No available pre-cache plugins")
def start_stagers(self): if 'stager' in self.plugins: try: self.logger.info("Starting stager plugin %s" % self.plugins['stager']) self.plugins['stager'].set_output_queue(self.output_queue) self.plugins['stager'].start() self.logger.info("Stager plugin %s started" % self.plugins['stager']) except Exception as error: self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError( "Stager plugin throws an exception: %s" % (error))
def stop_splitter_process(self): if 'splitter' in self.plugins: try: self.logger.info("Stopping splitter plugin %s" % self.plugins['splitter']) self.plugins['splitter'].stop() except Exception as error: self.logger.error( "Splitter plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError( "Splitter plugin throws an exception: %s" % (error)) else: self.logger.critical("No available splitter plugins") raise NoPluginException("No available splitter plugins")
def stage_out_outputs(self, outputs): """ Stage out outputs """ if 'stager' in self.plugins: try: self.logger.info("Staging out outputs: %s" % outputs) self.plugins['stager'].stage_out_outputs(outputs) except Exception as error: self.logger.error("Stager plugin throws an exception: %s, %s" % (error, traceback.format_exc())) raise DaemonPluginError( "Stager plugin throws an exception: %s" % (error)) else: self.logger.info( "No stager plugin, will directly register the outputs") for output in outputs: self.output_queue.put(output)