def test_set(self): pipeline = EDDPipeline("localhost", 1234, dict(foo='')) pipeline.set({"foo": "bar"}) self.assertEqual(pipeline._config['foo'], 'bar') with self.assertRaises(FailReply) as cm: yield pipeline.set({"bar": "foo"})
def set(self, config_json): cfg = yield self._cfgjson2dict(config_json) if 'output_data_streams' in cfg: log.debug("Stripping outputs from cfg before check") # Do not check output data streams, as the only relevant thing is here # that they are consecutive outputs = cfg.pop('output_data_streams') log.debug("Pipeline set") yield EDDPipeline.set(self, cfg) log.debug("Re-adding outputs") self._config['output_data_streams'] = outputs self._configUpdated() else: EDDPipeline.set(self, cfg)
def configure(self, config_json): """ Configure the EDD backend Args: config_json: A JSON dictionary object containing configuration information """ log.info("Configuring EDD backend for processing") #log.info("Resetting data streams") #TODo: INterface? Decide if this is always done #self.__eddDataStore._dataStreams.flushdb() log.debug("Received configuration string: '{}'".format(config_json)) try: cfg = json.loads(config_json) except: log.error("Error parsing json") raise FailReply( "Cannot handle config string {} - Not valid json!".format( config_json)) if not self.__provisioned: log.debug("Not provisioned. Using full config.") # Do not use set here, as there might not be a basic config from # provisioning cfg = self.__sanitizeConfig(cfg) self._config = cfg else: yield EDDPipeline.set(self, cfg) yield self._installController(self._config['products']) cfs = json.dumps(self._config, indent=4) log.debug("Starting configuration:\n" + cfs) # Data streams are only filled in on final configure as they may # require data from the configure command of previous products. As example, the packetizer # data stream has a sync time that is propagated to other components # The components are thus configured following the dependency tree, # which is a directed acyclic graph (DAG) log.debug("Build DAG from config") dag = nx.DiGraph() for product, product_config in self._config['products'].items(): log.debug("Adding node: {}".format(product)) dag.add_node(product) if "input_data_streams" in product_config: for stream in value_list(product_config["input_data_streams"]): if not stream["source"]: log.warning( "Ignoring stream without source for DAG from {}". format(product)) continue source_product = stream["source"].split(":")[0] if source_product not in self._config['products']: raise FailReply( "{} requires data stream of unknown product {}". format(product, stream["source"])) log.debug("Connecting: {} -> {}".format( source_product, product)) dag.add_edge(source_product, product) log.debug("Checking for loops in graph") try: cycle = nx.find_cycle(dag) FailReply("Cycle detected in dependency graph: {}".format(cycle)) except nx.NetworkXNoCycle: log.debug("No loop on graph found") pass graph = "\n".join( [" {} --> {}".format(k[0], k[1]) for k in dag.edges()]) log.info("Dependency graph of products:\n{}".format(graph)) self._configuration_graph.set_value(graph) configure_results = {} configure_futures = [] @coroutine def __process_node(node): """ Wrapper to parallelize configuration of nodes. Any Node will wait for its predecessors to be done. """ #Wait for all predecessors to be finished log.debug("DAG Processing {}: Waiting for {} predecessors".format( node, len(list(dag.predecessors(node))))) for pre in dag.predecessors(node): log.debug('DAG Processing {}: waiting for {}'.format( node, pre)) while not pre in configure_results: # python3 asyncio coroutines would not run until awaited, # so we could build the graph up front and then execute it # without waiting yield tornado.gen.sleep(0.5) log.debug('DAG Processing {}: Predecessor {} done.'.format( node, pre)) if not configure_results[pre]: log.error( 'DAG Processing {}: fails due to error in predecessor {}' .format(node, pre)) configure_results[node] = False raise Return log.debug('DAG Processing {}: Predecessor {} was successfull.'. format(node, pre)) log.debug("DAG Processing {}: All predecessors done.".format(node)) try: log.debug( "DAG Processing {}: Checking input data streams for updates." .format(node)) if "input_data_streams" in self._config['products'][node]: log.debug( 'DAG Processing {}: Update input streams'.format(node)) for stream in value_list(self._config['products'][node] ["input_data_streams"]): product_name, stream_name = stream["source"].split(":") stream.update(self._config['products'][product_name] ["output_data_streams"][stream_name]) log.debug('DAG Processing {}: Set Final config'.format(node)) yield self.__controller[node].set( self._config['products'][node]) log.debug( 'DAG Processing {}: Staring configuration'.format(node)) yield self.__controller[node].configure() log.debug( "DAG Processing {}: Getting updated config".format(node)) cfg = yield self.__controller[node].getConfig() log.debug("Got: {}".format(json.dumps(cfg, indent=4))) self._config["products"][node] = cfg except Exception as E: log.error( 'DAG Processing: {} Exception cought during configuration:\n {}:{}' .format(node, type(E).__name__, E)) configure_results[node] = False else: log.debug( 'DAG Processing: {} Successfully finished configuration'. format(node)) configure_results[node] = True log.debug("Creating processing futures") configure_futures = [__process_node(node) for node in dag.nodes()] yield configure_futures self._configUpdated() log.debug("Final configuration:\n '{}'".format( json.dumps(self._config, indent=2))) failed_prcts = [ k for k in configure_results if not configure_results[k] ] if failed_prcts: raise FailReply("Failed products: {}".format( ",".join(failed_prcts))) log.info("Updating data streams in database") for productname, product in self._config["products"].items(): log.debug(" - Checking {}".format(productname)) if "output_data_streams" in product and isinstance( product["output_data_streams"], dict): for stream, streamcfg in product["output_data_streams"].items( ): key = "{}:{}".format(productname, stream) self.__eddDataStore.addDataStream(key, streamcfg) log.info("Successfully configured EDD") raise Return("Successfully configured EDD")