def test_multi_sinks(self): edges = [make_edge(1, (('foo', 'bar'), ('baz', 'bad'))), make_edge(2, (('foz', 'bas'), ('boz', 'bat')))] with self.assertRaises(AssertionError): build_keys(edges)
def test_multi_edge(self): edges = [make_edge(1, (('foo', 'bar'), ('baz', 'bad'))), make_edge(1, (('foz', 'bas'), ('boz', 'bat')))] self.assertDictEqual(build_keys(edges), {'bar' : 42, 'bad' : 42, 'bas' : 42, 'bat' : 42})
def run(self, composition): """ Run clientside Job. :param composition: composition to run. :type composition: :class:`SystemComposition` """ # setup pomfile = os.path.join(composition.node_setting.path, 'pom.xml') setup_dependencies(pomfile, self._get_client_dependencies(composition)) composition.jvm.add_to_cp(get_classpath(pomfile)) self._composition = composition # save send for restoring send = self.send # replace with one that knows how to identify the composition if it is set if self.send is not None: self.send = partial(self.send, composition.hash()) composition.jvm.basepath = composition.node_setting.basepath _, edge_order = edgesort(composition.starts, composition.flow) for i in range(1, self.invocations + 1): log.info('Run invocation {0}'.format(i)) with tempdir(prefix='penchy-invocation{0}-'.format(i)): composition.jvm.run() log.info('Run pipeline') for sink, group in groupby(edge_order, attrgetter('sink')): kwargs = build_keys(group) if isinstance(sink, SystemFilter): kwargs[':environment:'] = self._build_environment() log.debug('Passing this input to {0}:\n{1}' .format(sink.__class__.__name__, kwargs)) try: sink.run(**kwargs) except TypeCheckError: log.error('Type check failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise except WrongInputError: log.error('Run failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise log.debug('{0} transformed input to:\n{1}' .format(sink.__class__.__name__, sink.out)) # reset state of filters for running multiple configurations composition._reset() # restore send self.send = send self._composition = None
def test_complete_copy(self): edges = [make_edge(1, (('foo', 'bar'), ('baz', 'bad'))), make_edge(1, (('foz', 'bas'), ('boz', 'bat')))] # set edges to None for complete copy for edge in edges: edge.map_ = None self.assertDictEqual(build_keys(edges), {'foo' : 42, 'baz' : 42, 'foz' : 42, 'boz' : 42})
def run_server_pipeline(self): """ Run the serverside pipeline. :raise: :exc:`ValueError` if there is no :class:`~penchy.jobs.filters.Receive` in the serverside pipeline """ # Do nothing if server_flow is empty if not self.server_flow: return starts = [edge.source for edge in self.server_flow if isinstance(edge.source, Receive)] if not starts: log.error('There is no Receiver in the serverside flow. Aborting.') raise ValueError('There is no Receiver in the serverside flow') _, edge_order = edgesort(starts, self.server_flow) # all starts are receivers, run them with the environment for start in starts: start.run(**{':environment:': self._build_environment()}) # run other filters for sink, group in groupby(edge_order, attrgetter('sink')): kwargs = build_keys(group) if isinstance(sink, SystemFilter): kwargs[':environment:'] = self._build_environment() log.debug('Passing this input to {0}:\n{1}' .format(sink.__class__.__name__, kwargs)) try: sink.run(**kwargs) except TypeCheckError: log.error('Type check failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise except WrongInputError: log.error('Run failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise log.debug('{0} transformed input to:\n{1}' .format(sink.__class__.__name__, sink.out))
def test_single_edge(self): edges = [make_edge(1, (('foo', 'bar'), ('baz', 'bad')))] self.assertDictEqual(build_keys(edges), {'bar' : 42, 'bad' : 42})