Esempio n. 1
0
 def test_multi_sinks(self):
     edges = [make_edge(1, (('foo', 'bar'),
                            ('baz', 'bad'))),
              make_edge(2, (('foz', 'bas'),
                            ('boz', 'bat')))]
     with self.assertRaises(AssertionError):
         build_keys(edges)
Esempio n. 2
0
 def test_multi_edge(self):
     edges = [make_edge(1, (('foo', 'bar'),
                            ('baz', 'bad'))),
              make_edge(1, (('foz', 'bas'),
                            ('boz', 'bat')))]
     self.assertDictEqual(build_keys(edges),
                          {'bar' : 42,
                           'bad' : 42,
                           'bas' : 42,
                           'bat' : 42})
Esempio n. 3
0
    def run(self, composition):
        """
        Run clientside Job.

        :param composition: composition to run.
        :type composition: :class:`SystemComposition`
        """
        # setup
        pomfile = os.path.join(composition.node_setting.path, 'pom.xml')
        setup_dependencies(pomfile, self._get_client_dependencies(composition))
        composition.jvm.add_to_cp(get_classpath(pomfile))
        self._composition = composition

        # save send for restoring
        send = self.send
        # replace with one that knows how to identify the composition if it is set
        if self.send is not None:
            self.send = partial(self.send, composition.hash())

        composition.jvm.basepath = composition.node_setting.basepath

        _, edge_order = edgesort(composition.starts, composition.flow)

        for i in range(1, self.invocations + 1):
            log.info('Run invocation {0}'.format(i))
            with tempdir(prefix='penchy-invocation{0}-'.format(i)):
                composition.jvm.run()

        log.info('Run pipeline')
        for sink, group in groupby(edge_order, attrgetter('sink')):
            kwargs = build_keys(group)
            if isinstance(sink, SystemFilter):
                kwargs[':environment:'] = self._build_environment()
            log.debug('Passing this input to {0}:\n{1}'
                      .format(sink.__class__.__name__,
                              kwargs))
            try:
                sink.run(**kwargs)
            except TypeCheckError:
                log.error('Type check failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            except WrongInputError:
                log.error('Run failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            log.debug('{0} transformed input to:\n{1}'
                      .format(sink.__class__.__name__, sink.out))

        # reset state of filters for running multiple configurations
        composition._reset()
        # restore send
        self.send = send
        self._composition = None
Esempio n. 4
0
    def test_complete_copy(self):
        edges = [make_edge(1, (('foo', 'bar'),
                               ('baz', 'bad'))),
                 make_edge(1, (('foz', 'bas'),
                               ('boz', 'bat')))]
        # set edges to None for complete copy
        for edge in edges:
            edge.map_ = None

        self.assertDictEqual(build_keys(edges),
                             {'foo' : 42,
                              'baz' : 42,
                              'foz' : 42,
                              'boz' : 42})
Esempio n. 5
0
    def run_server_pipeline(self):
        """
        Run the serverside pipeline.

        :raise: :exc:`ValueError` if there is no
                :class:`~penchy.jobs.filters.Receive` in the serverside pipeline
        """
        # Do nothing if server_flow is empty
        if not self.server_flow:
            return

        starts = [edge.source for edge in self.server_flow
                  if isinstance(edge.source, Receive)]

        if not starts:
            log.error('There is no Receiver in the serverside flow. Aborting.')
            raise ValueError('There is no Receiver in the serverside flow')

        _, edge_order = edgesort(starts, self.server_flow)

        # all starts are receivers, run them with the environment
        for start in starts:
            start.run(**{':environment:': self._build_environment()})

        # run other filters
        for sink, group in groupby(edge_order, attrgetter('sink')):
            kwargs = build_keys(group)
            if isinstance(sink, SystemFilter):
                kwargs[':environment:'] = self._build_environment()
            log.debug('Passing this input to {0}:\n{1}'
                      .format(sink.__class__.__name__,
                              kwargs))
            try:
                sink.run(**kwargs)
            except TypeCheckError:
                log.error('Type check failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            except WrongInputError:
                log.error('Run failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            log.debug('{0} transformed input to:\n{1}'
                      .format(sink.__class__.__name__, sink.out))
Esempio n. 6
0
 def test_single_edge(self):
     edges = [make_edge(1, (('foo', 'bar'),
                            ('baz', 'bad')))]
     self.assertDictEqual(build_keys(edges),
                          {'bar' : 42,
                           'bad' : 42})