def check(self): """ Check job for plausibility. :returns: if job is plausible, i.e. no failure is expected. :rtype: bool """ valid = True for composition in self.compositions: # check if workload is set if composition.jvm.workload is None: log.error('Check: composition "{0}" has no workload' .format(composition)) valid = False # check if there are no starts in client pipelines if not composition.starts: log.error('Check: composition {0} has no starts' .format(composition.name)) valid = False # check if there are cycles in client pipelines try: edgesort(composition.starts, composition.flow) except ValueError: log.exception('Check: cycle on composition "{0}"' .format(composition)) valid = False if not any(isinstance(e, Send) for e in composition.elements): log.error('Check: there is no Send in composition "{0}"' .format(composition)) valid = False if any(isinstance(e, Plot) for e in composition.elements): log.error('Check: there is a Plot in composition "{0}"' .format(composition)) valid = False # check if there are cycles in server pipeline starts = [edge.source for edge in self.server_flow if isinstance(edge.source, Receive)] if not starts: # if there are no receivers the server pipeline is not valid log.error('Check: There is no Receiver in server pipeline') valid = False else: try: edgesort(starts, self.server_flow) except ValueError: log.exception('Check: cycle in server pipeline') valid = False for flow in chain(chain(c.flow for c in self.compositions), [self.server_flow]): valid = valid and self._check_flow(flow) return valid
def test_multi_deps(self): starts = [0] edges = [Edge(0, 1), Edge(2, 1), Edge(0, 2)] order, edge_order = edgesort(starts, edges) self.assertEqual(order, [2, 1]) self.assertIn(edge_order, (edges[::-1], [edges[2], edges[0], edges[1]]))
def run(self, composition): """ Run clientside Job. :param composition: composition to run. :type composition: :class:`SystemComposition` """ # setup pomfile = os.path.join(composition.node_setting.path, 'pom.xml') setup_dependencies(pomfile, self._get_client_dependencies(composition)) composition.jvm.add_to_cp(get_classpath(pomfile)) self._composition = composition # save send for restoring send = self.send # replace with one that knows how to identify the composition if it is set if self.send is not None: self.send = partial(self.send, composition.hash()) composition.jvm.basepath = composition.node_setting.basepath _, edge_order = edgesort(composition.starts, composition.flow) for i in range(1, self.invocations + 1): log.info('Run invocation {0}'.format(i)) with tempdir(prefix='penchy-invocation{0}-'.format(i)): composition.jvm.run() log.info('Run pipeline') for sink, group in groupby(edge_order, attrgetter('sink')): kwargs = build_keys(group) if isinstance(sink, SystemFilter): kwargs[':environment:'] = self._build_environment() log.debug('Passing this input to {0}:\n{1}' .format(sink.__class__.__name__, kwargs)) try: sink.run(**kwargs) except TypeCheckError: log.error('Type check failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise except WrongInputError: log.error('Run failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise log.debug('{0} transformed input to:\n{1}' .format(sink.__class__.__name__, sink.out)) # reset state of filters for running multiple configurations composition._reset() # restore send self.send = send self._composition = None
def run_server_pipeline(self): """ Run the serverside pipeline. :raise: :exc:`ValueError` if there is no :class:`~penchy.jobs.filters.Receive` in the serverside pipeline """ # Do nothing if server_flow is empty if not self.server_flow: return starts = [edge.source for edge in self.server_flow if isinstance(edge.source, Receive)] if not starts: log.error('There is no Receiver in the serverside flow. Aborting.') raise ValueError('There is no Receiver in the serverside flow') _, edge_order = edgesort(starts, self.server_flow) # all starts are receivers, run them with the environment for start in starts: start.run(**{':environment:': self._build_environment()}) # run other filters for sink, group in groupby(edge_order, attrgetter('sink')): kwargs = build_keys(group) if isinstance(sink, SystemFilter): kwargs[':environment:'] = self._build_environment() log.debug('Passing this input to {0}:\n{1}' .format(sink.__class__.__name__, kwargs)) try: sink.run(**kwargs) except TypeCheckError: log.error('Type check failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise except WrongInputError: log.error('Run failed on component {0} and arguments {1}' .format(sink.__class__.__name__, kwargs)) raise log.debug('{0} transformed input to:\n{1}' .format(sink.__class__.__name__, sink.out))
def test_circ(self): starts = [] edges = [Edge(1, 2), Edge(2, 1)] with self.assertRaises(ValueError): edgesort(starts, edges)
def test_linear(self): starts = [0] edges = [Edge(0, 1), Edge(1, 2)] self.assertEqual(edgesort(starts, edges), ([1, 2], edges))