def testInvalidDictionaryOutput(self): b = UnitB() dag = {b: dag_pipeline.Input(b.input_type), dag_pipeline.Output(): b} with self.assertRaises(dag_pipeline.InvalidDictionaryOutput): dag_pipeline.DAGPipeline(dag) a = UnitA() dag = { a: dag_pipeline.Input(b.input_type), dag_pipeline.Output('output'): a } with self.assertRaises(dag_pipeline.InvalidDictionaryOutput): dag_pipeline.DAGPipeline(dag) a2 = UnitA() dag = { a: dag_pipeline.Input(a.input_type), a2: dag_pipeline.Input(a2.input_type), dag_pipeline.Output('output'): { 't1': a['t1'], 't2': a2['t2'] } } with self.assertRaises(dag_pipeline.InvalidDictionaryOutput): dag_pipeline.DAGPipeline(dag)
def testDAGPipelineInputAndOutputType(self): # Tests that the DAGPipeline has the correct `input_type` and # `output_type` values based on the DAG given to it. a, b, c, d = UnitA(), UnitB(), UnitC(), UnitD() dag = { a: dag_pipeline.Input(Type0), b: a['t1'], c: { 'A_data': a['t2'], 'B_data': b }, d: { '0': c['regular_data'], '1': b, '2': c['special_data'] }, dag_pipeline.Output('abcdz'): d } p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.input_type, Type0) self.assertEqual(p.output_type, {'abcdz': Type5}) dag = { a: dag_pipeline.Input(Type0), dag_pipeline.Output('t1'): a['t1'], dag_pipeline.Output('t2'): a['t2'] } p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.input_type, Type0) self.assertEqual(p.output_type, {'t1': Type1, 't2': Type2})
def testOutputConnectedToDict(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return { 'xy': [Type1(x=input_object.x, y=input_object.y)], 'z': [Type2(z=input_object.z)] } q = UnitQ() dag = {q: dag_pipeline.Input(q.input_type), dag_pipeline.Output(): q} p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.output_type, {'xy': Type1, 'z': Type2}) x, y, z = -3, 0, 8 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(output_dict, {'xy': [Type1(x, y)], 'z': [Type2(z)]}) dag = { q: dag_pipeline.Input(q.input_type), dag_pipeline.Output(): { 'xy': q['xy'], 'z': q['z'] } } p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.output_type, {'xy': Type1, 'z': Type2}) x, y, z = -3, 0, 8 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(output_dict, {'xy': [Type1(x, y)], 'z': [Type2(z)]})
def testInvalidStatisticsException(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, str, str) def transform(self, input_object): self._set_stats([statistics.Counter('stat_1', 5), 1234]) return [input_object] class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, str, str) def transform(self, input_object): self._set_stats(statistics.Counter('stat_1', 5)) return [input_object] q = UnitQ() dag = { q: dag_pipeline.Input(q.input_type), dag_pipeline.Output('output'): q } p = dag_pipeline.DAGPipeline(dag) with self.assertRaises(pipeline.InvalidStatisticsException): p.transform('hello world') r = UnitR() dag = { r: dag_pipeline.Input(q.input_type), dag_pipeline.Output('output'): r } p = dag_pipeline.DAGPipeline(dag) with self.assertRaises(pipeline.InvalidStatisticsException): p.transform('hello world')
def testDisjointGraph(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): pass class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, { 'a': Type2, 'b': Type3 }) def transform(self, input_object): pass q, r = UnitQ(), UnitR() dag = {q: dag_pipeline.Input(q.input_type), dag_pipeline.Output(): r} with self.assertRaises(dag_pipeline.NotConnectedException): dag_pipeline.DAGPipeline(dag) q, r = UnitQ(), UnitR() dag = { q: dag_pipeline.Input(q.input_type), dag_pipeline.Output(): { 'a': q, 'b': r['b'] } } with self.assertRaises(dag_pipeline.NotConnectedException): dag_pipeline.DAGPipeline(dag) # Pipelines that do not output to anywhere are not allowed. dag = { dag_pipeline.Output('output'): dag_pipeline.Input(q.input_type), q: dag_pipeline.Input(q.input_type), r: q } with self.assertRaises(dag_pipeline.NotConnectedException): dag_pipeline.DAGPipeline(dag) # Pipelines which need to be executed but don't have inputs are not allowed. dag = { dag_pipeline.Output('output'): dag_pipeline.Input(q.input_type), r: q, dag_pipeline.Output(): r } with self.assertRaises(dag_pipeline.NotConnectedException): dag_pipeline.DAGPipeline(dag)
def testBadInputOrOutputException(self): class UnitQ(pipeline.Pipeline): def __init__(self, name='UnitQ'): pipeline.Pipeline.__init__(self, Type0, Type1, name) def transform(self, input_object): pass class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, Type0) def transform(self, input_object): pass # Missing Input. q, r = UnitQ(), UnitR() dag = {r: q, dag_pipeline.Output('output'): r} with self.assertRaises(dag_pipeline.BadInputOrOutputException): dag_pipeline.DAGPipeline(dag) # Missing Output. dag = {q: dag_pipeline.Input(Type0), r: q} with self.assertRaises(dag_pipeline.BadInputOrOutputException): dag_pipeline.DAGPipeline(dag) # Multiple instances of Input with the same type IS allowed. q2 = UnitQ('UnitQ2') dag = { q: dag_pipeline.Input(Type0), q2: dag_pipeline.Input(Type0), dag_pipeline.Output(): { 'q': q, 'q2': q2 } } _ = dag_pipeline.DAGPipeline(dag) # Multiple instances with different types is not allowed. dag = { q: dag_pipeline.Input(Type0), r: dag_pipeline.Input(Type1), dag_pipeline.Output(): { 'q': q, 'r': r } } with self.assertRaises(dag_pipeline.BadInputOrOutputException): dag_pipeline.DAGPipeline(dag)
def testSingleOutputs(self): # Tests single object and dictionaries in the DAG. a, b, c, d = UnitA(), UnitB(), UnitC(), UnitD() dag = { a: dag_pipeline.Input(Type0), b: a['t1'], c: { 'A_data': a['t2'], 'B_data': b }, d: { '0': c['regular_data'], '1': b, '2': c['special_data'] }, dag_pipeline.Output('abcdz'): d } p = dag_pipeline.DAGPipeline(dag) inputs = [Type0(1, 2, 3), Type0(-1, -2, -3), Type0(3, -3, 2)] for input_object in inputs: x, y, z = input_object.x, input_object.y, input_object.z output_dict = p.transform(input_object) self.assertEqual(output_dict.keys(), ['abcdz']) results = output_dict['abcdz'] self.assertEqual(len(results), 1) result = results[0] self.assertEqual(result.a, x * 1000) self.assertEqual(result.b, y - 100) self.assertEqual(result.c, x * 1000 + z * 100) self.assertEqual(result.d, y - 100 - z * 100) self.assertEqual(result.z, z)
def get_pipeline(melody_encoder_decoder): """Returns the Pipeline instance which creates the RNN dataset. Args: melody_encoder_decoder: A melodies_lib.MelodyEncoderDecoder object. Returns: A pipeline.Pipeline instance. """ quantizer = pipelines_common.Quantizer(steps_per_beat=4) melody_extractor = pipelines_common.MonophonicMelodyExtractor( min_bars=7, min_unique_pitches=5, gap_bars=1.0, ignore_polyphonic_notes=False) encoder_pipeline = EncoderPipeline(melody_encoder_decoder) partitioner = pipelines_common.RandomPartition( tf.train.SequenceExample, ['eval_melodies', 'training_melodies'], [FLAGS.eval_ratio]) dag = { quantizer: dag_pipeline.Input(music_pb2.NoteSequence), melody_extractor: quantizer, encoder_pipeline: melody_extractor, partitioner: encoder_pipeline, dag_pipeline.Output(): partitioner } return dag_pipeline.DAGPipeline(dag)
def get_pipeline(config, eval_ratio): """Returns the Pipeline instance which creates the RNN dataset. Args: config: A MelodyRnnConfig object. eval_ratio: Fraction of input to set aside for evaluation set. Returns: A pipeline.Pipeline instance. """ quantizer = pipelines_common.Quantizer(steps_per_quarter=4) melody_extractor = melody_pipelines.MelodyExtractor( min_bars=7, max_steps=512, min_unique_pitches=5, gap_bars=1.0, ignore_polyphonic_notes=False) encoder_pipeline = EncoderPipeline(config) partitioner = pipelines_common.RandomPartition( tf.train.SequenceExample, ['eval_melodies', 'training_melodies'], [eval_ratio]) dag = { quantizer: dag_pipeline.Input(music_pb2.NoteSequence), melody_extractor: quantizer, encoder_pipeline: melody_extractor, partitioner: encoder_pipeline, dag_pipeline.Output(): partitioner } return dag_pipeline.DAGPipeline(dag)
def get_pipeline(config, eval_ratio): """Returns the Pipeline instance which creates the RNN dataset. Args: config: A DrumsRnnConfig object. eval_ratio: Fraction of input to set aside for evaluation set. Returns: A pipeline.Pipeline instance. """ quantizer = pipelines_common.Quantizer(steps_per_quarter=4) drums_extractor_train = drum_pipelines.DrumsExtractor( min_bars=7, max_steps=512, gap_bars=1.0, name='DrumsExtractorTrain') drums_extractor_eval = drum_pipelines.DrumsExtractor( min_bars=7, max_steps=512, gap_bars=1.0, name='DrumsExtractorEval') encoder_pipeline_train = EncoderPipeline(config, name='EncoderPipelineTrain') encoder_pipeline_eval = EncoderPipeline(config, name='EncoderPipelineEval') partitioner = pipelines_common.RandomPartition( music_pb2.NoteSequence, ['eval_drum_tracks', 'training_drum_tracks'], [eval_ratio]) dag = { quantizer: dag_pipeline.Input(music_pb2.NoteSequence), partitioner: quantizer, drums_extractor_train: partitioner['training_drum_tracks'], drums_extractor_eval: partitioner['eval_drum_tracks'], encoder_pipeline_train: drums_extractor_train, encoder_pipeline_eval: drums_extractor_eval, dag_pipeline.Output('training_drum_tracks'): encoder_pipeline_train, dag_pipeline.Output('eval_drum_tracks'): encoder_pipeline_eval } return dag_pipeline.DAGPipeline(dag)
def testUnequalOutputCounts(self): # Tests dictionary output type where each output list has a different size. class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): return [ Type1(x=input_object.x + i, y=input_object.y + i) for i in range(input_object.z) ] class Partitioner(pipeline.Pipeline): def __init__(self, input_type, training_set_name, test_set_name): self.training_set_name = training_set_name self.test_set_name = test_set_name pipeline.Pipeline.__init__(self, input_type, { training_set_name: input_type, test_set_name: input_type }) def transform(self, input_object): if input_object.x < 0: return { self.training_set_name: [], self.test_set_name: [input_object] } return { self.training_set_name: [input_object], self.test_set_name: [] } q = UnitQ() partition = Partitioner(q.output_type, 'training_set', 'test_set') dag = { q: dag_pipeline.Input(q.input_type), partition: q, dag_pipeline.Output('training_set'): partition['training_set'], dag_pipeline.Output('test_set'): partition['test_set'] } p = dag_pipeline.DAGPipeline(dag) x, y, z = -3, 0, 8 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(set(output_dict.keys()), set(['training_set', 'test_set'])) training_results = output_dict['training_set'] test_results = output_dict['test_set'] expected_training_results = [Type1(x + i, y + i) for i in range(-x, z)] expected_test_results = [Type1(x + i, y + i) for i in range(0, -x)] self.assertEqual(set(training_results), set(expected_training_results)) self.assertEqual(set(test_results), set(expected_test_results))
def testDuplicateNameException(self): class UnitQ(pipeline.Pipeline): def __init__(self, name='UnitQ'): pipeline.Pipeline.__init__(self, Type0, Type1, name) def transform(self, input_object): pass q, q2 = UnitQ(), UnitQ() dag = { q: dag_pipeline.Input(Type0), q2: dag_pipeline.Input(Type0), dag_pipeline.Output(): { 'q': q, 'q2': q2 } } with self.assertRaises(dag_pipeline.DuplicateNameException): dag_pipeline.DAGPipeline(dag)
def testStatistics(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) self.stats = [] def transform(self, input_object): self._set_stats( [statistics.Counter('output_count', input_object.z)]) return [ Type1(x=input_object.x + i, y=input_object.y + i) for i in range(input_object.z) ] class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, Type1) def transform(self, input_object): self._set_stats([statistics.Counter('input_count', 1)]) return [input_object] q, r = UnitQ(), UnitR() dag = { q: dag_pipeline.Input(q.input_type), r: q, dag_pipeline.Output('output'): r } p = dag_pipeline.DAGPipeline(dag, 'DAGPipelineName') for x, y, z in [(-3, 0, 8), (1, 2, 3), (5, -5, 5)]: p.transform(Type0(x, y, z)) stats_1 = p.get_stats() stats_2 = p.get_stats() self.assertEqual(stats_1, stats_2) for stat in stats_1: self.assertTrue(isinstance(stat, statistics.Counter)) names = sorted([stat.name for stat in stats_1]) self.assertEqual(names, (['DAGPipelineName_UnitQ_output_count'] + ['DAGPipelineName_UnitR_input_count'] * z)) for stat in stats_1: if stat.name == 'DAGPipelineName_UnitQ_output_count': self.assertEqual(stat.count, z) else: self.assertEqual(stat.count, 1)
def testMultiOutput(self): # Tests a pipeline.Pipeline that maps a single input to multiple outputs. class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 't1': Type1, 't2': Type2 }) def transform(self, input_object): t1 = [ Type1(x=input_object.x + i, y=input_object.y + i) for i in range(input_object.z) ] t2 = [Type2(z=input_object.z)] return {'t1': t1, 't2': t2} q, b, c = UnitQ(), UnitB(), UnitC() dag = { q: dag_pipeline.Input(Type0), b: q['t1'], c: { 'A_data': q['t2'], 'B_data': b }, dag_pipeline.Output('outputs'): c['regular_data'] } p = dag_pipeline.DAGPipeline(dag) x, y, z = 1, 2, 3 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(output_dict.keys(), ['outputs']) results = output_dict['outputs'] self.assertEqual(len(results), 3) expected_results = [ Type4((x + i) * 1000, (y + i) - 100, 0) for i in range(z) ] self.assertEqual(set(results), set(expected_results))
def testDirectConnection(self): # Tests a direct dict to dict connection in the DAG. class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return { 'xy': [Type1(x=input_object.x, y=input_object.y)], 'z': [Type2(z=input_object.z)] } class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, { 'xy': Type1, 'z': Type2 }, Type4) def transform(self, input_dict): return [ Type4(input_dict['xy'].x, input_dict['xy'].y, input_dict['z'].z) ] q, r = UnitQ(), UnitR() dag = { q: dag_pipeline.Input(q.input_type), r: q, dag_pipeline.Output('output'): r } p = dag_pipeline.DAGPipeline(dag) x, y, z = -3, 0, 8 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(output_dict, {'output': [Type4(x, y, z)]})
def testInvalidTransformOutputException(self): # This happens when the output of a pipeline's `transform` method does not # match the type signature given by the pipeline's `output_type`. class UnitQ1(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): return [Type2(1)] class UnitQ2(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): return [Type1(1, 2), Type2(1)] class UnitQ3(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): return Type1(1, 2) class UnitR1(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return {'xy': [Type1(1, 2)], 'z': [Type1(1, 2)]} class UnitR2(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return {'xy': [Type1(1, 2)]} class UnitR3(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return [{'xy': [Type1(1, 2)], 'z': Type2(1)}] class UnitR4(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return [{'xy': [Type1(1, 2), Type2(1)], 'z': [Type2(1)]}] class UnitR5(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return [{ 'xy': [Type1(1, 2), Type1(1, 3)], 'z': [Type2(1)], 'q': [] }] for pipeline_class in [ UnitQ1, UnitQ2, UnitQ3, UnitR1, UnitR2, UnitR3, UnitR4, UnitR5 ]: pipe = pipeline_class() output = (dag_pipeline.Output() if pipeline_class.__name__.startswith('UnitR') else dag_pipeline.Output('output')) dag = {pipe: dag_pipeline.Input(pipe.input_type), output: pipe} p = dag_pipeline.DAGPipeline(dag) with self.assertRaises( dag_pipeline.InvalidTransformOutputException): p.transform(Type0(1, 2, 3))
def testIntermediateUnequalOutputCounts(self): # Tests that intermediate output lists which are not the same length are # handled correctly. class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return { 'xy': [ Type1(x=input_object.x + i, y=input_object.y + i) for i in range(input_object.z) ], 'z': [Type2(z=i) for i in [-input_object.z, input_object.z]] } class Partitioner(pipeline.Pipeline): def __init__(self, input_type, training_set_name, test_set_name): self.training_set_name = training_set_name self.test_set_name = test_set_name pipeline.Pipeline.__init__(self, input_type, { training_set_name: Type0, test_set_name: Type0 }) def transform(self, input_dict): input_object = Type0(input_dict['xy'].x, input_dict['xy'].y, input_dict['z'].z) if input_object.x < 0: return { self.training_set_name: [], self.test_set_name: [input_object] } return { self.training_set_name: [input_object], self.test_set_name: [] } q = UnitQ() partition = Partitioner(q.output_type, 'training_set', 'test_set') dag = { q: dag_pipeline.Input(q.input_type), partition: { 'xy': q['xy'], 'z': q['z'] }, dag_pipeline.Output('training_set'): partition['training_set'], dag_pipeline.Output('test_set'): partition['test_set'] } p = dag_pipeline.DAGPipeline(dag) x, y, z = -3, 0, 8 output_dict = p.transform(Type0(x, y, z)) self.assertEqual(set(output_dict.keys()), set(['training_set', 'test_set'])) training_results = output_dict['training_set'] test_results = output_dict['test_set'] all_expected_results = [ Type0(x + i, y + i, zed) for i in range(0, z) for zed in [-z, z] ] expected_training_results = [ sample for sample in all_expected_results if sample.x >= 0 ] expected_test_results = [ sample for sample in all_expected_results if sample.x < 0 ] self.assertEqual(set(training_results), set(expected_training_results)) self.assertEqual(set(test_results), set(expected_test_results))
def testDependencyLoops(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): pass class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, Type0) def transform(self, input_object): pass class UnitS(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, { 'a': Type1, 'b': Type0 }, Type1) def transform(self, input_object): pass class UnitT(pipeline.Pipeline): def __init__(self, name='UnitT'): pipeline.Pipeline.__init__(self, Type0, Type0, name) def transform(self, input_object): pass q, r, s, t = UnitQ(), UnitR(), UnitS(), UnitT() dag = { q: dag_pipeline.Input(q.input_type), s: { 'a': q, 'b': r }, r: s, dag_pipeline.Output('output'): r, dag_pipeline.Output('output_2'): s } with self.assertRaises(dag_pipeline.BadTopologyException): dag_pipeline.DAGPipeline(dag) dag = { s: { 'a': dag_pipeline.Input(Type1), 'b': r }, r: s, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.BadTopologyException): dag_pipeline.DAGPipeline(dag) dag = {dag_pipeline.Output('output'): dag_pipeline.Input(Type0), t: t} with self.assertRaises(dag_pipeline.BadTopologyException): dag_pipeline.DAGPipeline(dag) t2 = UnitT('UnitT2') dag = { dag_pipeline.Output('output'): dag_pipeline.Input(Type0), t2: t, t: t2 } with self.assertRaises(dag_pipeline.BadTopologyException): dag_pipeline.DAGPipeline(dag)
def testTypeMismatchException(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_object): pass class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, { 'a': Type2, 'b': Type3 }) def transform(self, input_object): pass class UnitS(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, { 'x': Type2, 'y': Type3 }, Type4) def transform(self, input_object): pass class UnitT(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, { 'x': Type2, 'y': Type5 }, Type4) def transform(self, input_object): pass q, r, s, t = UnitQ(), UnitR(), UnitS(), UnitT() dag = { q: dag_pipeline.Input(Type1), r: q, s: r, dag_pipeline.Output('output'): s } with self.assertRaises(dag_pipeline.TypeMismatchException): dag_pipeline.DAGPipeline(dag) q2 = UnitQ() dag = { q: dag_pipeline.Input(Type0), q2: q, dag_pipeline.Output('output'): q2 } with self.assertRaises(dag_pipeline.TypeMismatchException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: q, s: { 'x': r['b'], 'y': r['a'] }, dag_pipeline.Output('output'): s } with self.assertRaises(dag_pipeline.TypeMismatchException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: q, t: r, dag_pipeline.Output('output'): t } with self.assertRaises(dag_pipeline.TypeMismatchException): dag_pipeline.DAGPipeline(dag)
def testInvalidDAGException(self): class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'a': Type1, 'b': Type2 }) def transform(self, input_object): pass class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type1, Type2) def transform(self, input_object): pass q, r = UnitQ(), UnitR() dag = { q: dag_pipeline.Input(Type0), UnitR: q, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), 'r': q, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: UnitQ, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: 123, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = {dag_pipeline.Input(Type0): q, dag_pipeline.Output(): q} with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = {q: dag_pipeline.Input(Type0), q: dag_pipeline.Output('output')} with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: { 'abc': q['a'], 'def': 123 }, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag) dag = { q: dag_pipeline.Input(Type0), r: { 123: q['a'] }, dag_pipeline.Output('output'): r } with self.assertRaises(dag_pipeline.InvalidDAGException): dag_pipeline.DAGPipeline(dag)
def testNoPipelines(self): dag = {dag_pipeline.Output('output'): dag_pipeline.Input(Type0)} p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.transform(Type0(1, 2, 3)), {'output': [Type0(1, 2, 3)]})
def testNoOutputs(self): # Test that empty lists or dicts as intermediate or final outputs don't # break anything. class UnitQ(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, { 'xy': Type1, 'z': Type2 }) def transform(self, input_object): return {'xy': [], 'z': []} class UnitR(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, { 'xy': Type1, 'z': Type2 }, Type4) def transform(self, input_dict): return [ Type4(input_dict['xy'].x, input_dict['xy'].y, input_dict['z'].z) ] class UnitS(pipeline.Pipeline): def __init__(self): pipeline.Pipeline.__init__(self, Type0, Type1) def transform(self, input_dict): return [] q, r, s = UnitQ(), UnitR(), UnitS() dag = { q: dag_pipeline.Input(Type0), r: q, dag_pipeline.Output('output'): r } p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.transform(Type0(1, 2, 3)), {'output': []}) dag = { q: dag_pipeline.Input(Type0), s: dag_pipeline.Input(Type0), r: { 'xy': s, 'z': q['z'] }, dag_pipeline.Output('output'): r } p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.transform(Type0(1, 2, 3)), {'output': []}) dag = {s: dag_pipeline.Input(Type0), dag_pipeline.Output('output'): s} p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.transform(Type0(1, 2, 3)), {'output': []}) dag = {q: dag_pipeline.Input(Type0), dag_pipeline.Output(): q} p = dag_pipeline.DAGPipeline(dag) self.assertEqual(p.transform(Type0(1, 2, 3)), {'xy': [], 'z': []})