def test_flow_with_filter(): flow = dflow.DataFlow() @flow.factory(requires=['a', 'b'], provides='c') def div(a, b): return a / b @flow.filter('b') def filter_zero(b): return b if b else 1e-7 pipe = dflow.SerialPipeline(flow) assert pipe.product('c', dict(a=4, b=2)) == 2 assert pipe.product('c', dict(a=1, b=0)) != float('inf') flow = dflow.DataFlow() @flow.factory(requires=['a', 'b'], provides='c') def div(a, b): return a / b @flow.filter(['a', 'b']) def filter_zero(a, b): return a, (b if b else 1e-7) pipe = dflow.SerialPipeline(flow) assert pipe.product('c', dict(a=0, b=2)) == 0 assert pipe.product('c', dict(a=0, b=0)) == 0
def test_filter(): flow = dflow.DataFlow() print('here') @flow.filter("number") def number_filter(x: int) -> int: return x * x @flow.filter("name") def name_filter(name: str) -> str: return '{}{}'.format(name[0].upper(), name[1:].lower()) assert number_filter(2) == 4 assert flow.operation(number_filter)(2) == 4 assert flow.operation(number_filter).flow is flow assert typing.cast(dflow.Filter, flow.operation(number_filter)).fields == ('number', ) assert name_filter('tOM') == 'Tom' assert flow.operation(name_filter)('tOM') == 'Tom' assert flow.operation(name_filter).flow is flow assert typing.cast(dflow.Filter, flow.operation(name_filter)).fields == ('name', ) assert flow._filters['name'] == flow.operation(name_filter) assert flow._filters['number'] == flow.operation(number_filter)
def test_basic_flow(): flow = dflow.DataFlow() @flow.factory(requires=['a', 'b'], provides=['e']) def multiply(a, b): return a * b @flow.factory(requires=['c', 'd'], provides=['f']) def division(c, d): return c / d @flow.factory(requires=['e', 'f'], provides=['g']) def add(e, f): return e + f pipe = dflow.SerialPipeline(flow) a, b, c, d = 1, 2, 3, 4 e = a * b f = c / d g = e + f inputs = dict(a=1, b=2, c=3, d=4) assert len(pipe._cache_for_route) == 0 assert pipe.product('g', inputs) == g assert len(pipe._cache_for_route) == 1 assert pipe.product(['e', 'f', 'g'], inputs) == (e, f, g) assert len(pipe._cache_for_route) == 2 assert pipe.exec(add, inputs) == g
def test_missing_inputs(): flow = dflow.DataFlow() @flow.factory(requires=['e', 'd'], provides='g') def compute_g(e, f): return e + f @flow.factory(requires=['a', 'f'], provides=['e']) def compute_e(a, f): return a * f @flow.factory(requires=['b', 'c'], provides='f') def compute_f(b, c): return b - c pipe = dflow.SerialPipeline(flow) inputs = dict(a=1, b=2, c=3, d=4) assert pipe.product('g', inputs) == 3 del inputs['c'] err = None try: pipe.product('g', inputs) except TypeError as e: err = e assert isinstance(err, TypeError)
def test_complex_flow(): flow = dflow.DataFlow() @flow.filter(['a', 'b']) def filter_a_b(a, b): return a * 2, b + 1 @flow.filter('c') def filter_c(_): return 0 @flow.factory(requires=['a', 'b'], provides='g') def compute_g(a, b): return a * b @flow.factory(requires=['g', 'b'], provides='h') def compute_h(g, b): return g + b @flow.factory(requires=['c', 'd'], provides='i') def compute_i(c, d): return c - d @flow.factory(requires=['d', 'k'], provides='j') def compute_j(d, k): return d * k * 2 @flow.factory(requires=['e', 'f'], provides='k') def compute_k(e, f): return e * f / 5 @flow.factory(requires=['h', 'i'], provides='l') def compute_l(h, i): return h / i @flow.factory(requires=['i', 'd', 'j', 'k'], provides=['m', 'n']) def compute_m(i, d, j, k): return i + d + j, j / k inputs = dict(a=1, b=2, c=3, d=4, e=5, f=6) a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 a, b = filter_a_b(a, b) c = filter_c(c) g = compute_g(a, b) h = compute_h(g, b) i = compute_i(c, d) k = compute_k(e, f) j = compute_j(d, k) l = compute_l(h, i) m, n = compute_m(i, d, j, k) pipe = dflow.SerialPipeline(flow) assert pipe.product(['l', 'm', 'n'], inputs) == (l, m, n)
def test_multi_output(): flow = dflow.DataFlow() @flow.factory(requires='a', provides=['b', 'c']) def get_b_c(a): return a + 1, a + 2 @flow.factory(requires=['b', 'c'], provides='d') def get_d(b, c): return b * c pipe = dflow.SerialPipeline(flow) assert pipe.product('d', dict(a=1)) == 6
def test_circle_flow(): flow = dflow.DataFlow() @flow.factory(requires=['a', 'b'], provides=['c']) def compute_c(a, b): return a + b @flow.factory(requires=['e'], provides=['b']) def compute_e(b): return b + 1 @flow.factory(requires=['c', 'd'], provides=['e']) def compute_e(c, d): return c * d pipe = dflow.SerialPipeline(flow) err = None try: pipe.product('e', inputs=dict(a=1, d=2)) except ValueError as e: err = e assert type(err) == dflow.CircularDependence
self.valid_data_flow.vocabs) print('src:' + input_text) print('output: ' + ' '.join(output_text)) print('target: ' + ' '.join(target_text)) return bleu.compute_bleu(target_results, output_results)[0] * 100 if __name__ == '__main__': pid = os.getpid() if not check_dir(dataflow.lstm_save_dir): create_dir(dataflow.lstm_save_dir) print('loading training data...') train_data_flow = dataflow.DataFlow(dataflow.batch_size, data_dir=dataflow.data_path + 'train/') print('loading evaluation data...') valid_data_flow = dataflow.DataFlow(dataflow.batch_size, data_dir=dataflow.data_path + 'test/', shuffle=True) q = queue.Queue(maxsize=100) pt = Producter(queue=q, data_loader=train_data_flow, step=maxstep) ce = Consumer(step=maxstep, queue=q, pid=pid, voc_size=train_data_flow.vocab_size, valid_data_flow=valid_data_flow) pt.start() ce.start()