def test_graph_feedback_do_not_develop_sub_graph(self): SimulExec = dfp.DataflowEnvironment() SimulExec.set_call_rets('position') SimulExec.set_call_args('trade') SimulExec.add_node('Execution', execution) SimulExec.add_node('PositionUpdate', add) SimulExec.add_node('LaggedPosition', dfp.Lag(0)) SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a') SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b') SimulExec.add_edge('PositionUpdate', 'LaggedPosition') SimulExec.add_edge_call_args('Execution', 'trade') SimulExec.add_edge_call_rets('LaggedPosition', 'position') SimulExec.start() g = dfp.DataflowEnvironment() g.add_node('Decision', decision) g.add_node('SimulExec', SimulExec) g.add_edge('Decision', 'SimulExec', 'trade', 'trade') g.add_edge('SimulExec', 'Decision', 'position', 'position') g.set_call_args('obs') g.set_call_rets('position') g.add_edge_call_args('Decision', 'observation') g.add_edge_call_rets('SimulExec', 'position', 'position') f = lambda: g.start(develop=False) self.assertRaises(dfpe.StalledDataflowCallError, f)
def test_terminal_node_data(self): g = dfp.DataflowEnvironment() g.add_node('Decision', decision, rets='trade') g.add_node('Execution', execution, rets='trade') g.add_node('PositionUpdate', add, args='position,trade', rets='position') g.add_node('LaggedPosition', dfp.Lag(0), args='position', rets='position') g.add_edge('LaggedPosition', 'Decision') g.add_edge('Decision', 'Execution') g.add_edge('Execution', 'PositionUpdate') g.add_edge('LaggedPosition', 'PositionUpdate') g.add_edge('PositionUpdate', 'LaggedPosition') g.set_call_args('observation') g.set_call_rets('position') g.add_edge_call_args('Decision') g.add_edge_call_rets('LaggedPosition') g.start() datas = [] for __ in range(10): data = g(observation=10) datas.append(data.position) assert datas[-1] == 89.10000000000001
def test_graph_feedback(self): SimulExec = dfp.DataflowEnvironment() SimulExec.set_call_rets('position') SimulExec.set_call_args('trade') SimulExec.add_node('Execution', execution) SimulExec.add_node('PositionUpdate', add) SimulExec.add_node('LaggedPosition', dfp.Lag(0)) SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a') SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b') SimulExec.add_edge('PositionUpdate', 'LaggedPosition') SimulExec.add_edge_call_args('Execution', 'trade') SimulExec.add_edge_call_rets('LaggedPosition', 'position') SimulExec.start() g = dfp.DataflowEnvironment() g.add_node('Decision', decision) g.add_node('SimulExec', SimulExec) g.add_edge('Decision', 'SimulExec', 'trade', 'trade') g.add_edge('SimulExec', 'Decision', 'position', 'position') g.set_call_args('obs') g.set_call_rets('position') g.add_edge_call_args('Decision', 'observation') g.add_edge_call_rets('SimulExec', 'position') g.start(develop=True) datas = [g(obs=10) for __ in range(10)] assert datas[-1].position == 89.10000000000001
def test_use_graph_as_callable_automatic_edges_2_args(self): g = dfp.DataflowEnvironment() g.add_node('Decision', decision, rets='trade') g.add_node('Execution', execution, args='trade', rets='trade') g.add_node('PositionUpdate', add) g.add_node('LaggedPosition', dfp.Lag(0)) g.add_edge('LaggedPosition', 'Decision', 'position') ############################### # call add_edge with 2 arguments ############################### g.add_edge('Decision', 'Execution') ############################### g.add_edge('Execution', 'PositionUpdate', 'trade', 'a') g.add_edge('LaggedPosition', 'PositionUpdate', 'b') g.add_edge('PositionUpdate', 'LaggedPosition') g.set_call_args('obs') g.set_call_rets('position') g.add_edge_call_args('Decision', 'obs', 'observation') g.add_edge_call_rets('LaggedPosition', 'position') g.start() datas = [] for __ in range(10): data = g(obs=10) datas.append(data.position) assert datas[-1] == 89.10000000000001
def test_terminal_node_data_no_names(self): # declare functions position_update = add # lambda position, trade : position + trade pos_lag = dfp.Lag(0) g = dfp.DataflowEnvironment() g.args = 'observation' g.rets = 'position' g.add_node(decision) g.add_node(execution, rets='trade') g.add_node(position_update, args='position,trade') g.add_node(pos_lag, args='position', rets='position') g.add_edge(pos_lag, decision) g.add_edge(decision, execution) g.add_edge(execution, position_update) g.add_edge(pos_lag, position_update) g.add_edge(position_update, pos_lag) g.add_edge_call_args(decision) g.add_edge_call_rets(pos_lag) g.start() datas = [] for __ in range(10): data = g(observation=10) datas.append(data.position) assert datas[-1] == 89.10000000000001
def test_terminal_node_data_with_tasks_and_automatic_link(self): g = dfp.DataflowEnvironment() g.args = 'observation' g.add_task('Decision', decision, rets='trade', filters=dict(args=['LaggedPosition'])) g.add_edge_call_args('Decision') g.add_task('Execution', execution, rets='trade') g.add_task('PositionUpdate', add, args='trade, position', rets='position', filters=dict(args=['Execution', 'LaggedPosition'])) g.add_task('LaggedPosition', dfp.Lag(0), args='position', rets='position') g.rets = 'position' g.add_edge_call_rets('LaggedPosition') g.start() datas = [] for __ in range(10): data = g(observation=10) datas.append(data.position) assert datas[-1] == 89.10000000000001
def test_lag(): lag = dfp.Lag(10) assert lag(11) == 10 assert lag('a') == 11 assert lag.reset() == 'a' assert lag('b') is None assert lag.reset() == 'b'
def setUp(self): self.terminal_node = DataAppender() self.mlag = dfp.Lag(0) def data_gen(): for i in range(10): yield [i + 1], {} self.data_gen = data_gen
def test_lag_state(): lag = dfp.DataflowEnvironment() lag.args = 'lag_in' lag.add_task('lag', dfp.Lag(None)) lag.rets = 'lag_out' lag.add_edge_call_rets('lag') lag.start() assert lag(11).lag_out == None assert lag('a').lag_out == 11 assert lag(None).lag_out == 'a' assert lag('b').lag_out is None assert lag(None).lag_out == 'b'
def test_graph_lqg_feedback(self): """ epsilonp variable is set in order to have coverage on the "self.add_edge(*edge)" part of the code. """ # Make SimulExec SimulExec = dfp.DataflowEnvironment(name='SimulExec') SimulExec.set_call_rets('position') SimulExec.set_call_args('trade') SimulExec.add_node('Execution', execution) SimulExec.add_node('PositionUpdate', add) SimulExec.add_node('LaggedPosition', dfp.Lag(0)) SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a') SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b') SimulExec.add_edge('PositionUpdate', 'LaggedPosition') SimulExec.add_edge_call_args('Execution', 'trade') SimulExec.add_edge_call_rets('LaggedPosition', 'position') SimulExec.start() # Make Market Plant MarketPlant = dfp.DataflowEnvironment(name='MarketPlant') MarketPlant.set_call_rets('position,observation, epsilonp') MarketPlant.set_call_args('trade,observation, epsilonp') MarketPlant.add_edge_call_args_rets('epsilonp') MarketPlant.add_node('SimulExec', SimulExec) MarketPlant.add_edge_call_args_rets('observation') MarketPlant.add_edge_call_rets('SimulExec', 'position', 'position') MarketPlant.add_edge_call_args('SimulExec', 'trade', 'trade') MarketPlant.start() # Make final Dataflow def decision(epsilonp, position, observation): return {'trade': 10, 'espilonp': epsilonp} LQG = dfp.DataflowEnvironment(name='LQG') LQG.add_node('Decision', decision) LQG.add_node('MarketPlant', MarketPlant) LQG.add_node('Data', DataGenerator()) LQG.add_edge('Decision', 'MarketPlant', 'trade', 'trade') LQG.add_edge('MarketPlant', 'Decision') LQG.add_edge('Data', 'MarketPlant', 'observations', 'observation') LQG.set_call_args('epsilonp') LQG.add_edge_call_args('MarketPlant') LQG.set_call_rets('position, epsilonp') LQG.add_edge_call_rets('MarketPlant') LQG.start(develop=True, level=2) datas = [LQG(0.1) for __ in range(10)] assert datas[-1].position == 89.10000000000001
def setUp(self): self.nodes = { 'Decision': decision, 'Execution': execution, 'PositionUpdate': add, 'LaggedPosition': dfp.Lag(0), } self.edges = [ ('LaggedPosition', 'Decision', 'position'), ('Decision', 'Execution', 'trade', 'trade'), ('Execution', 'PositionUpdate', 'trade', 'a'), ('LaggedPosition', 'PositionUpdate', 'b'), ('PositionUpdate', 'LaggedPosition'), ]
def setUp(self): self.terminal_node = DataAppender() self.nodes = { 'Data': DataGenerator(), 'Decision': decision, 'Execution': execution, 'PositionUpdate': add, 'LaggedPosition': dfp.Lag(0), 'Terminal Node': self.terminal_node, } self.edges = [ ('Data', 'Decision', 'observations', 'observation'), ('LaggedPosition', 'Decision', 'position'), ('Decision', 'Execution', 'trade', 'trade'), ('Execution', 'PositionUpdate', 'trade', 'a'), ('LaggedPosition', 'PositionUpdate', 'b'), ('PositionUpdate', 'LaggedPosition'), ('LaggedPosition', 'Terminal Node', 'data'), ]
def test_terminal_node_data_with_tasks_and_automatic_link_no_names(self): # declare functions position_update = lambda position, trade: position + trade pos_lag = dfp.Lag(0) g = dfp.DataflowEnvironment() g.args = 'observation' g.rets = 'position' g.add_task(decision, filters=dict(args=[pos_lag])) g.add_edge_call_args(decision) g.add_task(execution) g.add_task(position_update, filters=dict(args=[execution, pos_lag])) g.add_task(pos_lag) g.add_edge_call_rets(pos_lag) g.start() datas = [] for __ in range(10): data = g(observation=10) datas.append(data.position) assert datas[-1] == 89.10000000000001
def add_lag(self, n, initial_state, attr_dict=None, **attr): """ Add a Lag node to the DataflowEnvironment Parameters ---------- :param n : node A node can be any hashable Python object except None. :param initial_state: The initial state of the Lag node :param attr_dict: dictionary, optional (default= no attributes) Dictionary of node attributes. Key/value pairs will :param attr: keyword arguments, optional Set attributes using key=value. """ func = dfp.Lag(initial_state) self.add_node(n, func=func, attr_dict=attr_dict, detect_func=False, **attr)
def test_workflow_with_lag(): """ We test a workkflow with a feedback implemented with a lag """ results = [0, 1000, 11001, 111012, 1111123, 11112234, 111123345, 1111234456, 11112345567, 111123456678, 1111234567789] attr_f = {'color': 'red'} data_in = DataGenerator() data_out = DataAppender() mlag = dfp.Lag(0) dtf = dfp.DataflowEnvironment() dtf.add_gentask('indata', data_in.gen, initial=True) dtf.add_task('f', f2, filters=dict(args=['indata', 'lag', ]), **attr_f) dtf.add_task('g', g, filters=dict(args=['f'])) dtf.add_task('lag', mlag, filters=dict(args=['g'])) dtf.add_task('terminal', data_out.func, filters=dict(args=['lag'])) dtf.start() dtf.run() assert data_out.out == results
def test_coroutine_from_callable_obj(): mlag = dfp.Lag(0) f_lag = func_from_coroutine(coroutine_from_func(mlag)) assert 0 == f_lag(10) assert 10 == f_lag(11)
def test_graph_lqg_feedback2(self): # Make SimulExec SimulExec = dfp.DataflowEnvironment(name='SimulExec') SimulExec.set_call_rets('position') SimulExec.set_call_args('trade') SimulExec.add_node('Execution', execution) SimulExec.add_node('PositionUpdate', add) SimulExec.add_node('LaggedPosition', dfp.Lag(0)) SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a') SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b') SimulExec.add_edge('PositionUpdate', 'LaggedPosition') SimulExec.add_edge_call_args('Execution', 'trade') SimulExec.add_edge_call_rets('LaggedPosition', 'position') SimulExec.start(develop=False) # Make Market Plant MarketPlant = dfp.DataflowEnvironment(name='MarketPlant') MarketPlant.add_node('SimulExec', SimulExec) MarketPlant.set_call_rets('position,observation') MarketPlant.set_call_args('trade,observation') MarketPlant.add_edge_call_args_rets('observation') MarketPlant.add_edge_call_rets('SimulExec', 'position', 'position') MarketPlant.add_edge_call_args('SimulExec', 'trade', 'trade') MarketPlant.start(develop=False) # Make Decision Graph DecisionGraph = dfp.DataflowEnvironment(name='DecisionGraph') DecisionGraph.add_node('Decision', decision) DecisionGraph.set_call_rets('trade') DecisionGraph.set_call_args('observation,position') DecisionGraph.add_edge_call_rets('Decision', 'trade', 'trade') DecisionGraph.add_edge_call_args('Decision', 'position', 'position') DecisionGraph.add_edge_call_args('Decision', 'observation', 'observation') DecisionGraph.start(develop=False) # Make final Dataflow LQG = dfp.DataflowEnvironment(name='LQG') LQG.add_node('Decision', DecisionGraph) LQG.add_node('MarketPlant', MarketPlant) LQG.add_node('Data', DataGenerator()) LQG.add_edge('Decision', 'MarketPlant', 'trade', 'trade') LQG.add_edge('MarketPlant', 'Decision', 'position', 'position') LQG.add_edge('MarketPlant', 'Decision', 'observation') LQG.add_edge('Data', 'MarketPlant', 'observations', 'observation') LQG.set_call_rets('position') LQG.add_edge_call_rets('MarketPlant', 'position', 'position') # First the graph is develop only at the first level assert len(LQG) == 3 f = lambda: LQG.start(develop=True) self.assertRaises(dfpe.StalledDataflowCallError, f) # We develop the second level LQG.start(develop=True, level=2) assert len(LQG) == 5 datas = [LQG() for __ in range(10)] assert datas[-1].position == 89.10000000000001
def test_use_graph_as_callable_test_args(self): g = dfp.DataflowEnvironment() g.add_node('Decision', decision, args='test_obs,test_pos') g.add_node('LaggedPosition', dfp.Lag(0)) f = lambda: g.add_edge('LaggedPosition', 'Decision', 'position') self.assertRaises(dfpe.WrongEdgeArgsError, f)