def test_graph_feedback_do_not_develop_sub_graph(self):
SimulExec = dfp.DataflowEnvironment()
SimulExec.set_call_rets('position')
SimulExec.set_call_args('trade')
SimulExec.add_node('Execution', execution)
SimulExec.add_node('PositionUpdate', add)
SimulExec.add_node('LaggedPosition', dfp.Lag(0))
SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a')
SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b')
SimulExec.add_edge('PositionUpdate', 'LaggedPosition')
SimulExec.add_edge_call_args('Execution', 'trade')
SimulExec.add_edge_call_rets('LaggedPosition', 'position')
SimulExec.start()
g = dfp.DataflowEnvironment()
g.add_node('Decision', decision)
g.add_node('SimulExec', SimulExec)
g.add_edge('Decision', 'SimulExec', 'trade', 'trade')
g.add_edge('SimulExec', 'Decision', 'position', 'position')
g.set_call_args('obs')
g.set_call_rets('position')
g.add_edge_call_args('Decision', 'observation')
g.add_edge_call_rets('SimulExec', 'position', 'position')
f = lambda: g.start(develop=False)
self.assertRaises(dfpe.StalledDataflowCallError, f)
def test_terminal_node_data(self):
g = dfp.DataflowEnvironment()
g.add_node('Decision', decision, rets='trade')
g.add_node('Execution', execution, rets='trade')
g.add_node('PositionUpdate',
add,
args='position,trade',
rets='position')
g.add_node('LaggedPosition',
dfp.Lag(0),
args='position',
rets='position')
g.add_edge('LaggedPosition', 'Decision')
g.add_edge('Decision', 'Execution')
g.add_edge('Execution', 'PositionUpdate')
g.add_edge('LaggedPosition', 'PositionUpdate')
g.add_edge('PositionUpdate', 'LaggedPosition')
g.set_call_args('observation')
g.set_call_rets('position')
g.add_edge_call_args('Decision')
g.add_edge_call_rets('LaggedPosition')
g.start()
datas = []
for __ in range(10):
data = g(observation=10)
datas.append(data.position)
assert datas[-1] == 89.10000000000001
def test_graph_feedback(self):
SimulExec = dfp.DataflowEnvironment()
SimulExec.set_call_rets('position')
SimulExec.set_call_args('trade')
SimulExec.add_node('Execution', execution)
SimulExec.add_node('PositionUpdate', add)
SimulExec.add_node('LaggedPosition', dfp.Lag(0))
SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a')
SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b')
SimulExec.add_edge('PositionUpdate', 'LaggedPosition')
SimulExec.add_edge_call_args('Execution', 'trade')
SimulExec.add_edge_call_rets('LaggedPosition', 'position')
SimulExec.start()
g = dfp.DataflowEnvironment()
g.add_node('Decision', decision)
g.add_node('SimulExec', SimulExec)
g.add_edge('Decision', 'SimulExec', 'trade', 'trade')
g.add_edge('SimulExec', 'Decision', 'position', 'position')
g.set_call_args('obs')
g.set_call_rets('position')
g.add_edge_call_args('Decision', 'observation')
g.add_edge_call_rets('SimulExec', 'position')
g.start(develop=True)
datas = [g(obs=10) for __ in range(10)]
assert datas[-1].position == 89.10000000000001
def test_use_graph_as_callable_automatic_edges_2_args(self):
g = dfp.DataflowEnvironment()
g.add_node('Decision', decision, rets='trade')
g.add_node('Execution', execution, args='trade', rets='trade')
g.add_node('PositionUpdate', add)
g.add_node('LaggedPosition', dfp.Lag(0))
g.add_edge('LaggedPosition', 'Decision', 'position')
###############################
# call add_edge with 2 arguments
###############################
g.add_edge('Decision', 'Execution')
###############################
g.add_edge('Execution', 'PositionUpdate', 'trade', 'a')
g.add_edge('LaggedPosition', 'PositionUpdate', 'b')
g.add_edge('PositionUpdate', 'LaggedPosition')
g.set_call_args('obs')
g.set_call_rets('position')
g.add_edge_call_args('Decision', 'obs', 'observation')
g.add_edge_call_rets('LaggedPosition', 'position')
g.start()
datas = []
for __ in range(10):
data = g(obs=10)
datas.append(data.position)
assert datas[-1] == 89.10000000000001
def test_terminal_node_data_no_names(self):
# declare functions
position_update = add # lambda position, trade : position + trade
pos_lag = dfp.Lag(0)
g = dfp.DataflowEnvironment()
g.args = 'observation'
g.rets = 'position'
g.add_node(decision)
g.add_node(execution, rets='trade')
g.add_node(position_update, args='position,trade')
g.add_node(pos_lag, args='position', rets='position')
g.add_edge(pos_lag, decision)
g.add_edge(decision, execution)
g.add_edge(execution, position_update)
g.add_edge(pos_lag, position_update)
g.add_edge(position_update, pos_lag)
g.add_edge_call_args(decision)
g.add_edge_call_rets(pos_lag)
g.start()
datas = []
for __ in range(10):
data = g(observation=10)
datas.append(data.position)
assert datas[-1] == 89.10000000000001
def test_terminal_node_data_with_tasks_and_automatic_link(self):
g = dfp.DataflowEnvironment()
g.args = 'observation'
g.add_task('Decision',
decision,
rets='trade',
filters=dict(args=['LaggedPosition']))
g.add_edge_call_args('Decision')
g.add_task('Execution', execution, rets='trade')
g.add_task('PositionUpdate',
add,
args='trade, position',
rets='position',
filters=dict(args=['Execution', 'LaggedPosition']))
g.add_task('LaggedPosition',
dfp.Lag(0),
args='position',
rets='position')
g.rets = 'position'
g.add_edge_call_rets('LaggedPosition')
g.start()
datas = []
for __ in range(10):
data = g(observation=10)
datas.append(data.position)
assert datas[-1] == 89.10000000000001
def test_lag():
lag = dfp.Lag(10)
assert lag(11) == 10
assert lag('a') == 11
assert lag.reset() == 'a'
assert lag('b') is None
assert lag.reset() == 'b'
def setUp(self):
self.terminal_node = DataAppender()
self.mlag = dfp.Lag(0)
def data_gen():
for i in range(10):
yield [i + 1], {}
self.data_gen = data_gen
def test_lag_state():
lag = dfp.DataflowEnvironment()
lag.args = 'lag_in'
lag.add_task('lag', dfp.Lag(None))
lag.rets = 'lag_out'
lag.add_edge_call_rets('lag')
lag.start()
assert lag(11).lag_out == None
assert lag('a').lag_out == 11
assert lag(None).lag_out == 'a'
assert lag('b').lag_out is None
assert lag(None).lag_out == 'b'
def test_graph_lqg_feedback(self):
"""
epsilonp variable is set in order to have coverage on the "self.add_edge(*edge)" part of the code.
"""
# Make SimulExec
SimulExec = dfp.DataflowEnvironment(name='SimulExec')
SimulExec.set_call_rets('position')
SimulExec.set_call_args('trade')
SimulExec.add_node('Execution', execution)
SimulExec.add_node('PositionUpdate', add)
SimulExec.add_node('LaggedPosition', dfp.Lag(0))
SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a')
SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b')
SimulExec.add_edge('PositionUpdate', 'LaggedPosition')
SimulExec.add_edge_call_args('Execution', 'trade')
SimulExec.add_edge_call_rets('LaggedPosition', 'position')
SimulExec.start()
# Make Market Plant
MarketPlant = dfp.DataflowEnvironment(name='MarketPlant')
MarketPlant.set_call_rets('position,observation, epsilonp')
MarketPlant.set_call_args('trade,observation, epsilonp')
MarketPlant.add_edge_call_args_rets('epsilonp')
MarketPlant.add_node('SimulExec', SimulExec)
MarketPlant.add_edge_call_args_rets('observation')
MarketPlant.add_edge_call_rets('SimulExec', 'position', 'position')
MarketPlant.add_edge_call_args('SimulExec', 'trade', 'trade')
MarketPlant.start()
# Make final Dataflow
def decision(epsilonp, position, observation):
return {'trade': 10, 'espilonp': epsilonp}
LQG = dfp.DataflowEnvironment(name='LQG')
LQG.add_node('Decision', decision)
LQG.add_node('MarketPlant', MarketPlant)
LQG.add_node('Data', DataGenerator())
LQG.add_edge('Decision', 'MarketPlant', 'trade', 'trade')
LQG.add_edge('MarketPlant', 'Decision')
LQG.add_edge('Data', 'MarketPlant', 'observations', 'observation')
LQG.set_call_args('epsilonp')
LQG.add_edge_call_args('MarketPlant')
LQG.set_call_rets('position, epsilonp')
LQG.add_edge_call_rets('MarketPlant')
LQG.start(develop=True, level=2)
datas = [LQG(0.1) for __ in range(10)]
assert datas[-1].position == 89.10000000000001
def setUp(self):
self.nodes = {
'Decision': decision,
'Execution': execution,
'PositionUpdate': add,
'LaggedPosition': dfp.Lag(0),
}
self.edges = [
('LaggedPosition', 'Decision', 'position'),
('Decision', 'Execution', 'trade', 'trade'),
('Execution', 'PositionUpdate', 'trade', 'a'),
('LaggedPosition', 'PositionUpdate', 'b'),
('PositionUpdate', 'LaggedPosition'),
]
def setUp(self):
self.terminal_node = DataAppender()
self.nodes = {
'Data': DataGenerator(),
'Decision': decision,
'Execution': execution,
'PositionUpdate': add,
'LaggedPosition': dfp.Lag(0),
'Terminal Node': self.terminal_node,
}
self.edges = [
('Data', 'Decision', 'observations', 'observation'),
('LaggedPosition', 'Decision', 'position'),
('Decision', 'Execution', 'trade', 'trade'),
('Execution', 'PositionUpdate', 'trade', 'a'),
('LaggedPosition', 'PositionUpdate', 'b'),
('PositionUpdate', 'LaggedPosition'),
('LaggedPosition', 'Terminal Node', 'data'),
]
def test_terminal_node_data_with_tasks_and_automatic_link_no_names(self):
# declare functions
position_update = lambda position, trade: position + trade
pos_lag = dfp.Lag(0)
g = dfp.DataflowEnvironment()
g.args = 'observation'
g.rets = 'position'
g.add_task(decision, filters=dict(args=[pos_lag]))
g.add_edge_call_args(decision)
g.add_task(execution)
g.add_task(position_update, filters=dict(args=[execution, pos_lag]))
g.add_task(pos_lag)
g.add_edge_call_rets(pos_lag)
g.start()
datas = []
for __ in range(10):
data = g(observation=10)
datas.append(data.position)
assert datas[-1] == 89.10000000000001
def add_lag(self, n, initial_state, attr_dict=None, **attr):
"""
Add a Lag node to the DataflowEnvironment
Parameters
----------
:param n : node
A node can be any hashable Python object except None.
:param initial_state:
The initial state of the Lag node
:param attr_dict: dictionary, optional (default= no attributes)
Dictionary of node attributes. Key/value pairs will
:param attr: keyword arguments, optional
Set attributes using key=value.
"""
func = dfp.Lag(initial_state)
self.add_node(n,
func=func,
attr_dict=attr_dict,
detect_func=False,
**attr)
def test_workflow_with_lag():
"""
We test a workkflow with a feedback implemented with a lag
"""
results = [0, 1000, 11001, 111012, 1111123, 11112234, 111123345, 1111234456, 11112345567, 111123456678,
1111234567789]
attr_f = {'color': 'red'}
data_in = DataGenerator()
data_out = DataAppender()
mlag = dfp.Lag(0)
dtf = dfp.DataflowEnvironment()
dtf.add_gentask('indata', data_in.gen, initial=True)
dtf.add_task('f', f2,
filters=dict(args=['indata', 'lag', ]),
**attr_f)
dtf.add_task('g', g, filters=dict(args=['f']))
dtf.add_task('lag', mlag, filters=dict(args=['g']))
dtf.add_task('terminal', data_out.func, filters=dict(args=['lag']))
dtf.start()
dtf.run()
assert data_out.out == results
def test_coroutine_from_callable_obj():
mlag = dfp.Lag(0)
f_lag = func_from_coroutine(coroutine_from_func(mlag))
assert 0 == f_lag(10)
assert 10 == f_lag(11)
def test_graph_lqg_feedback2(self):
# Make SimulExec
SimulExec = dfp.DataflowEnvironment(name='SimulExec')
SimulExec.set_call_rets('position')
SimulExec.set_call_args('trade')
SimulExec.add_node('Execution', execution)
SimulExec.add_node('PositionUpdate', add)
SimulExec.add_node('LaggedPosition', dfp.Lag(0))
SimulExec.add_edge('Execution', 'PositionUpdate', 'trade', 'a')
SimulExec.add_edge('LaggedPosition', 'PositionUpdate', 'b')
SimulExec.add_edge('PositionUpdate', 'LaggedPosition')
SimulExec.add_edge_call_args('Execution', 'trade')
SimulExec.add_edge_call_rets('LaggedPosition', 'position')
SimulExec.start(develop=False)
# Make Market Plant
MarketPlant = dfp.DataflowEnvironment(name='MarketPlant')
MarketPlant.add_node('SimulExec', SimulExec)
MarketPlant.set_call_rets('position,observation')
MarketPlant.set_call_args('trade,observation')
MarketPlant.add_edge_call_args_rets('observation')
MarketPlant.add_edge_call_rets('SimulExec', 'position', 'position')
MarketPlant.add_edge_call_args('SimulExec', 'trade', 'trade')
MarketPlant.start(develop=False)
# Make Decision Graph
DecisionGraph = dfp.DataflowEnvironment(name='DecisionGraph')
DecisionGraph.add_node('Decision', decision)
DecisionGraph.set_call_rets('trade')
DecisionGraph.set_call_args('observation,position')
DecisionGraph.add_edge_call_rets('Decision', 'trade', 'trade')
DecisionGraph.add_edge_call_args('Decision', 'position', 'position')
DecisionGraph.add_edge_call_args('Decision', 'observation',
'observation')
DecisionGraph.start(develop=False)
# Make final Dataflow
LQG = dfp.DataflowEnvironment(name='LQG')
LQG.add_node('Decision', DecisionGraph)
LQG.add_node('MarketPlant', MarketPlant)
LQG.add_node('Data', DataGenerator())
LQG.add_edge('Decision', 'MarketPlant', 'trade', 'trade')
LQG.add_edge('MarketPlant', 'Decision', 'position', 'position')
LQG.add_edge('MarketPlant', 'Decision', 'observation')
LQG.add_edge('Data', 'MarketPlant', 'observations', 'observation')
LQG.set_call_rets('position')
LQG.add_edge_call_rets('MarketPlant', 'position', 'position')
# First the graph is develop only at the first level
assert len(LQG) == 3
f = lambda: LQG.start(develop=True)
self.assertRaises(dfpe.StalledDataflowCallError, f)
# We develop the second level
LQG.start(develop=True, level=2)
assert len(LQG) == 5
datas = [LQG() for __ in range(10)]
assert datas[-1].position == 89.10000000000001
def test_use_graph_as_callable_test_args(self):
g = dfp.DataflowEnvironment()
g.add_node('Decision', decision, args='test_obs,test_pos')
g.add_node('LaggedPosition', dfp.Lag(0))
f = lambda: g.add_edge('LaggedPosition', 'Decision', 'position')
self.assertRaises(dfpe.WrongEdgeArgsError, f)
