def setUp(self):
custom_module = '{}/custom_port_nodes.py'.format(
os.path.dirname(os.path.realpath(__file__)))
points_task_spec = {
TaskSpecSchema.task_id: 'points_task',
TaskSpecSchema.node_type: 'PointNode',
TaskSpecSchema.filepath: custom_module,
TaskSpecSchema.conf: {
'npts': 1000
},
TaskSpecSchema.inputs: {}
}
self.points_task = Task(points_task_spec)
distance_task_spec = {
TaskSpecSchema.task_id: 'distance_by_cudf',
TaskSpecSchema.node_type: 'DistanceNode',
TaskSpecSchema.filepath: custom_module,
TaskSpecSchema.conf: {},
TaskSpecSchema.inputs: {
'points_df_in': 'points_task.points_df_out'
}
}
self.distance_task = Task(distance_task_spec)
def test_drop(self):
'''Test node columns drop'''
node_obj = {
"id": "abc",
"type": "IndicatorNode",
"conf": self.conf,
"inputs": {}
}
task = Task(node_obj)
inN = IndicatorNode(task)
o = inN.process({"stock_in": self._cudf_data})['stock_out']
msg = "bad error: df len %d is not right" % (len(o))
self.assertTrue(len(o) == 162, msg)
newConf = copy.deepcopy(self.conf)
newConf['remove_na'] = False
node_obj = {
"id": "abc",
"type": "IndicatorNode",
"conf": newConf,
"inputs": {}
}
task = Task(node_obj)
inN = IndicatorNode(task)
o = inN.process({"stock_in": self._cudf_data})['stock_out']
msg = "bad error: df len %d is not right" % (len(o))
self.assertTrue(len(o) == 200, msg)
def test_return(self):
'''Test return feature node'''
conf = {}
node_obj = {
"id": "abc",
"type": "ReturnFeatureNode",
"conf": conf,
"inputs": {}
}
task = Task(node_obj)
inN = ReturnFeatureNode(task)
o = inN.process({'stock_in': self._cudf_data})['stock_out']
err, index_err = error_function_index(o['returns'], self.gt)
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
msg = "bad error %f\n" % (index_err, )
self.assertTrue(np.isclose(index_err, 0, atol=1e-6), msg)
def test_indicator(self):
'''Test indicator node'''
conf = {
"indicators": [{
"function": "port_chaikin_oscillator",
"columns": ["high", "low", "close", "volume"],
"args": [10, 20]
}, {
"function": "port_bollinger_bands",
"columns": ["close"],
"args": [10],
"outputs": ["b1", "b2"]
}],
"remove_na":
True
}
node_obj = {
"id": "abc",
"type": "IndicatorNode",
"conf": conf,
"inputs": {}
}
task = Task(node_obj)
inN = IndicatorNode(task)
o = inN.process({'stock_in': self._cudf_data})['stock_out']
err, index_err = error_function_index(o['CH_OS_10_20'], self.gt1)
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
msg = "bad error %f\n" % (index_err, )
self.assertTrue(np.isclose(index_err, 0, atol=1e-6), msg)
err, index_err = error_function_index(o['BO_BA_b1_10'], self.gt2)
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
msg = "bad error %f\n" % (index_err, )
self.assertTrue(np.isclose(index_err, 0, atol=1e-6), msg)
err, index_err = error_function_index(o['BO_BA_b2_10'], self.gt3)
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
msg = "bad error %f\n" % (index_err, )
self.assertTrue(np.isclose(index_err, 0, atol=1e-6), msg)
def test_asset_indicator(self):
'''Test asset indicator node'''
conf = {}
node_obj = {
"id": "abc",
"type": "AssetIndicatorNode",
"conf": conf,
"inputs": {}
}
task = Task(node_obj)
inN = AssetIndicatorNode(task)
gt = self._cudf_data.to_pandas()['indicator']
o = inN.process(
{'stock_in': self._cudf_data.drop('indicator',
axis=1)})['stock_out']
err, index_err = error_function_index(o['indicator'], gt)
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
msg = "bad error %f\n" % (index_err, )
self.assertTrue(np.isclose(index_err, 0, atol=1e-6), msg)
def test_colums(self):
'''Test node columns requirments'''
node_obj = {
"id": "abc",
"type": "IndicatorNode",
"conf": self.conf,
"inputs": {}
}
task = Task(node_obj)
inN = get_node_tgraphmixin_instance(IndicatorNode, task)
# inN = IndicatorNode(task)
out_cols = inN.meta_setup().outports
col = "indicator"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in inN.meta_setup().inports['stock_in'], msg)
col = "high"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in inN.meta_setup().inports['stock_in'], msg)
col = "low"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in inN.meta_setup().inports['stock_in'], msg)
col = "close"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in inN.meta_setup().inports['stock_in'], msg)
col = "volume"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in inN.meta_setup().inports['stock_in'], msg)
col = "CH_OS_10_20"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in out_cols['stock_out'], msg)
col = "BO_BA_b1_10"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in out_cols['stock_out'], msg)
col = "BO_BA_b2_10"
msg = "bad error: %s is missing" % (col)
self.assertTrue(col in out_cols['stock_out'], msg)
def add_nodes():
"""
It will load all the nodes for the UI client so user can add new node
to the graph. The nodes are from two sources: default greenflow nodes and
customized node modules.
The output is a dictionary whose keys are module names and values are a
list of the nodes inside that module.
Arguments
-------
Returns
-------
dict
dictionary of all the nodes that can be added in the client
"""
loaded_node_classes = []
all_modules = get_greenflow_config_modules()
# print('Greenflow config modules: {}\n'.format(all_modules))
all_nodes = {}
# not implemented yet for greenflow
for item in inspect.getmembers(plugin_nodes):
if inspect.ismodule(item[1]):
# print('Greenflow builtin plugin: {}'.format(item))
labmod_pkg = 'greenflow.{}'.format(item[0])
all_nodes[labmod_pkg] = []
for node in inspect.getmembers(item[1]):
nodecls = node[1]
if not inspect.isclass(nodecls):
continue
if not issubclass(nodecls, Node):
continue
if nodecls in loaded_node_classes:
continue
task = {
'id': 'node_' + str(uuid.uuid4()),
'type': node[0],
'conf': {},
'inputs': []
}
task_inst = Task(task)
node_inst = get_node_tgraphmixin_instance(nodecls, task_inst)
nodeObj = init_get_labnode_obj(node_inst, False)
all_nodes[labmod_pkg].append(nodeObj)
loaded_node_classes.append(nodecls)
for module in all_modules:
module_file_or_path = Path(all_modules[module])
loaded = load_modules(all_modules[module], module)
mod = loaded.mod
modulename = module
# all_nodes[modulename] = []
for node in inspect.getmembers(mod):
nodecls = node[1]
if not inspect.isclass(nodecls):
continue
if nodecls == Node:
continue
if not issubclass(nodecls, Node):
continue
if nodecls in loaded_node_classes:
continue
task = {
'id': 'node_' + str(uuid.uuid4()),
'type': node[0],
'conf': {},
'inputs': [],
'module': module
}
task_inst = Task(task)
node_inst = get_node_tgraphmixin_instance(nodecls, task_inst)
nodeObj = init_get_labnode_obj(node_inst, False)
if module_file_or_path.is_dir():
# submod = nodecls.__module__.split('.')[1:]
# flatten out the namespace hierarchy
submod = nodecls.__module__.split('.')[1:2]
modulename_ = '.'.join([modulename, '.'.join(submod)]) \
if submod else modulename
all_nodes.setdefault(modulename_, []).append(nodeObj)
else:
all_nodes.setdefault(modulename, []).append(nodeObj)
loaded_node_classes.append(nodecls)
for module in DYNAMIC_MODULES.keys():
modulename = module
all_nodes[modulename] = []
for class_name in DYNAMIC_MODULES[module].keys():
nodecls = DYNAMIC_MODULES[module][class_name]
if not issubclass(nodecls, Node):
continue
if nodecls in loaded_node_classes:
continue
task = {
'id': 'node_' + str(uuid.uuid4()),
'type': nodecls.__name__,
'conf': {},
'inputs': [],
'module': module
}
task_inst = Task(task)
node_inst = get_node_tgraphmixin_instance(nodecls, task_inst)
nodeObj = init_get_labnode_obj(node_inst, False)
all_nodes.setdefault(modulename, []).append(nodeObj)
loaded_node_classes.append(nodecls)
# load all the plugins from entry points
for entry_point in importlib_metadata.entry_points().get(
'greenflow.plugin', ()):
mod = entry_point.load()
modulename = entry_point.name
for node in inspect.getmembers(mod):
nodecls = node[1]
if not inspect.isclass(nodecls):
continue
if nodecls == Node:
continue
if not issubclass(nodecls, Node):
continue
if nodecls in loaded_node_classes:
continue
task = {
'id': 'node_' + str(uuid.uuid4()),
'type': node[0],
'conf': {},
'inputs': [],
'module': modulename
}
task_inst = Task(task)
node_inst = get_node_tgraphmixin_instance(nodecls, task_inst)
nodeObj = init_get_labnode_obj(node_inst, False)
all_nodes.setdefault(modulename, []).append(nodeObj)
loaded_node_classes.append(nodecls)
return all_nodes
class TestNodeAPI(unittest.TestCase):
def setUp(self):
custom_module = '{}/custom_port_nodes.py'.format(
os.path.dirname(os.path.realpath(__file__)))
points_task_spec = {
TaskSpecSchema.task_id: 'points_task',
TaskSpecSchema.node_type: 'PointNode',
TaskSpecSchema.filepath: custom_module,
TaskSpecSchema.conf: {
'npts': 1000
},
TaskSpecSchema.inputs: {}
}
self.points_task = Task(points_task_spec)
distance_task_spec = {
TaskSpecSchema.task_id: 'distance_by_cudf',
TaskSpecSchema.node_type: 'DistanceNode',
TaskSpecSchema.filepath: custom_module,
TaskSpecSchema.conf: {},
TaskSpecSchema.inputs: {
'points_df_in': 'points_task.points_df_out'
}
}
self.distance_task = Task(distance_task_spec)
def tearDown(self):
pass
@ordered
def test_node_instantiation(self):
'''Test node instantiation.
1. Test that you cannot instantiate an abstract base class without
first implementing the methods requiring override.
2. Check for the base and base mixin classes in a Node class
implementation.
'''
points_task = self.points_task
# assert cannot instantiate Node without overriding meta_setup
# and process
with self.assertRaises(TypeError) as cm:
_ = Node(points_task)
err_msg = '{}'.format(cm.exception)
self.assertEqual(
err_msg,
"Can't instantiate abstract class Node with abstract methods "
"meta_setup, process")
points_node = points_task.get_node_obj()
self.assertIsInstance(points_node, _Node)
self.assertIsInstance(points_node, Node)
self.assertIsInstance(points_node, _PortsMixin)
self.assertNotIsInstance(points_node, NodeTaskGraphMixin)
points_node = points_task.get_node_obj(tgraph_mixin=True)
self.assertIsInstance(points_node, NodeTaskGraphMixin)
@ordered
def test_node_ports(self):
'''Test the ports related APIs such as existence of ports, input ports,
and output ports.
'''
distance_node = self.distance_task.get_node_obj()
iports = distance_node._get_input_ports()
oports = distance_node._get_output_ports()
self.assertEqual(iports, ['points_df_in'])
self.assertEqual(oports, ['distance_df', 'distance_abs_df'])
def test_signal(self):
'''Test signal computation'''
newConf = copy.deepcopy(self.conf)
newConf['remove_na'] = False
node_obj = {
"id": "abc",
"type": "IndicatorNode",
"conf": newConf,
"inputs": {}
}
task = Task(node_obj)
inN = IndicatorNode(task)
o = inN.process({'stock_in': self._cudf_data})['stock_out']
# check chaikin oscillator computation
r_cudf = gi.chaikin_oscillator(self._cudf_data[:self.half]['high'],
self._cudf_data[:self.half]['low'],
self._cudf_data[:self.half]['close'],
self._cudf_data[:self.half]['volume'],
10, 20)
computed = o[:self.half]['CH_OS_10_20'].to_array('pandas')
ref = r_cudf.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
r_cudf = gi.chaikin_oscillator(self._cudf_data[self.half:]['high'],
self._cudf_data[self.half:]['low'],
self._cudf_data[self.half:]['close'],
self._cudf_data[self.half:]['volume'],
10, 20)
computed = o[self.half:]['CH_OS_10_20'].to_array('pandas')
ref = r_cudf.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
# check bollinger bands computation
r_cudf = gi.bollinger_bands(self._cudf_data[:self.half]['close'], 10)
computed = o[:self.half]["BO_BA_b1_10"].to_array('pandas')
ref = r_cudf.b1.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
computed = o[:self.half]["BO_BA_b2_10"].to_array('pandas')
ref = r_cudf.b2.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
r_cudf = gi.bollinger_bands(self._cudf_data[self.half:]['close'], 10)
computed = o[self.half:]["BO_BA_b1_10"].to_array('pandas')
ref = r_cudf.b1.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)
computed = o[self.half:]["BO_BA_b2_10"].to_array('pandas')
ref = r_cudf.b2.to_array('pandas')
err = np.abs(computed[~np.isnan(computed)] - ref[~np.isnan(ref)]).max()
msg = "bad error %f\n" % (err, )
self.assertTrue(np.isclose(err, 0, atol=1e-6), msg)