def test_dynamic_varnode_pickle(self):
# shift B by a temporary varnode with a value set in the root context
temp_varnode = varnode(default="temp_varnode_default")
shifted_ctx = self.ctx.shift({B: temp_varnode})
self.ctx[temp_varnode] = "temp_varnode"
self.assertEquals(shifted_ctx[B], "temp_varnode")
# pickle the context, unregister the node and recreate the context
x = pickle.dumps(self.ctx)
MDFContext.unregister_node(temp_varnode)
new_ctx = pickle.loads(x)
# find the shifted context (can't reshift as temp_varnode was unregistered)
self.assertEquals(len(new_ctx.get_shifted_contexts()), 1)
shifted_ctx = new_ctx.get_shifted_contexts()[0]
# check that B in the shifted context is still set
self.assertEquals(shifted_ctx[B], "temp_varnode")
# check that temp_varnode refers to the original node, not the new one
# that had to be recreated when un-pickled.
self.assertEquals(shifted_ctx[temp_varnode], "temp_varnode_default")
class ContextTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), periods=3, freq=datetools.yearEnd)
self.ctx = MDFContext()
def test_shift(self):
# C yields a value based on shifted contexts
res = []
for t in self.daterange:
self.ctx.set_date(t)
res.append(self.ctx[C])
assert_array_almost_equal(res, [(1,2,3), (3,5,7), (6,9,12)])
def test_shift2(self):
# C yields a value based on shifted contexts
res = []
for t in self.daterange:
self.ctx.set_date(t)
res.append(self.ctx[D])
assert_array_almost_equal(res, [(1,2,3), (3,5,7), (6,9,12)])
def test_dynamic_varnode_pickle(self):
# shift B by a temporary varnode with a value set in the root context
temp_varnode = varnode(default="temp_varnode_default")
shifted_ctx = self.ctx.shift({B : temp_varnode})
self.ctx[temp_varnode] = "temp_varnode"
self.assertEquals(shifted_ctx[B], "temp_varnode")
# pickle the context, unregister the node and recreate the context
x = pickle.dumps(self.ctx)
MDFContext.unregister_node(temp_varnode)
new_ctx = pickle.loads(x)
# find the shifted context (can't reshift as temp_varnode was unregistered)
self.assertEquals(len(new_ctx.get_shifted_contexts()), 1)
shifted_ctx = new_ctx.get_shifted_contexts()[0]
# check that B in the shifted context is still set
self.assertEquals(shifted_ctx[B], "temp_varnode")
# check that temp_varnode refers to the original node, not the new one
# that had to be recreated when un-pickled.
self.assertEquals(shifted_ctx[temp_varnode], "temp_varnode_default")
class ContextTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1),
periods=3,
freq=datetools.yearEnd)
self.ctx = MDFContext()
def test_shift(self):
# C yields a value based on shifted contexts
res = []
for t in self.daterange:
self.ctx.set_date(t)
res.append(self.ctx[C])
assert_array_almost_equal(res, [(1, 2, 3), (3, 5, 7), (6, 9, 12)])
def test_shift2(self):
# C yields a value based on shifted contexts
res = []
for t in self.daterange:
self.ctx.set_date(t)
res.append(self.ctx[D])
assert_array_almost_equal(res, [(1, 2, 3), (3, 5, 7), (6, 9, 12)])
class ExceptionTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
def test_exception_1(self):
# A should raise an exception if A is not True
self.ctx[A] = False
self.assertRaises(TestException, self.ctx.get_value, B)
# when called again the exception should be re-raised
self.assertRaises(TestException, self.ctx.get_value, B)
# setting A should cause B to return True
self.ctx[A] = True
self.assertTrue(self.ctx[B])
def test_exception_2(self):
# C should raise an exception while 'now' is before 2001
self.ctx.set_date(datetime(2000, 12, 31))
self.assertRaises(TestException, self.ctx.get_value, C)
self.ctx.set_date(datetime(2001, 1, 1))
self.assertTrue(self.ctx[C])
class ExceptionTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
def test_exception_1(self):
# A should raise an exception if A is not True
self.ctx[A] = False
self.assertRaises(TestException, self.ctx.get_value, B)
# when called again the exception should be re-raised
self.assertRaises(TestException, self.ctx.get_value, B)
# setting A should cause B to return True
self.ctx[A] = True
self.assertTrue(self.ctx[B])
def test_exception_2(self):
# C should raise an exception while 'now' is before 2001
self.ctx.set_date(datetime(2000, 12, 31))
self.assertRaises(TestException, self.ctx.get_value, C)
self.ctx.set_date(datetime(2001, 1, 1))
self.assertTrue(self.ctx[C])
def test_save(self):
tmpdir = tempfile.mkdtemp()
try:
for ext in (".dag", ".zip", ".bz2", ".gz"):
filename = os.path.join(tmpdir, "ctx" + ext)
a = self.ctx[A] = random.randint(0, 100)
self.assertEquals(self.ctx[A], a)
self.ctx.save(filename)
_logger.info("context file '%s' is %db" % (os.path.basename(filename),
os.stat(filename).st_size))
new_ctx = MDFContext.load(filename)
self.assertEquals(new_ctx[A], a)
os.unlink(filename)
finally:
shutil.rmtree(tmpdir, True)
def test_save(self):
tmpdir = tempfile.mkdtemp()
try:
for ext in (".dag", ".zip", ".bz2", ".gz"):
filename = os.path.join(tmpdir, "ctx" + ext)
a = self.ctx[A] = random.randint(0, 100)
self.assertEquals(self.ctx[A], a)
self.ctx.save(filename)
_logger.info(
"context file '%s' is %db" %
(os.path.basename(filename), os.stat(filename).st_size))
new_ctx = MDFContext.load(filename)
self.assertEquals(new_ctx[A], a)
os.unlink(filename)
finally:
shutil.rmtree(tmpdir, True)
def setUp(self):
self.ctx = MDFContext()
class ClsNodeTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1),
datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[_TestNodes.C] = 10
self.ctx[_TestNodes.D] = 20
def test_queuenode(self):
self._run(_TestNodes.queue_output)
queue = self.ctx[_TestNodes.queue_output]
self.assertEqual(len(queue), len(self.daterange))
def test_nansumnode(self):
self._run(_TestNodes.nansum_output)
nansum = self.ctx[_TestNodes.nansum_output]
self.assertEqual(nansum, 812)
def test_cumprodnode(self):
self._run(_TestNodes.cumprod_output)
cumprod = self.ctx[_TestNodes.cumprod_output]
self.assertEqual(cumprod, 14201189062704000)
def test_subclass(self):
base_a = self.ctx[_TestNodes.A]
sub_a = self.ctx[SubClass.A]
self.assertEqual(base_a, 200)
self.assertEqual(sub_a, 30)
self._run(_TestNodes.cumprod_output, SubClass.cumprod_output)
base_cumprod = self.ctx[_TestNodes.cumprod_output]
sub_cumprod = self.ctx[SubClass.cumprod_output]
self.assertEqual(base_cumprod, 14201189062704000)
self.assertEqual(sub_cumprod, 42072307200)
def test_nodenames(self):
self.assertNotEqual(SubClass.A.name, _TestNodes.A.name)
def test_queue_size(self):
self._run(
_TestNodes.queue_size_test, # check basic queue size
_TestNodes.queue_size_test2, # check evalnode queue size
SubClass.queue_size_test2) # check overriding of queue size
self.assertEqual(len(self.ctx[_TestNodes.queue_size_test]), 2)
self.assertEqual(len(self.ctx[_TestNodes.queue_size_test2]), 3)
self.assertEqual(len(self.ctx[SubClass.queue_size_test2]), 4)
def test_datanode(self):
self._run(_TestNodes.df_data, _TestNodes.df_filter)
self.assertEqual(self.ctx[_TestNodes.df_data]["A"],
len(self.daterange) - 1)
self.assertTrue(self.ctx[_TestNodes.df_filter])
self.ctx.set_date(datetime(1990, 1, 1))
self.assertFalse(self.ctx[_TestNodes.df_filter])
def _run(self, *nodes):
for t in self.daterange:
self.ctx.set_date(t)
for node in nodes:
self.ctx[node]
class OverrideTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
global num_calls_A, num_calls_A_override
num_calls_A = 0
num_calls_A_override = 0
def test_override(self):
b = self.ctx[B] = 1
c = self.ctx[C] = 2
self.assertEqual(self.ctx[A], b)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 0)
# override A
self.ctx[A] = A_override
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 1)
# check the dependencies are working
c = self.ctx[C] = 100
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 2)
# check A doesn't still depend on B
b = self.ctx[B] = 500
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 2)
def test_shift_override(self):
b = self.ctx[B] = 1
c = self.ctx[C] = 2
# D = A * 2
# A = B
self.assertEqual(self.ctx[D], b * 2)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 0)
# shift_test = D [ A := A_override ]
# A_override = C
self.assertEqual(self.ctx[shift_test], c * 2)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 1)
def test_shift_override2(self):
c = self.ctx[C] = 10
# shift_test = D [ B := B_override ]
# D = A * 2
# A = B
# B_override = C * 3
self.assertEqual(self.ctx[shift_test2], c * 6)
def test_shift_override3(self):
# overriding a node in the root context and then
# evaluating it in a shifted context should work
self.ctx[D] = D_override # = A + C
self.ctx[A] = 10
self.ctx[C] = 20
shifted_ctx = self.ctx.shift({A: 100})
# - if D is used instead of D_override A will equal 2 * 100 = 200
# - if D_override is used but in the root context the
# result will be A + C = 30
# - if D_override is used and the correct context is used
# the results will be A + C = 100 + 20 = 120
self.assertEqual(shifted_ctx[D], 120)
class NodeTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[params.C] = 10
self.ctx[params.D] = 20
def test_queuenode(self):
self._run(queue_output)
queue = self.ctx[queue_output]
self.assertEqual(len(queue), len(self.daterange))
def test_queueyield(self):
self._run(queue_yield)
queue = self.ctx[queue_yield]
self.assertEqual(len(queue), len(self.daterange))
def test_queue_filter(self):
self._run(queue_filter_test)
queue = self.ctx[queue_filter_test]
self.assertEqual(list(queue), [0] * (len(self.daterange) - 1))
@staticmethod
def diff_dfs(lhs_df, rhs_df, tolerance):
diffs = np.abs(lhs_df - rhs_df)
mask = (diffs > tolerance).values
mask &= ~(np.isnan(lhs_df) and np.isnan(rhs_df)).values
mask |= np.isnan(lhs_df).values & ~np.isnan(rhs_df).values
mask |= np.isnan(rhs_df).values & ~np.isnan(lhs_df).values
return mask.any()
def test_nansumnode(self):
self._run(nansum_output)
nansum = self.ctx[nansum_output]
self.assertEqual(nansum, 812)
def test_cumprodnode(self):
self._run(cumprod_output)
cumprod = self.ctx[cumprod_output]
self.assertEqual(cumprod, 14201189062704000)
def test_delaynode(self):
self._run(DelayNodeTest.delay_test, DelayNodeTest.delay_test_lazy)
value = self.ctx[DelayNodeTest.delay_test]
value_lazy = self.ctx[DelayNodeTest.delay_test_lazy]
self.assertEqual(list(value), range(1, len(self.daterange) + 1))
self.assertEqual(list(value_lazy), range(1, len(self.daterange) + 1))
def test_ffillnode(self):
self._run(ffill_queue)
value = self.ctx[ffill_queue]
self.assertEqual(tuple(value), (0.0, 0.0, 2.0, 2.0, 4.0, 4.0, 6.0))
def test_ffill_array(self):
self._run(ffill_array_test)
value = self.ctx[ffill_array_test]
unfilled_value = self.ctx[ffill_array_test_not_filled]
self.assertTrue(np.isnan(unfilled_value).all())
self.assertEquals(value.tolist(), [10.0, 10.0, 10.0, 10.0, 10.0])
def test_datanode_ffill(self):
data = pd.Series(range(len(self.daterange)), self.daterange, dtype=float)
data = data[[bool(i % 2) for i in xrange(len(data.index))]]
expected = data.reindex(self.daterange, method="ffill")
expected[np.isnan(expected)] = np.inf
node = datanode("test_datanode_ffill", data, ffill=True, missing_value=np.inf)
qnode = node.queuenode()
self._run(qnode)
value = self.ctx[qnode]
self.assertEquals(list(value), expected.values.tolist())
def test_lookahead_node(self):
B_queue = B.queuenode()
B_lookahead = B.lookaheadnode(periods=len(self.daterange))
self.ctx.set_date(self.daterange[0])
actual = self.ctx[B_lookahead]
self._run(B_queue)
expected = self.ctx[B_queue]
self.assertEquals(actual.values.tolist(), list(expected))
self.assertEquals(actual.index.tolist(), list(self.daterange))
def test_apply_node(self):
actual_node = A.applynode(func=operator.add, args=(B,)).queuenode()
expected_node = A_plus_B.queuenode()
self._run(actual_node, expected_node)
actual = self.ctx[actual_node]
expected = self.ctx[expected_node]
self.assertEquals(actual, expected)
def test_binary_operators_with_constant(self):
self._test(Counter, [-2.0, -1.5, -1.0, -0.5, 0.5, 1.0, 1.5])
self._test(Counter + 0.2, [-1.8, -1.3, -0.8, -0.3, 0.7, 1.2, 1.7])
self._test(Counter - 0.2, [-2.2, -1.7, -1.2, -0.7, 0.3, 0.8, 1.3])
self._test(Counter * 2.0, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(Counter / 0.5, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(0.2 + Counter, [-1.8, -1.3, -0.8, -0.3, 0.7, 1.2, 1.7])
self._test(1.0 - Counter, [3.0, 2.5, 2.0, 1.5, 0.5, 0.0, -0.5])
self._test(2.0 * Counter, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(
12 / (Counter + 0.25),
[-6.8571428571428568, -9.6000000000000014, -16.0, -48.0, 16.0, 9.6000000000000014, 6.8571428571428568],
)
def test_binary_operators_with_node(self):
self._test(Counter + Counter, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(Counter - Counter, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
self._test(Counter * Counter, [4.0, 2.25, 1.0, 0.25, 0.25, 1.0, 2.25])
self._test(Counter / Counter, [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
def _test(self, node, expected_values):
values = node.queuenode()
self._run(values)
actual = self.ctx[values]
self.assertEquals(list(actual), expected_values)
def _run_for_daterange(self, date_range, *nodes):
for t in date_range:
self.ctx.set_date(t)
for node in nodes:
self.ctx[node]
def _run(self, *nodes):
self._run_for_daterange(self.daterange, *nodes)
class ClsNodeTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[_TestNodes.C] = 10
self.ctx[_TestNodes.D] = 20
def test_queuenode(self):
self._run(_TestNodes.queue_output)
queue = self.ctx[_TestNodes.queue_output]
self.assertEqual(len(queue), len(self.daterange))
def test_nansumnode(self):
self._run(_TestNodes.nansum_output)
nansum = self.ctx[_TestNodes.nansum_output]
self.assertEqual(nansum, 812)
def test_cumprodnode(self):
self._run(_TestNodes.cumprod_output)
cumprod = self.ctx[_TestNodes.cumprod_output]
self.assertEqual(cumprod, 14201189062704000)
def test_subclass(self):
base_a = self.ctx[_TestNodes.A]
sub_a = self.ctx[SubClass.A]
self.assertEqual(base_a, 200)
self.assertEqual(sub_a, 30)
self._run(_TestNodes.cumprod_output, SubClass.cumprod_output)
base_cumprod = self.ctx[_TestNodes.cumprod_output]
sub_cumprod = self.ctx[SubClass.cumprod_output]
self.assertEqual(base_cumprod, 14201189062704000)
self.assertEqual(sub_cumprod, 42072307200)
def test_nodenames(self):
self.assertNotEqual(SubClass.A.name, _TestNodes.A.name)
def test_queue_size(self):
self._run(_TestNodes.queue_size_test, # check basic queue size
_TestNodes.queue_size_test2, # check evalnode queue size
SubClass.queue_size_test2) # check overriding of queue size
self.assertEqual(len(self.ctx[_TestNodes.queue_size_test]), 2)
self.assertEqual(len(self.ctx[_TestNodes.queue_size_test2]), 3)
self.assertEqual(len(self.ctx[SubClass.queue_size_test2]), 4)
def test_datanode(self):
self._run(_TestNodes.df_data, _TestNodes.df_filter)
self.assertEqual(self.ctx[_TestNodes.df_data]["A"], len(self.daterange) - 1)
self.assertTrue(self.ctx[_TestNodes.df_filter])
self.ctx.set_date(datetime(1990, 1, 1))
self.assertFalse(self.ctx[_TestNodes.df_filter])
def _run(self, *nodes):
for t in self.daterange:
self.ctx.set_date(t)
for node in nodes:
self.ctx[node]
class PickleTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
def test_pickle(self):
a = self.ctx[A] = 10
b = self.ctx[B]
c = self.ctx[C]
num_calls = _b_num_calls
x = pickle.dumps(self.ctx)
new_ctx = pickle.loads(x)
self.assertEquals(new_ctx[A], a)
self.assertEquals(new_ctx[B], b)
self.assertEquals(new_ctx[C], c)
# unpickling the context shouldn't re-evaluate anything
self.assertEquals(num_calls, _b_num_calls)
def test_node_method_pickle(self):
# get another node via a nodetype method
# use nan in the args as there were problems pickling/unpicking nan
node = A.samplenode(initial_value=np.nan,
offset=pa.datetools.BMonthEnd())
# test text and binary pickle formats
for protocol in (0, pickle.HIGHEST_PROTOCOL):
x = pickle.dumps(node, protocol)
new_node = pickle.loads(x)
# unpickling a node constructed via a notetype method call should retrieve
# the same node instance
self.assertTrue(node is new_node)
def test_shifted_pickle(self):
shifted_ctx = self.ctx.shift({A: 100})
a = shifted_ctx[A]
b = shifted_ctx[B]
c = shifted_ctx[C]
num_calls = _b_num_calls
x = pickle.dumps(self.ctx)
new_ctx = pickle.loads(x)
# pickling preserves shifted contexts so this should return
# an existing shifted context
new_shifted_ctx = new_ctx.shift({A: a})
self.assertNotEquals(new_shifted_ctx, shifted_ctx)
self.assertEquals(new_shifted_ctx[A], a)
self.assertEquals(new_shifted_ctx[B], b)
self.assertEquals(new_shifted_ctx[C], c)
# unpickling the context shouldn't re-evaluate anything
self.assertEquals(num_calls, _b_num_calls)
def test_dynamic_varnode_pickle(self):
# shift B by a temporary varnode with a value set in the root context
temp_varnode = varnode(default="temp_varnode_default")
shifted_ctx = self.ctx.shift({B: temp_varnode})
self.ctx[temp_varnode] = "temp_varnode"
self.assertEquals(shifted_ctx[B], "temp_varnode")
# pickle the context, unregister the node and recreate the context
x = pickle.dumps(self.ctx)
MDFContext.unregister_node(temp_varnode)
new_ctx = pickle.loads(x)
# find the shifted context (can't reshift as temp_varnode was unregistered)
self.assertEquals(len(new_ctx.get_shifted_contexts()), 1)
shifted_ctx = new_ctx.get_shifted_contexts()[0]
# check that B in the shifted context is still set
self.assertEquals(shifted_ctx[B], "temp_varnode")
# check that temp_varnode refers to the original node, not the new one
# that had to be recreated when un-pickled.
self.assertEquals(shifted_ctx[temp_varnode], "temp_varnode_default")
def test_save(self):
tmpdir = tempfile.mkdtemp()
try:
for ext in (".dag", ".zip", ".bz2", ".gz"):
filename = os.path.join(tmpdir, "ctx" + ext)
a = self.ctx[A] = random.randint(0, 100)
self.assertEquals(self.ctx[A], a)
self.ctx.save(filename)
_logger.info(
"context file '%s' is %db" %
(os.path.basename(filename), os.stat(filename).st_size))
new_ctx = MDFContext.load(filename)
self.assertEquals(new_ctx[A], a)
os.unlink(filename)
finally:
shutil.rmtree(tmpdir, True)
class OverrideTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
global num_calls_A, num_calls_A_override
num_calls_A = 0
num_calls_A_override = 0
def test_override(self):
b = self.ctx[B] = 1
c = self.ctx[C] = 2
self.assertEqual(self.ctx[A], b)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 0)
# override A
self.ctx[A] = A_override
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 1)
# check the dependencies are working
c = self.ctx[C] = 100
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 2)
# check A doesn't still depend on B
b = self.ctx[B] = 500
self.assertEqual(self.ctx[A], c)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 2)
def test_shift_override(self):
b = self.ctx[B] = 1
c = self.ctx[C] = 2
# D = A * 2
# A = B
self.assertEqual(self.ctx[D], b * 2)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 0)
# shift_test = D [ A := A_override ]
# A_override = C
self.assertEqual(self.ctx[shift_test], c * 2)
self.assertEqual(num_calls_A, 1)
self.assertEqual(num_calls_A_override, 1)
def test_shift_override2(self):
c = self.ctx[C] = 10
# shift_test = D [ B := B_override ]
# D = A * 2
# A = B
# B_override = C * 3
self.assertEqual(self.ctx[shift_test2], c * 6)
def test_shift_override3(self):
# overriding a node in the root context and then
# evaluating it in a shifted context should work
self.ctx[D] = D_override # = A + C
self.ctx[A] = 10
self.ctx[C] = 20
shifted_ctx = self.ctx.shift({A : 100})
# - if D is used instead of D_override A will equal 2 * 100 = 200
# - if D_override is used but in the root context the
# result will be A + C = 30
# - if D_override is used and the correct context is used
# the results will be A + C = 100 + 20 = 120
self.assertEqual(shifted_ctx[D], 120)
class PickleTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext()
def test_pickle(self):
a = self.ctx[A] = 10
b = self.ctx[B]
c = self.ctx[C]
num_calls = _b_num_calls
x = pickle.dumps(self.ctx)
new_ctx = pickle.loads(x)
self.assertEquals(new_ctx[A], a)
self.assertEquals(new_ctx[B], b)
self.assertEquals(new_ctx[C], c)
# unpickling the context shouldn't re-evaluate anything
self.assertEquals(num_calls, _b_num_calls)
def test_node_method_pickle(self):
# get another node via a nodetype method
# use nan in the args as there were problems pickling/unpicking nan
node = A.samplenode(initial_value=np.nan, offset=pa.datetools.BMonthEnd())
# test text and binary pickle formats
for protocol in (0, pickle.HIGHEST_PROTOCOL):
x = pickle.dumps(node, protocol)
new_node = pickle.loads(x)
# unpickling a node constructed via a notetype method call should retrieve
# the same node instance
self.assertTrue(node is new_node)
def test_shifted_pickle(self):
shifted_ctx = self.ctx.shift({A : 100})
a = shifted_ctx[A]
b = shifted_ctx[B]
c = shifted_ctx[C]
num_calls = _b_num_calls
x = pickle.dumps(self.ctx)
new_ctx = pickle.loads(x)
# pickling preserves shifted contexts so this should return
# an existing shifted context
new_shifted_ctx = new_ctx.shift({A : a})
self.assertNotEquals(new_shifted_ctx, shifted_ctx)
self.assertEquals(new_shifted_ctx[A], a)
self.assertEquals(new_shifted_ctx[B], b)
self.assertEquals(new_shifted_ctx[C], c)
# unpickling the context shouldn't re-evaluate anything
self.assertEquals(num_calls, _b_num_calls)
def test_dynamic_varnode_pickle(self):
# shift B by a temporary varnode with a value set in the root context
temp_varnode = varnode(default="temp_varnode_default")
shifted_ctx = self.ctx.shift({B : temp_varnode})
self.ctx[temp_varnode] = "temp_varnode"
self.assertEquals(shifted_ctx[B], "temp_varnode")
# pickle the context, unregister the node and recreate the context
x = pickle.dumps(self.ctx)
MDFContext.unregister_node(temp_varnode)
new_ctx = pickle.loads(x)
# find the shifted context (can't reshift as temp_varnode was unregistered)
self.assertEquals(len(new_ctx.get_shifted_contexts()), 1)
shifted_ctx = new_ctx.get_shifted_contexts()[0]
# check that B in the shifted context is still set
self.assertEquals(shifted_ctx[B], "temp_varnode")
# check that temp_varnode refers to the original node, not the new one
# that had to be recreated when un-pickled.
self.assertEquals(shifted_ctx[temp_varnode], "temp_varnode_default")
def test_save(self):
tmpdir = tempfile.mkdtemp()
try:
for ext in (".dag", ".zip", ".bz2", ".gz"):
filename = os.path.join(tmpdir, "ctx" + ext)
a = self.ctx[A] = random.randint(0, 100)
self.assertEquals(self.ctx[A], a)
self.ctx.save(filename)
_logger.info("context file '%s' is %db" % (os.path.basename(filename),
os.stat(filename).st_size))
new_ctx = MDFContext.load(filename)
self.assertEquals(new_ctx[A], a)
os.unlink(filename)
finally:
shutil.rmtree(tmpdir, True)
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1),
periods=3,
freq=datetools.yearEnd)
self.ctx = MDFContext()
def setUp(self):
self.ctx = MDFContext()
class NodeFilterTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext(index[0])
def test_queue(self):
actual = self._run(datanode.queuenode(as_list=True))
self.assertEqual(actual.tolist(), queue_expected)
actual = self._run(datanode.queuenode(filter=filternode, as_list=True))
self.assertEqual(actual.tolist(), queue_filtered_expected)
def test_delay(self):
actual = self._run(datanode.delaynode(periods=1, initial_value=np.nan))
self.assertEqual(_normalize(actual), _normalize(delay_expected))
actual = self._run(datanode.delaynode(periods=1, initial_value=np.nan, filter=filternode))
self.assertEqual(_normalize(actual), _normalize(delay_filtered_expected))
def test_delay_df(self):
actual = self._run(dfnode.delaynode(periods=1, initial_value=np.nan))
self.assertEqual(_normalize(actual), _normalize(delay_df_expected))
actual = self._run(dfnode.delaynode(periods=1, initial_value=np.nan, filter=filternode))
self.assertEqual(_normalize(actual), _normalize(delay_df_filtered_expected))
def test_nansum(self):
actual = self._run(datanode.nansumnode())
self.assertEqual(_normalize(actual), _normalize(nansum_expected))
actual = self._run(datanode.nansumnode(filter=filternode))
self.assertEqual(_normalize(actual), _normalize(nansum_filtered_expected))
def test_nansum_df(self):
actual = self._run(dfnode.nansumnode())
self.assertEqual(_normalize(actual), _normalize(nansum_df_expected))
actual = self._run(dfnode.nansumnode(filter=filternode))
self.assertEqual(_normalize(actual), _normalize(nansum_df_filtered_expected))
def test_cumprod(self):
actual = self._run(datanode.cumprodnode())
self.assertEqual(_normalize(actual), _normalize(cumprod_expected))
actual = self._run(datanode.cumprodnode(filter=filternode))
self.assertEqual(_normalize(actual), _normalize(cumprod_filtered_expected))
def test_cumprod_df(self):
actual = self._run(dfnode.cumprodnode())
self.assertEqual(_normalize(actual), _normalize(cumprod_df_expected))
actual = self._run(dfnode.cumprodnode(filter=filternode))
self.assertEqual(_normalize(actual), _normalize(cumprod_df_filtered_expected))
def _run(self, node):
result = []
for t in index:
self.ctx.set_date(t)
result.append(self.ctx[node])
if isinstance(result[0], pa.Series):
return pa.DataFrame(result, index=index)
return pa.Series(result, index, dtype=object)
def setUp(self):
self.ctx = MDFContext()
global num_calls_A, num_calls_A_override
num_calls_A = 0
num_calls_A_override = 0
def setUp(self):
self.ctx = MDFContext(index[0])
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), periods=3, freq=datetools.yearEnd)
self.ctx = MDFContext()
class NodeExtensionsTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext(datetime.now())
self.ctx[D] = 10
self.ctx[E] = 20
def test_node_names(self):
self.assertEqual(A.short_name, "A")
self.assertEqual(B.short_name, "B")
self.assertEqual(C.short_name, "C")
self.assertEqual(D.short_name, "D")
self.assertEqual(E.short_name, "E")
self.assertEqual(F.short_name, "F")
def test_eval(self, ctx=None):
ctx = ctx or self.ctx
# get the calculated value A
a = ctx[A]
# calcualte the expected value
b = ctx[D] * ctx[E]
c = ctx[now].year - 1970
expected_a = b + c
self.assertEqual(a, expected_a)
def test_update(self, ctx=None):
ctx = ctx or self.ctx
orig_b = ctx[B]
ctx[D] *= 2
self.assertEqual(orig_b * 2, ctx[B])
b = ctx[D] * ctx[E]
c = ctx[now].year - 1970
expected_a = b + c
self.assertEqual(ctx[A], expected_a)
def test_clone(self):
# get A before doing anything
orig_a = self.ctx[A]
# clone the context and run the eval and update checks
new_date = self.ctx[now] + timedelta(days=365)
shifted_ctx = self.ctx.shift({now : new_date})
self.test_eval(shifted_ctx)
# shifted contexts are read-only
self.assertRaises(AttributeError,
lambda: self.test_update(shifted_ctx))
# check the original hasn't been affected
self.assertEqual(orig_a, self.ctx[A])
def test_shift(self):
# get A before doing anything
orig_a = self.ctx[A]
# shift the context and run the eval check
shifted_ctx = self.ctx.shift({E : self.ctx[E] * 2})
self.assertEqual(shifted_ctx[E], self.ctx[E] * 2)
self.test_eval(shifted_ctx)
# check the original hasn't been affected
self.assertEqual(orig_a, self.ctx[A])
def test_set_evalnode(self):
# setting an evalnode should fix its value
self.ctx[A] = 100
self.assertEqual(self.ctx[A], 100)
# changing its dependencies shouldn't affect it
self.ctx[B] *= 2
self.ctx[C] *= 2
self.assertEqual(self.ctx[A], 100)
class NodeTest(unittest.TestCase):
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[params.C] = 10
self.ctx[params.D] = 20
def test_queuenode(self):
self._run(queue_output)
queue = self.ctx[queue_output]
self.assertEqual(len(queue), len(self.daterange))
def test_queueyield(self):
self._run(queue_yield)
queue = self.ctx[queue_yield]
self.assertEqual(len(queue), len(self.daterange))
def test_queue_filter(self):
self._run(queue_filter_test)
queue = self.ctx[queue_filter_test]
self.assertEqual(list(queue), [0] * (len(self.daterange) - 1))
@staticmethod
def diff_dfs(lhs_df, rhs_df, tolerance):
diffs = np.abs(lhs_df - rhs_df)
mask = (diffs > tolerance).values
mask &= ~(np.isnan(lhs_df) and np.isnan(rhs_df)).values
mask |= np.isnan(lhs_df).values & ~np.isnan(rhs_df).values
mask |= np.isnan(rhs_df).values & ~np.isnan(lhs_df).values
return mask.any()
def test_nansumnode(self):
self._run(nansum_output)
nansum = self.ctx[nansum_output]
self.assertEqual(nansum, 812)
def test_cumprodnode(self):
self._run(cumprod_output)
cumprod = self.ctx[cumprod_output]
self.assertEqual(cumprod, 14201189062704000)
def test_delaynode(self):
self._run(DelayNodeTest.delay_test, DelayNodeTest.delay_test_lazy)
value = self.ctx[DelayNodeTest.delay_test]
value_lazy = self.ctx[DelayNodeTest.delay_test_lazy]
self.assertEqual(list(value), list(range(1, len(self.daterange)+1)))
self.assertEqual(list(value_lazy), list(range(1, len(self.daterange)+1)))
def test_ffillnode(self):
self._run(ffill_queue)
value = self.ctx[ffill_queue]
self.assertEqual(tuple(value), (0.0, 0.0, 2.0, 2.0, 4.0, 4.0, 6.0))
def test_ffill_array(self):
self._run(ffill_array_test)
value = self.ctx[ffill_array_test]
unfilled_value = self.ctx[ffill_array_test_not_filled]
self.assertTrue(np.isnan(unfilled_value).all())
self.assertEquals(value.tolist(), [10., 10., 10., 10., 10.])
def test_datanode_ffill(self):
data = pd.Series(range(len(self.daterange)), self.daterange, dtype=float)
data = data[[bool(i % 2) for i in range(len(data.index))]]
expected = data.reindex(self.daterange, method="ffill")
expected[np.isnan(expected)] = np.inf
node = datanode("test_datanode_ffill", data, ffill=True, missing_value=np.inf)
qnode = node.queuenode()
self._run(qnode)
value = self.ctx[qnode]
self.assertEquals(list(value), expected.values.tolist())
def test_lookahead_node(self):
B_queue = B.queuenode()
B_lookahead = B.lookaheadnode(periods=len(self.daterange))
self.ctx.set_date(self.daterange[0])
actual = self.ctx[B_lookahead]
self._run(B_queue)
expected = self.ctx[B_queue]
self.assertEquals(actual.values.tolist(), list(expected))
self.assertEquals(actual.index.tolist(), list(self.daterange))
def test_apply_node(self):
actual_node = A.applynode(func=operator.add, args=(B,)).queuenode()
expected_node = A_plus_B.queuenode()
self._run(actual_node, expected_node)
actual = self.ctx[actual_node]
expected = self.ctx[expected_node]
self.assertEquals(actual, expected)
def test_binary_operators_with_constant(self):
self._test(Counter, [-2.0, -1.5, -1.0, -0.5, 0.5, 1.0, 1.5])
self._test(Counter + 0.2, [-1.8, -1.3, -0.8, -0.3, 0.7, 1.2, 1.7])
self._test(Counter - 0.2, [-2.2, -1.7, -1.2, -0.7, 0.3, 0.8, 1.3])
self._test(Counter * 2.0, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(Counter / 0.5, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(0.2 + Counter, [-1.8, -1.3, -0.8, -0.3, 0.7, 1.2, 1.7])
self._test(1.0 - Counter, [ 3.0, 2.5, 2.0, 1.5, 0.5, 0.0, -0.5])
self._test(2.0 * Counter, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(12 / (Counter+.25), [-6.8571428571428568, -9.6000000000000014, -16.0,
-48.0, 16.0, 9.6000000000000014, 6.8571428571428568])
def test_binary_operators_with_node(self):
self._test(Counter + Counter, [-4.0, -3.0, -2.0, -1.0, 1.0, 2.0, 3.0])
self._test(Counter - Counter, [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
self._test(Counter * Counter, [ 4.0, 2.25, 1.0, 0.25, 0.25, 1.0, 2.25])
self._test(Counter / Counter, [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
def _test(self, node, expected_values):
values = node.queuenode()
self._run(values)
actual = self.ctx[values]
self.assertEquals(list(actual), expected_values)
def _run_for_daterange(self, date_range, *nodes):
for t in date_range:
self.ctx.set_date(t)
for node in nodes:
self.ctx[node]
def _run(self, *nodes):
self._run_for_daterange(self.daterange, *nodes)
def setUp(self):
self.ctx = MDFContext(datetime.now())
self.ctx[D] = 10
self.ctx[E] = 20
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1), datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[params.C] = 10
self.ctx[params.D] = 20
def setUp(self):
self.ctx = MDFContext(datetime.now())
self.ctx[x] = 1
self.ctx[y] = 2
self.ctx[z] = 3
def setUp(self):
self.ctx = MDFContext()
global num_calls_A, num_calls_A_override
num_calls_A = 0
num_calls_A_override = 0
class NodeExtensionsTest(unittest.TestCase):
def setUp(self):
self.ctx = MDFContext(datetime.now())
self.ctx[D] = 10
self.ctx[E] = 20
def test_node_names(self):
self.assertEqual(A.short_name, "A")
self.assertEqual(B.short_name, "B")
self.assertEqual(C.short_name, "C")
self.assertEqual(D.short_name, "D")
self.assertEqual(E.short_name, "E")
self.assertEqual(F.short_name, "F")
def test_eval(self, ctx=None):
ctx = ctx or self.ctx
# get the calculated value A
a = ctx[A]
# calcualte the expected value
b = ctx[D] * ctx[E]
c = ctx[now].year - 1970
expected_a = b + c
self.assertEqual(a, expected_a)
def test_update(self, ctx=None):
ctx = ctx or self.ctx
orig_b = ctx[B]
ctx[D] *= 2
self.assertEqual(orig_b * 2, ctx[B])
b = ctx[D] * ctx[E]
c = ctx[now].year - 1970
expected_a = b + c
self.assertEqual(ctx[A], expected_a)
def test_clone(self):
# get A before doing anything
orig_a = self.ctx[A]
# clone the context and run the eval and update checks
new_date = self.ctx[now] + timedelta(days=365)
shifted_ctx = self.ctx.shift({now: new_date})
self.test_eval(shifted_ctx)
# shifted contexts are read-only
self.assertRaises(AttributeError,
lambda: self.test_update(shifted_ctx))
# check the original hasn't been affected
self.assertEqual(orig_a, self.ctx[A])
def test_shift(self):
# get A before doing anything
orig_a = self.ctx[A]
# shift the context and run the eval check
shifted_ctx = self.ctx.shift({E: self.ctx[E] * 2})
self.assertEqual(shifted_ctx[E], self.ctx[E] * 2)
self.test_eval(shifted_ctx)
# check the original hasn't been affected
self.assertEqual(orig_a, self.ctx[A])
def test_set_evalnode(self):
# setting an evalnode should fix its value
self.ctx[A] = 100
self.assertEqual(self.ctx[A], 100)
# changing its dependencies shouldn't affect it
self.ctx[B] *= 2
self.ctx[C] *= 2
self.assertEqual(self.ctx[A], 100)
def setUp(self):
self.daterange = pd.bdate_range(datetime(1970, 1, 1),
datetime(1970, 1, 10))
self.ctx = MDFContext()
self.ctx[_TestNodes.C] = 10
self.ctx[_TestNodes.D] = 20
def setUp(self):
self.ctx = MDFContext(datetime.now())
self.ctx[D] = 10
self.ctx[E] = 20
