def test_simple(self): """Run some simple tests for date-time cycling.""" init(time_zone='Z') p_start = ISO8601Point('20100808T00') p_stop = ISO8601Point('20100808T02') i = ISO8601Interval('PT6H') self.assertEqual(p_start - i, ISO8601Point('20100807T18')) self.assertEqual(p_stop + i, ISO8601Point('20100808T08')) sequence = ISO8601Sequence('PT10M', str(p_start), str(p_stop),) sequence.set_offset(- ISO8601Interval('PT10M')) point = sequence.get_next_point(ISO8601Point('20100808T0000')) self.assertEqual(point, ISO8601Point('20100808T0010')) output = [] # Test point generation forwards. while point and point < p_stop: output.append(point) self.assertTrue(sequence.is_on_sequence(point)) point = sequence.get_next_point(point) self.assertEqual([str(out) for out in output], ['20100808T0010Z', '20100808T0020Z', '20100808T0030Z', '20100808T0040Z', '20100808T0050Z', '20100808T0100Z', '20100808T0110Z', '20100808T0120Z', '20100808T0130Z', '20100808T0140Z', '20100808T0150Z']) self.assertEqual(point, ISO8601Point('20100808T0200')) # Test point generation backwards. output = [] while point and point >= p_start: output.append(point) self.assertTrue(sequence.is_on_sequence(point)) point = sequence.get_prev_point(point) self.assertEqual([str(out) for out in output], ['20100808T0200Z', '20100808T0150Z', '20100808T0140Z', '20100808T0130Z', '20100808T0120Z', '20100808T0110Z', '20100808T0100Z', '20100808T0050Z', '20100808T0040Z', '20100808T0030Z', '20100808T0020Z', '20100808T0010Z', '20100808T0000Z']) self.assertFalse( sequence.is_on_sequence(ISO8601Point('20100809T0005')))
def test_exclusions_simple(self): """Test the generation of points for sequences with exclusions.""" init(time_zone='Z') sequence = ISO8601Sequence('PT1H!20000101T02Z', '20000101T00Z') output = [] point = sequence.get_start_point() count = 0 while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] self.assertEqual(output, [ '20000101T0000Z', '20000101T0100Z', '20000101T0300Z', '20000101T0400Z' ])
def test_check_xtriggers(xtrigger_mgr): """Test process_xtriggers call.""" xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation # add an xtrigger get_name = SubFuncContext(label="get_name", func_name="get_name", func_args=[], func_kwargs={}) xtrigger_mgr.add_trig("get_name", get_name, 'fdir') get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]" tdef1 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1, initial_point=1) init() sequence = ISO8601Sequence('P1D', '2019') tdef1.xtrig_labels[sequence] = ["get_name"] start_point = ISO8601Point('2019') itask1 = TaskProxy(tdef1, start_point, FlowLabelMgr().get_new_label()) itask1.state.xtriggers["get_name"] = False # satisfied? # add a clock xtrigger wall_clock = SubFuncContext(label="wall_clock", func_name="wall_clock", func_args=[], func_kwargs={}) wall_clock.out = "[\"True\", \"1\"]" xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir") # create a task tdef2 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1, initial_point=1) tdef2.xtrig_labels[sequence] = ["wall_clock"] init() start_point = ISO8601Point('20000101T0000+05') # create task proxy itask2 = TaskProxy(tdef2, start_point, FlowLabelMgr().get_new_label()) xtrigger_mgr.check_xtriggers(itask1, lambda foo: None) # won't be satisfied, as it is async, we are are not calling callback assert not xtrigger_mgr.sat_xtrig
def test_check_xtriggers(xtrigger_mgr): """Test check_xtriggers call. check_xtriggers does pretty much the same as collate. The difference is that besides tallying on all the xtriggers and clock xtriggers available, it then proceeds to trying to satisfy them.""" xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation # add a xtrigger # that will cause all_xtrig to be populated, but not all_xclock get_name = SubFuncContext(label="get_name", func_name="get_name", func_args=[], func_kwargs={}) xtrigger_mgr.add_trig("get_name", get_name, 'fdir') get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]" tdef1 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1) init() sequence = ISO8601Sequence('P1D', '2019') tdef1.xtrig_labels[sequence] = ["get_name"] start_point = ISO8601Point('2019') itask1 = TaskProxy(tdef1, start_point, FlowLabelMgr().get_new_label()) itask1.state.xtriggers["get_name"] = False # satisfied? # add a clock xtrigger # that will cause both all_xclock to be populated but not all_xtrig wall_clock = SubFuncContext(label="wall_clock", func_name="wall_clock", func_args=[], func_kwargs={}) wall_clock.out = "[\"True\", \"1\"]" xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir") # create a task tdef2 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1) tdef2.xtrig_labels[sequence] = ["wall_clock"] init() start_point = ISO8601Point('20000101T0000+05') # create task proxy itask2 = TaskProxy(tdef2, start_point, FlowLabelMgr().get_new_label()) xtrigger_mgr.check_xtriggers([itask1, itask2]) # won't be satisfied, as it is async, we are are not calling callback assert not xtrigger_mgr.sat_xtrig assert xtrigger_mgr.all_xtrig
def test_exclusions_sequences_points(self): """Test ISO8601Sequence methods for sequences with exclusions""" init(time_zone='Z') # Run every hour from 01:00 excluding every 3rd hour sequence = ISO8601Sequence('T01/PT1H!PT3H', '20000101T01Z') point_0 = ISO8601Point('20000101T00Z') point_1 = ISO8601Point('20000101T01Z') point_2 = ISO8601Point('20000101T02Z') point_3 = ISO8601Point('20000101T03Z') point_4 = ISO8601Point('20000101T04Z') self.assertFalse(point_0 in sequence.exclusions) self.assertTrue(point_1 in sequence.exclusions) self.assertTrue(sequence.is_on_sequence(point_2)) self.assertTrue(sequence.is_on_sequence(point_3)) self.assertFalse(sequence.is_on_sequence(point_4)) self.assertTrue(point_4 in sequence.exclusions)
def test_multiple_exclusions_simple(self): """Tests generation of points for sequences with multiple exclusions """ init(time_zone='Z') sequence = ISO8601Sequence('PT1H!(20000101T02Z,20000101T03Z)', '20000101T00Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make four sequence points while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect two of the hours to be excluded: T02 and T03 self.assertEqual(output, ['20000101T0000Z', '20000101T0100Z', '20000101T0400Z', '20000101T0500Z'])
def test_collate(xtrigger_mgr): """Test that collate properly tallies the totals of current xtriggers.""" xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation xtrigger_mgr.collate(itasks=[]) assert not xtrigger_mgr.all_xtrig # add a xtrigger # that will cause all_xtrig to be populated get_name = SubFuncContext(label="get_name", func_name="get_name", func_args=[], func_kwargs={}) xtrigger_mgr.add_trig("get_name", get_name, 'fdir') get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]" tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1) init() sequence = ISO8601Sequence('P1D', '20190101T00Z') tdef.xtrig_labels[sequence] = ["get_name"] start_point = ISO8601Point('2019') itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label()) itask.state.xtriggers["get_name"] = get_name xtrigger_mgr.collate([itask]) assert xtrigger_mgr.all_xtrig # add a clock xtrigger # that will cause both all_xclock to be populated but not all_xtrig wall_clock = SubFuncContext(label="wall_clock", func_name="wall_clock", func_args=[], func_kwargs={}) wall_clock.out = "[\"True\", \"1\"]" xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir") # create a task tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1) tdef.xtrig_labels[sequence] = ["wall_clock"] start_point = ISO8601Point('20000101T0000+05') # create task proxy itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label()) xtrigger_mgr.collate([itask]) assert not xtrigger_mgr.all_xtrig
def test_multiple_exclusions_complex1(self): """Tests sequences that have multiple exclusions and a more complicated format""" # A sequence that specifies a dep start time sequence = ISO8601Sequence('20000101T01Z/PT1H!20000101T02Z', '20000101T01Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make four sequence points while point and count < 4: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect one of the hours to be excluded: T02 self.assertEqual(output, ['20000101T0100Z', '20000101T0300Z', '20000101T0400Z', '20000101T0500Z'])
def test_advanced_exclusions_sequences_implied_start_point(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run every hour from 01:00 excluding every 3rd hour. sequence = ISO8601Sequence('T05/PT1H!PT3H', '20000101T00Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] self.assertEqual(output, ['20000101T0600Z', '20000101T0700Z', '20000101T0900Z', '20000101T1000Z', '20000101T1200Z', '20000101T1300Z'])
def test_advanced_exclusions_sequences3(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run daily at 12:00 except every 3rd day sequence = ISO8601Sequence('T12!P3D', '20000101T12Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] self.assertEqual(output, ['20000102T1200Z', '20000103T1200Z', '20000105T1200Z', '20000106T1200Z', '20000108T1200Z', '20000109T1200Z'])
def test_advanced_exclusions_sequences2(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run hourly on the hour but not 3 hourly on the hour sequence = ISO8601Sequence('T-00!T-00/PT3H', '20000101T00Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] self.assertEqual(output, ['20000101T0100Z', '20000101T0200Z', '20000101T0400Z', '20000101T0500Z', '20000101T0700Z', '20000101T0800Z'])
def test_advanced_exclusions_sequences1(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run hourly from the ICP but not 3-hourly sequence = ISO8601Sequence('PT1H!PT3H', '20000101T01Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make six sequence points while point and count < 6: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect to see hourly from ICP but not 3 hourly self.assertEqual(output, ['20000101T0200Z', '20000101T0300Z', '20000101T0500Z', '20000101T0600Z', '20000101T0800Z', '20000101T0900Z'])
def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr): """The housekeeping method makes sure only satisfied xtrigger function are kept.""" xtrig = SubFuncContext(label="get_name", func_name="get_name", func_args=[], func_kwargs={}) xtrigger_mgr.add_trig("get_name", xtrig, 'fdir') xtrig.out = "[\"True\", {\"name\": \"Yossarian\"}]" tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1) init() sequence = ISO8601Sequence('P1D', '2019') tdef.xtrig_labels[sequence] = ["get_name"] start_point = ISO8601Point('2019') itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label()) xtrigger_mgr.collate([itask]) # pretend the function has been activated xtrigger_mgr.active.append(xtrig.get_signature()) xtrigger_mgr.callback(xtrig) assert xtrigger_mgr.sat_xtrig xtrigger_mgr.housekeep() # here we still have the same number as before assert xtrigger_mgr.sat_xtrig
def test_advanced_exclusions_partial_datetime1(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run 3-hourly but not at 06:00 (from the ICP) sequence = ISO8601Sequence('PT3H!T06', '20000101T00Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make ten sequence points while point and count < 10: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect every T06 to be excluded self.assertEqual(output, ['20000101T0000Z', '20000101T0300Z', '20000101T0900Z', '20000101T1200Z', '20000101T1500Z', '20000101T1800Z', '20000101T2100Z', '20000102T0000Z', '20000102T0300Z', '20000102T0900Z'])
def test_advanced_exclusions_partial_datetime4(self): """Advanced exclusions refers to exclusions that are not just simple points but could be time periods or recurrences such as '!T06' or similar""" init(time_zone='Z') # Run daily at 00:00 except on Mondays sequence = ISO8601Sequence('T00!W-1T00', '20170422T00Z') output = [] point = sequence.get_start_point() count = 0 # We are going to make 19 sequence points while point and count < 9: output.append(point) point = sequence.get_next_point(point) count += 1 output = [str(out) for out in output] # We should expect Monday 24th April and Monday 1st May # to be excluded. self.assertEqual(output, [ '20170422T0000Z', '20170423T0000Z', '20170425T0000Z', '20170426T0000Z', '20170427T0000Z', '20170428T0000Z', '20170429T0000Z', '20170430T0000Z', '20170502T0000Z' ])
def test__call_xtriggers_async(xtrigger_mgr): """Test _call_xtriggers_async""" xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation # the echo1 xtrig (not satisfied) echo1_xtrig = SubFuncContext( label="echo1", func_name="echo1", func_args=[], func_kwargs={} ) echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]" xtrigger_mgr.add_trig("echo1", echo1_xtrig, "fdir") # the echo2 xtrig (satisfied through callback later) echo2_xtrig = SubFuncContext( label="echo2", func_name="echo2", func_args=[], func_kwargs={} ) echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]" xtrigger_mgr.add_trig("echo2", echo2_xtrig, "fdir") # create a task tdef = TaskDef( name="foo", rtcfg=None, run_mode="live", start_point=1, initial_point=1 ) init() sequence = ISO8601Sequence('P1D', '2000') tdef.xtrig_labels[sequence] = ["echo1", "echo2"] # cycle point for task proxy init() start_point = ISO8601Point('2019') # create task proxy itask = TaskProxy(tdef, start_point) # we start with no satisfied xtriggers, and nothing active assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 0 # after calling the first time, we get two active xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 2 # calling again does not change anything xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 0 assert len(xtrigger_mgr.active) == 2 # now we call callback manually as the proc_pool we passed is a mock # then both should be satisfied xtrigger_mgr.callback(echo1_xtrig) xtrigger_mgr.callback(echo2_xtrig) # so both were satisfied, and nothing is active assert len(xtrigger_mgr.sat_xtrig) == 2 assert len(xtrigger_mgr.active) == 0 # calling satisfy_xtriggers again still does not change anything xtrigger_mgr.call_xtriggers_async(itask) assert len(xtrigger_mgr.sat_xtrig) == 2 assert len(xtrigger_mgr.active) == 0