def test_advanced_exclusions_partial_datetime3(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run 5 minutely but not at 15 minutes past the hour from ICP
sequence = ISO8601Sequence('PT5M!T-15', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make 15 sequence points
while point and count < 15:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect xx:15 (15 minutes past the hour) to be excluded
self.assertEqual(output, ['20000101T0000Z', '20000101T0005Z',
'20000101T0010Z',
'20000101T0020Z', '20000101T0025Z',
'20000101T0030Z', '20000101T0035Z',
'20000101T0040Z', '20000101T0045Z',
'20000101T0050Z', '20000101T0055Z',
'20000101T0100Z', '20000101T0105Z',
'20000101T0110Z', '20000101T0120Z'])
def test_advanced_exclusions_partial_datetime2(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run hourly but not at 00:00, 06:00, 12:00, 18:00
sequence = ISO8601Sequence('T-00!(T00, T06, T12, T18)', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make 18 sequence points
while point and count < 18:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect T00, T06, T12, and T18 to be excluded
self.assertEqual(output, ['20000101T0100Z', '20000101T0200Z',
'20000101T0300Z', '20000101T0400Z',
'20000101T0500Z', '20000101T0700Z',
'20000101T0800Z', '20000101T0900Z',
'20000101T1000Z', '20000101T1100Z',
'20000101T1300Z', '20000101T1400Z',
'20000101T1500Z', '20000101T1600Z',
'20000101T1700Z', '20000101T1900Z',
'20000101T2000Z', '20000101T2100Z'])
def test_exclusions_to_string(self):
init(time_zone='Z')
# Check that exclusions are not included where they should not be.
basic = ISO8601Sequence('PT1H', '2000', '2001')
self.assertFalse('!' in str(basic))
# Check that exclusions are parsable.
sequence = ISO8601Sequence('PT1H!(20000101T10Z, PT6H)', '2000', '2001')
sequence2 = ISO8601Sequence(str(sequence), '2000', '2001')
self.assertEqual(sequence, sequence2)
def test_exclusions_simple(self):
"""Test the generation of points for sequences with exclusions."""
init(time_zone='Z')
sequence = ISO8601Sequence('PT1H!20000101T02Z', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
while point and count < 4:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000101T0000Z', '20000101T0100Z',
'20000101T0300Z', '20000101T0400Z'])
def test_simple(self):
"""Run some simple tests for date-time cycling."""
init(time_zone='Z')
p_start = ISO8601Point('20100808T00')
p_stop = ISO8601Point('20100808T02')
i = ISO8601Interval('PT6H')
self.assertEqual(p_start - i, ISO8601Point('20100807T18'))
self.assertEqual(p_stop + i, ISO8601Point('20100808T08'))
sequence = ISO8601Sequence('PT10M', str(p_start), str(p_stop),)
sequence.set_offset(- ISO8601Interval('PT10M'))
point = sequence.get_next_point(ISO8601Point('20100808T0000'))
self.assertEqual(point, ISO8601Point('20100808T0010'))
output = []
# Test point generation forwards.
while point and point < p_stop:
output.append(point)
self.assertTrue(sequence.is_on_sequence(point))
point = sequence.get_next_point(point)
self.assertEqual([str(out) for out in output],
['20100808T0010Z', '20100808T0020Z',
'20100808T0030Z', '20100808T0040Z',
'20100808T0050Z', '20100808T0100Z',
'20100808T0110Z', '20100808T0120Z',
'20100808T0130Z', '20100808T0140Z',
'20100808T0150Z'])
self.assertEqual(point, ISO8601Point('20100808T0200'))
# Test point generation backwards.
output = []
while point and point >= p_start:
output.append(point)
self.assertTrue(sequence.is_on_sequence(point))
point = sequence.get_prev_point(point)
self.assertEqual([str(out) for out in output],
['20100808T0200Z', '20100808T0150Z',
'20100808T0140Z', '20100808T0130Z',
'20100808T0120Z', '20100808T0110Z',
'20100808T0100Z', '20100808T0050Z',
'20100808T0040Z', '20100808T0030Z',
'20100808T0020Z', '20100808T0010Z',
'20100808T0000Z'])
self.assertFalse(
sequence.is_on_sequence(ISO8601Point('20100809T0005')))
def test_exclusions_simple(self):
"""Test the generation of points for sequences with exclusions."""
init(time_zone='Z')
sequence = ISO8601Sequence('PT1H!20000101T02Z', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
while point and count < 4:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, [
'20000101T0000Z', '20000101T0100Z', '20000101T0300Z',
'20000101T0400Z'
])
def test_check_xtriggers(xtrigger_mgr):
"""Test check_xtriggers call.
check_xtriggers does pretty much the same as collate. The
difference is that besides tallying on all the xtriggers and
clock xtriggers available, it then proceeds to trying to
satisfy them."""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
# add a xtrigger
# that will cause all_xtrig to be populated, but not all_xclock
get_name = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef1 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef1.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask1 = TaskProxy(tdef1, start_point, FlowLabelMgr().get_new_label())
itask1.state.xtriggers["get_name"] = False # satisfied?
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef2 = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1)
tdef2.xtrig_labels[sequence] = ["wall_clock"]
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask2 = TaskProxy(tdef2, start_point, FlowLabelMgr().get_new_label())
xtrigger_mgr.check_xtriggers([itask1, itask2])
# won't be satisfied, as it is async, we are are not calling callback
assert not xtrigger_mgr.sat_xtrig
assert xtrigger_mgr.all_xtrig
def test_check_xtriggers(xtrigger_mgr):
"""Test process_xtriggers call."""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
# add an xtrigger
get_name = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef1 = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef1.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask1 = TaskProxy(tdef1, start_point, FlowLabelMgr().get_new_label())
itask1.state.xtriggers["get_name"] = False # satisfied?
# add a clock xtrigger
wall_clock = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef2 = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1)
tdef2.xtrig_labels[sequence] = ["wall_clock"]
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask2 = TaskProxy(tdef2, start_point, FlowLabelMgr().get_new_label())
xtrigger_mgr.check_xtriggers(itask1, lambda foo: None)
# won't be satisfied, as it is async, we are are not calling callback
assert not xtrigger_mgr.sat_xtrig
def test_exclusions_sequences_points(self):
"""Test ISO8601Sequence methods for sequences with exclusions"""
init(time_zone='Z')
# Run every hour from 01:00 excluding every 3rd hour
sequence = ISO8601Sequence('T01/PT1H!PT3H', '20000101T01Z')
point_0 = ISO8601Point('20000101T00Z')
point_1 = ISO8601Point('20000101T01Z')
point_2 = ISO8601Point('20000101T02Z')
point_3 = ISO8601Point('20000101T03Z')
point_4 = ISO8601Point('20000101T04Z')
self.assertFalse(point_0 in sequence.exclusions)
self.assertTrue(point_1 in sequence.exclusions)
self.assertTrue(sequence.is_on_sequence(point_2))
self.assertTrue(sequence.is_on_sequence(point_3))
self.assertFalse(sequence.is_on_sequence(point_4))
self.assertTrue(point_4 in sequence.exclusions)
def test_exclusions_sequences_points(self):
"""Test ISO8601Sequence methods for sequences with exclusions"""
init(time_zone='Z')
# Run every hour from 01:00 excluding every 3rd hour
sequence = ISO8601Sequence('T01/PT1H!PT3H', '20000101T01Z')
point_0 = ISO8601Point('20000101T00Z')
point_1 = ISO8601Point('20000101T01Z')
point_2 = ISO8601Point('20000101T02Z')
point_3 = ISO8601Point('20000101T03Z')
point_4 = ISO8601Point('20000101T04Z')
self.assertFalse(point_0 in sequence.exclusions)
self.assertTrue(point_1 in sequence.exclusions)
self.assertTrue(sequence.is_on_sequence(point_2))
self.assertTrue(sequence.is_on_sequence(point_3))
self.assertFalse(sequence.is_on_sequence(point_4))
self.assertTrue(point_4 in sequence.exclusions)
def test_process_icp(
scheduling_cfg: Dict[str, Any], expected_icp: Optional[str],
expected_opt_icp: Optional[str],
expected_err: Optional[Tuple[Type[Exception], str]],
monkeypatch: Fixture, cycling_mode: Fixture):
"""Test SuiteConfig.process_initial_cycle_point().
"now" is assumed to be 2005-01-02T06:15+0530
Params:
scheduling_cfg: 'scheduling' section of workflow config.
expected_icp: The expected icp value that gets set.
expected_opt_icp: The expected value of options.icp that gets set
(this gets stored in the workflow DB).
expected_err: Exception class expected to be raised plus the message.
"""
int_cycling_mode = True
if scheduling_cfg['cycling mode'] == loader.ISO8601_CYCLING_TYPE:
int_cycling_mode = False
iso8601.init(time_zone="+0530")
cycling_mode(integer=int_cycling_mode)
mocked_config = Mock()
mocked_config.cfg = {
'scheduling': scheduling_cfg
}
mocked_config.options.icp = None
monkeypatch.setattr('cylc.flow.config.get_current_time_string',
lambda: '20050102T0615+0530')
if expected_err:
err, msg = expected_err
with pytest.raises(err) as exc:
SuiteConfig.process_initial_cycle_point(mocked_config)
assert msg in str(exc.value)
else:
SuiteConfig.process_initial_cycle_point(mocked_config)
assert mocked_config.cfg[
'scheduling']['initial cycle point'] == expected_icp
assert str(mocked_config.initial_point) == expected_icp
opt_icp = mocked_config.options.icp
if opt_icp is not None:
opt_icp = str(loader.get_point(opt_icp).standardise())
assert opt_icp == expected_opt_icp
def test_multiple_exclusions_simple(self):
"""Tests generation of points for sequences with multiple exclusions
"""
init(time_zone='Z')
sequence = ISO8601Sequence('PT1H!(20000101T02Z,20000101T03Z)',
'20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make four sequence points
while point and count < 4:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect two of the hours to be excluded: T02 and T03
self.assertEqual(output, ['20000101T0000Z', '20000101T0100Z',
'20000101T0400Z', '20000101T0500Z'])
def test_multiple_exclusions_simple(self):
"""Tests generation of points for sequences with multiple exclusions
"""
init(time_zone='Z')
sequence = ISO8601Sequence('PT1H!(20000101T02Z,20000101T03Z)',
'20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make four sequence points
while point and count < 4:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect two of the hours to be excluded: T02 and T03
self.assertEqual(output, ['20000101T0000Z', '20000101T0100Z',
'20000101T0400Z', '20000101T0500Z'])
def test_collate(xtrigger_mgr):
"""Test that collate properly tallies the totals of current xtriggers."""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
xtrigger_mgr.collate(itasks=[])
assert not xtrigger_mgr.all_xtrig
# add a xtrigger
# that will cause all_xtrig to be populated
get_name = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1)
init()
sequence = ISO8601Sequence('P1D', '20190101T00Z')
tdef.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label())
itask.state.xtriggers["get_name"] = get_name
xtrigger_mgr.collate([itask])
assert xtrigger_mgr.all_xtrig
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1)
tdef.xtrig_labels[sequence] = ["wall_clock"]
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label())
xtrigger_mgr.collate([itask])
assert not xtrigger_mgr.all_xtrig
def test_multiple_exclusions_extensive(self):
"""Test ISO8601Sequence methods for sequences with multiple exclusions
"""
init(time_zone='+05')
sequence = ISO8601Sequence('PT1H!(20000101T02,20000101T03)',
'20000101T00',
'20000101T06')
point_0 = ISO8601Point('20000101T0000+05')
point_1 = ISO8601Point('20000101T0100+05')
point_2 = ISO8601Point('20000101T0200+05') # First excluded point
point_3 = ISO8601Point('20000101T0300+05') # Second excluded point
point_4 = ISO8601Point('20000101T0400+05')
# Check the excluded points are not on the sequence
self.assertFalse(sequence.is_on_sequence(point_2))
self.assertFalse(sequence.is_on_sequence(point_3))
self.assertFalse(sequence.is_valid(point_2)) # Should be excluded
self.assertFalse(sequence.is_valid(point_3)) # Should be excluded
# Check that we can correctly retrieve previous points
self.assertEqual(sequence.get_prev_point(point_2), point_1)
# Should skip two excluded points
self.assertEqual(sequence.get_prev_point(point_4), point_1)
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_nearest_prev_point(point_4), point_1)
self.assertEqual(sequence.get_next_point(point_1), point_4)
self.assertEqual(sequence.get_next_point(point_3), point_4)
sequence = ISO8601Sequence('PT1H!20000101T00+05', '20000101T00')
# Check that the first point is after 00.
self.assertEqual(sequence.get_first_point(point_0), point_1)
self.assertEqual(sequence.get_start_point(), point_1)
# Check a longer list of exclusions
# Also note you can change the format of the exclusion list
# (removing the parentheses)
sequence = ISO8601Sequence('PT1H!(20000101T02+05, 20000101T03+05,'
'20000101T04+05)',
'20000101T00',
'20000101T06')
self.assertEqual(sequence.get_prev_point(point_3), point_1)
self.assertEqual(sequence.get_prev_point(point_4), point_1)
def test_multiple_exclusions_extensive(self):
"""Test ISO8601Sequence methods for sequences with multiple exclusions
"""
init(time_zone='+05')
sequence = ISO8601Sequence('PT1H!(20000101T02,20000101T03)',
'20000101T00',
'20000101T06')
point_0 = ISO8601Point('20000101T0000+05')
point_1 = ISO8601Point('20000101T0100+05')
point_2 = ISO8601Point('20000101T0200+05') # First excluded point
point_3 = ISO8601Point('20000101T0300+05') # Second excluded point
point_4 = ISO8601Point('20000101T0400+05')
# Check the excluded points are not on the sequence
self.assertFalse(sequence.is_on_sequence(point_2))
self.assertFalse(sequence.is_on_sequence(point_3))
self.assertFalse(sequence.is_valid(point_2)) # Should be excluded
self.assertFalse(sequence.is_valid(point_3)) # Should be excluded
# Check that we can correctly retrieve previous points
self.assertEqual(sequence.get_prev_point(point_2), point_1)
# Should skip two excluded points
self.assertEqual(sequence.get_prev_point(point_4), point_1)
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_nearest_prev_point(point_4), point_1)
self.assertEqual(sequence.get_next_point(point_1), point_4)
self.assertEqual(sequence.get_next_point(point_3), point_4)
sequence = ISO8601Sequence('PT1H!20000101T00+05', '20000101T00')
# Check that the first point is after 00.
self.assertEqual(sequence.get_first_point(point_0), point_1)
self.assertEqual(sequence.get_start_point(), point_1)
# Check a longer list of exclusions
# Also note you can change the format of the exclusion list
# (removing the parentheses)
sequence = ISO8601Sequence('PT1H!(20000101T02+05, 20000101T03+05,'
'20000101T04+05)',
'20000101T00',
'20000101T06')
self.assertEqual(sequence.get_prev_point(point_3), point_1)
self.assertEqual(sequence.get_prev_point(point_4), point_1)
def test_process_startcp(startcp: Optional[str], expected: str,
monkeypatch: Fixture, cycling_mode: Fixture):
"""Test SuiteConfig.process_start_cycle_point().
An icp of 1899-05-01T00+0530 is assumed, and "now" is assumed to be
2005-01-02T06:15+0530
Params:
startcp: The start cycle point given by cli option.
expected: The expected startcp value that gets set.
"""
iso8601.init(time_zone="+0530")
cycling_mode(integer=False)
mocked_config = Mock(initial_point='18990501T0000+0530')
mocked_config.options.startcp = startcp
monkeypatch.setattr('cylc.flow.config.get_current_time_string',
lambda: '20050102T0615+0530')
SuiteConfig.process_start_cycle_point(mocked_config)
assert str(mocked_config.start_point) == expected
def test_advanced_exclusions_sequences_implied_start_point(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run every hour from 01:00 excluding every 3rd hour.
sequence = ISO8601Sequence('T05/PT1H!PT3H', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000101T0600Z', '20000101T0700Z',
'20000101T0900Z', '20000101T1000Z',
'20000101T1200Z', '20000101T1300Z'])
def test_advanced_exclusions_sequences3(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run daily at 12:00 except every 3rd day
sequence = ISO8601Sequence('T12!P3D', '20000101T12Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000102T1200Z', '20000103T1200Z',
'20000105T1200Z', '20000106T1200Z',
'20000108T1200Z', '20000109T1200Z'])
def test_advanced_exclusions_sequences2(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run hourly on the hour but not 3 hourly on the hour
sequence = ISO8601Sequence('T-00!T-00/PT3H', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000101T0100Z', '20000101T0200Z',
'20000101T0400Z', '20000101T0500Z',
'20000101T0700Z', '20000101T0800Z'])
def test_advanced_exclusions_sequences1(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run hourly from the ICP but not 3-hourly
sequence = ISO8601Sequence('PT1H!PT3H', '20000101T01Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect to see hourly from ICP but not 3 hourly
self.assertEqual(output, ['20000101T0200Z', '20000101T0300Z',
'20000101T0500Z', '20000101T0600Z',
'20000101T0800Z', '20000101T0900Z'])
def test_advanced_exclusions_sequences1(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run hourly from the ICP but not 3-hourly
sequence = ISO8601Sequence('PT1H!PT3H', '20000101T01Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect to see hourly from ICP but not 3 hourly
self.assertEqual(output, ['20000101T0200Z', '20000101T0300Z',
'20000101T0500Z', '20000101T0600Z',
'20000101T0800Z', '20000101T0900Z'])
def test_advanced_exclusions_sequences3(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run daily at 12:00 except every 3rd day
sequence = ISO8601Sequence('T12!P3D', '20000101T12Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000102T1200Z', '20000103T1200Z',
'20000105T1200Z', '20000106T1200Z',
'20000108T1200Z', '20000109T1200Z'])
def test_advanced_exclusions_sequences_implied_start_point(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run every hour from 01:00 excluding every 3rd hour.
sequence = ISO8601Sequence('T05/PT1H!PT3H', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000101T0600Z', '20000101T0700Z',
'20000101T0900Z', '20000101T1000Z',
'20000101T1200Z', '20000101T1300Z'])
def test_advanced_exclusions_sequences2(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run hourly on the hour but not 3 hourly on the hour
sequence = ISO8601Sequence('T-00!T-00/PT3H', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make six sequence points
while point and count < 6:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
self.assertEqual(output, ['20000101T0100Z', '20000101T0200Z',
'20000101T0400Z', '20000101T0500Z',
'20000101T0700Z', '20000101T0800Z'])
def test_advanced_exclusions_partial_datetime1(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run 3-hourly but not at 06:00 (from the ICP)
sequence = ISO8601Sequence('PT3H!T06', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make ten sequence points
while point and count < 10:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect every T06 to be excluded
self.assertEqual(output, ['20000101T0000Z', '20000101T0300Z',
'20000101T0900Z', '20000101T1200Z',
'20000101T1500Z', '20000101T1800Z',
'20000101T2100Z', '20000102T0000Z',
'20000102T0300Z', '20000102T0900Z'])
def test_exclusions_extensive(self):
"""Test ISO8601Sequence methods for sequences with exclusions"""
init(time_zone='+05')
sequence = ISO8601Sequence('PT1H!20000101T02+05', '20000101T00',
'20000101T05')
point_0 = ISO8601Point('20000101T0000+05')
point_1 = ISO8601Point('20000101T0100+05')
point_2 = ISO8601Point('20000101T0200+05') # The excluded point.
point_3 = ISO8601Point('20000101T0300+05')
self.assertFalse(sequence.is_on_sequence(point_2))
self.assertFalse(sequence.is_valid(point_2))
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_prev_point(point_3), point_1)
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_nearest_prev_point(point_3), point_1)
self.assertEqual(sequence.get_next_point(point_1), point_3)
self.assertEqual(sequence.get_next_point(point_2), point_3)
sequence = ISO8601Sequence('PT1H!20000101T00+05', '20000101T00+05')
self.assertEqual(sequence.get_first_point(point_0), point_1)
self.assertEqual(sequence.get_start_point(), point_1)
def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr):
"""The housekeeping method makes sure only satisfied xtrigger function
are kept."""
xtrig = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", xtrig, 'fdir')
xtrig.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(name="foo", rtcfg=None, run_mode="live", start_point=1)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label())
xtrigger_mgr.collate([itask])
# pretend the function has been activated
xtrigger_mgr.active.append(xtrig.get_signature())
xtrigger_mgr.callback(xtrig)
assert xtrigger_mgr.sat_xtrig
xtrigger_mgr.housekeep()
# here we still have the same number as before
assert xtrigger_mgr.sat_xtrig
def test_advanced_exclusions_partial_datetime1(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run 3-hourly but not at 06:00 (from the ICP)
sequence = ISO8601Sequence('PT3H!T06', '20000101T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make ten sequence points
while point and count < 10:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect every T06 to be excluded
self.assertEqual(output, ['20000101T0000Z', '20000101T0300Z',
'20000101T0900Z', '20000101T1200Z',
'20000101T1500Z', '20000101T1800Z',
'20000101T2100Z', '20000102T0000Z',
'20000102T0300Z', '20000102T0900Z'])
def test_exclusions_extensive(self):
"""Test ISO8601Sequence methods for sequences with exclusions"""
init(time_zone='+05')
sequence = ISO8601Sequence('PT1H!20000101T02+05', '20000101T00',
'20000101T05')
point_0 = ISO8601Point('20000101T0000+05')
point_1 = ISO8601Point('20000101T0100+05')
point_2 = ISO8601Point('20000101T0200+05') # The excluded point.
point_3 = ISO8601Point('20000101T0300+05')
self.assertFalse(sequence.is_on_sequence(point_2))
self.assertFalse(sequence.is_valid(point_2))
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_prev_point(point_3), point_1)
self.assertEqual(sequence.get_prev_point(point_2), point_1)
self.assertEqual(sequence.get_nearest_prev_point(point_3), point_1)
self.assertEqual(sequence.get_next_point(point_1), point_3)
self.assertEqual(sequence.get_next_point(point_2), point_3)
sequence = ISO8601Sequence('PT1H!20000101T00+05', '20000101T00+05')
self.assertEqual(sequence.get_first_point(point_0), point_1)
self.assertEqual(sequence.get_start_point(), point_1)
def test_advanced_exclusions_partial_datetime4(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run daily at 00:00 except on Mondays
sequence = ISO8601Sequence('T00!W-1T00', '20170422T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make 19 sequence points
while point and count < 9:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect Monday 24th April and Monday 1st May
# to be excluded.
self.assertEqual(output, [
'20170422T0000Z', '20170423T0000Z', '20170425T0000Z',
'20170426T0000Z', '20170427T0000Z', '20170428T0000Z',
'20170429T0000Z', '20170430T0000Z', '20170502T0000Z'
])
def test_advanced_exclusions_partial_datetime4(self):
"""Advanced exclusions refers to exclusions that are not just
simple points but could be time periods or recurrences such as
'!T06' or similar"""
init(time_zone='Z')
# Run daily at 00:00 except on Mondays
sequence = ISO8601Sequence('T00!W-1T00', '20170422T00Z')
output = []
point = sequence.get_start_point()
count = 0
# We are going to make 19 sequence points
while point and count < 9:
output.append(point)
point = sequence.get_next_point(point)
count += 1
output = [str(out) for out in output]
# We should expect Monday 24th April and Monday 1st May
# to be excluded.
self.assertEqual(output, ['20170422T0000Z', '20170423T0000Z',
'20170425T0000Z', '20170426T0000Z',
'20170427T0000Z', '20170428T0000Z',
'20170429T0000Z', '20170430T0000Z',
'20170502T0000Z'])
def test_process_fcp(scheduling_cfg: dict, options_fcp: Optional[str],
expected_fcp: Optional[str],
expected_err: Optional[Tuple[Type[Exception], str]],
cycling_mode: Fixture):
"""Test SuiteConfig.process_final_cycle_point().
Params:
scheduling_cfg: 'scheduling' section of workflow config.
options_fcp: The fcp set by cli option.
expected_fcp: The expected fcp value that gets set.
expected_err: Exception class expected to be raised plus the message.
"""
if scheduling_cfg['cycling mode'] == loader.ISO8601_CYCLING_TYPE:
iso8601.init(time_zone="+0530")
cycling_mode(integer=False)
else:
cycling_mode(integer=True)
mocked_config = Mock(cycling_type=scheduling_cfg['cycling mode'])
mocked_config.cfg = {
'scheduling': scheduling_cfg
}
mocked_config.initial_point = loader.get_point(
scheduling_cfg['initial cycle point']).standardise()
mocked_config.final_point = None
mocked_config.options.fcp = options_fcp
if expected_err:
err, msg = expected_err
with pytest.raises(err) as exc:
SuiteConfig.process_final_cycle_point(mocked_config)
assert msg in str(exc.value)
else:
SuiteConfig.process_final_cycle_point(mocked_config)
assert mocked_config.cfg[
'scheduling']['final cycle point'] == expected_fcp
assert str(mocked_config.final_point) == str(expected_fcp)
def test_satisfy_xtrigger():
"""Test satisfy_xtriggers"""
# the XtriggerManager instance
xtrigger_mgr = XtriggerManager(
suite="sample_suite",
user="******",
proc_pool=MockedProcPool(),
broadcast_mgr=MockedBroadcastMgr(suite_db_mgr=None))
# the echo1 xtrig (not satisfied)
echo1_xtrig = SubFuncContext(label="echo1",
func_name="echo1",
func_args=[],
func_kwargs={})
echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo1", echo1_xtrig)
# the echo2 xtrig (satisfied through callback later)
echo2_xtrig = SubFuncContext(label="echo2",
func_name="echo2",
func_args=[],
func_kwargs={})
echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo2", echo2_xtrig)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xtrig_labels.add("echo1")
tdef.xtrig_labels.add("echo2")
# cycle point for task proxy
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
# we start with no satisfied xtriggers, and nothing active
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 0
# after calling satisfy_xtriggers the first time, we get two active
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# calling satisfy_xtriggers again does not change anything
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# now we call callback manually as the proc_pool we passed is a mock
# then both should be satisfied
xtrigger_mgr.callback(echo1_xtrig)
xtrigger_mgr.callback(echo2_xtrig)
# so both were satisfied, and nothing is active
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
# calling satisfy_xtriggers again still does not change anything
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
def setUp(self):
init(time_zone='Z')
def test__call_xtriggers_async(xtrigger_mgr):
"""Test _call_xtriggers_async"""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
# the echo1 xtrig (not satisfied)
echo1_xtrig = SubFuncContext(
label="echo1",
func_name="echo1",
func_args=[],
func_kwargs={}
)
echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo1", echo1_xtrig, "fdir")
# the echo2 xtrig (satisfied through callback later)
echo2_xtrig = SubFuncContext(
label="echo2",
func_name="echo2",
func_args=[],
func_kwargs={}
)
echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo2", echo2_xtrig, "fdir")
# create a task
tdef = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1
)
init()
sequence = ISO8601Sequence('P1D', '2000')
tdef.xtrig_labels[sequence] = ["echo1", "echo2"]
# cycle point for task proxy
init()
start_point = ISO8601Point('2019')
# create task proxy
itask = TaskProxy(tdef, start_point)
# we start with no satisfied xtriggers, and nothing active
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 0
# after calling the first time, we get two active
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# calling again does not change anything
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# now we call callback manually as the proc_pool we passed is a mock
# then both should be satisfied
xtrigger_mgr.callback(echo1_xtrig)
xtrigger_mgr.callback(echo2_xtrig)
# so both were satisfied, and nothing is active
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
# calling satisfy_xtriggers again still does not change anything
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
def setUp(self):
init(time_zone='Z')