def get_valid_and_stale_uow():
return create_unit_of_work(PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_REQUESTED,
creation_at=datetime.utcnow() - timedelta(hours=LIFE_SUPPORT_HOURS),
uow_id=0)
def get_valid_and_fresh_uow():
return create_unit_of_work(PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_IN_PROGRESS,
creation_at=datetime.utcnow(),
uow_id=0)
def test_non_finalizable_job(self):
""" coverage scope: valid timeperiod with non_finalizable job => call non_finalizable method"""
for timeperiod in [TEST_PAST_TIMEPERIOD, TEST_ACTUAL_TIMEPERIOD]:
job_record = get_job_record(job.STATE_IN_PROGRESS, timeperiod,
PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1,
timeperiod)
self.uow_dao_mocked.get_one = mock.MagicMock(
return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(
return_value=False)
self.sm_real.manage_job(job_record)
self.sm_real._process_state_in_progress.assert_called_once_with(
mock.ANY)
self.sm_real._StateMachineDiscrete__process_non_finalizable_job.assert_called_once_with(
mock.ANY, mock.ANY)
self.assertTrue(
self.sm_real._StateMachineDiscrete__process_finalizable_job.
call_args_list == [])
# reset mock call count
self.sm_real._process_state_in_progress.reset_mock()
self.sm_real._StateMachineDiscrete__process_non_finalizable_job.reset_mock(
)
def get_invalid_and_stale_uow():
return create_unit_of_work(PROCESS_UNIT_TEST,
0,
1,
state=unit_of_work.STATE_INVALID,
creation_at=datetime.utcnow() - timedelta(hours=LIFE_SUPPORT_HOURS),
uow_id=0)
def get_invalid_and_fresh_uow():
return create_unit_of_work(PROCESS_UNIT_TEST,
0,
1,
state=unit_of_work.STATE_INVALID,
creation_at=datetime.utcnow(),
uow_id=0)
def create_test_uow():
return create_unit_of_work(PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_IN_PROGRESS,
created_at=datetime.utcnow(),
uow_id=0)
def get_valid_and_fresh_uow():
return create_unit_of_work(PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_IN_PROGRESS,
created_at=datetime.utcnow(),
submitted_at=datetime.utcnow(),
uow_id=0)
def setUp(self):
list_of_stats = self.virtual_set_up()
self.aggregator = get_test_aggregator(self.baseclass, self.process_name)
# creating unit_of_work entity, requesting to process created above statistics
self.uow_id = base_fixtures.create_unit_of_work(self.process_name,
str(list_of_stats[0]),
str(list_of_stats[-1]))
def mock_insert_uow_return_uow(process_name, timeperiod, start_timeperiod,
end_timeperiod, start_id, end_id):
"""mocks AbstractStateMachine._insert_uow"""
return create_unit_of_work(process_name,
start_id,
end_id,
timeperiod,
uow_id='a_uow_id')
def get_invalid_and_fresh_uow():
return create_unit_of_work(
PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_INVALID,
created_at=datetime.utcnow(),
submitted_at=datetime.utcnow(),
uow_id=0)
def get_valid_and_stale_uow():
return create_unit_of_work(
PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_REQUESTED,
created_at=datetime.utcnow() - timedelta(hours=settings.settings['gc_life_support_hours']),
submitted_at=datetime.utcnow() - timedelta(hours=settings.settings['gc_resubmit_after_hours']),
uow_id=0)
def test_transfer_to_final_timeperiod_state_in_progress(self):
""" method tests job records in STATE_IN_PROGRESS state"""
when(self.time_table_mocked).can_finalize_job_record(any(str), any(Job)).thenReturn(True)
uow_dao_mock = mock(UnitOfWorkDao)
when(uow_dao_mock).get_one(any()). \
thenReturn(create_unit_of_work(PROCESS_UNIT_TEST, 1, 1, None, unit_of_work.STATE_REQUESTED)). \
thenReturn(create_unit_of_work(PROCESS_UNIT_TEST, 1, 1, None, unit_of_work.STATE_PROCESSED))
self.pipeline_real.uow_dao = uow_dao_mock
self.pipeline_real.compute_scope_of_processing = then_raise
pipeline = spy(self.pipeline_real)
job_record = get_job_record(job.STATE_IN_PROGRESS,
TEST_PRESET_TIMEPERIOD,
PROCESS_SITE_HOURLY)
pipeline.manage_pipeline_for_process(job_record.process_name, job_record)
verify(self.time_table_mocked, times=1). \
update_job_record(any(str), any(Job), any(UnitOfWork), any(str))
def test_cancelled_state_final_run(self):
"""method tests job records in STATE_FINAL_RUN state"""
self.uow_dao_mocked.get_one = mock.MagicMock(
side_effect=lambda *_: create_unit_of_work(PROCESS_SITE_HOURLY, 1, 1, None, unit_of_work.STATE_CANCELED))
job_record = get_job_record(job.STATE_FINAL_RUN, TEST_PRESET_TIMEPERIOD, PROCESS_SITE_HOURLY)
self.sm_real.manage_job(job_record)
self.assertEqual(len(self.time_table_mocked.update_job_record.call_args_list), 1)
self.assertEqual(len(self.time_table_mocked.get_tree.call_args_list), 1)
def test_cancelled_state_final_run(self):
"""method tests job records in STATE_FINAL_RUN state"""
self.uow_dao_mocked.get_one = mock.MagicMock(
side_effect=lambda *_: create_unit_of_work(
PROCESS_SITE_HOURLY, 1, 1, TEST_ACTUAL_TIMEPERIOD, unit_of_work.STATE_CANCELED))
job_record = get_job_record(job.STATE_FINAL_RUN, TEST_PRESET_TIMEPERIOD, PROCESS_SITE_HOURLY)
self.sm_real.manage_job(job_record)
self.assertEqual(len(self.sm_real.update_job.call_args_list), 1)
self.assertEqual(len(self.time_table_mocked.get_tree.call_args_list), 1)
def get_valid_and_stale_uow():
return create_unit_of_work(
PROCESS_SITE_HOURLY,
0,
1,
state=unit_of_work.STATE_REQUESTED,
created_at=datetime.utcnow() -
timedelta(hours=settings.settings['gc_life_support_hours']),
submitted_at=datetime.utcnow() -
timedelta(hours=settings.settings['gc_resubmit_after_hours']),
uow_id=0)
def test_future_timeperiod_state_in_progress(self):
""" method tests job records in STATE_IN_PROGRESS state"""
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_FUTURE_TIMEPERIOD, PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1, TEST_ACTUAL_TIMEPERIOD)
self.uow_dao_mocked.get_one = mock.MagicMock(return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=True)
self.sm_real.insert_and_publish_uow = then_return_duplicate_uow
self.sm_real.manage_job(job_record)
self.assertEqual(len(self.sm_real.update_job.call_args_list), 0)
def test_finalizable_job(self):
""" coverage scope: past timeperiod + finalizable job => call on_finalizable method"""
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_PAST_TIMEPERIOD, PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1, TEST_ACTUAL_TIMEPERIOD)
self.uow_dao_mocked.get_one = mock.MagicMock(return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=True)
self.sm_real.manage_job(job_record)
self.sm_real._process_state_in_progress.assert_called_once_with(mock.ANY)
self.assertTrue(self.sm_real._StateMachineDiscrete__process_non_finalizable_job.call_args_list == [])
self.sm_real._StateMachineDiscrete__process_finalizable_job.assert_called_once_with(mock.ANY, mock.ANY)
def test_preset_timeperiod_state_in_progress(self):
""" method tests timetable records in STATE_IN_PROGRESS state"""
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=True)
self.uow_dao_mocked.get_one = mock.MagicMock(
side_effect=lambda *_: create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1, TEST_ACTUAL_TIMEPERIOD))
self.sm_real.insert_and_publish_uow = then_return_uow
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_PRESET_TIMEPERIOD, PROCESS_SITE_HOURLY)
self.sm_real.manage_job(job_record)
self.assertEqual(len(self.sm_real.update_job.call_args_list), 0)
self.assertEqual(len(self.sm_real._process_state_final_run.call_args_list), 0)
def test_future_timeperiod(self):
""" coverage scope: make sure no actual processing methods are called for the Future timeperiod"""
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_FUTURE_TIMEPERIOD, PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1, TEST_ACTUAL_TIMEPERIOD)
self.uow_dao_mocked.get_one = mock.MagicMock(return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=True)
self.sm_real.manage_job(job_record)
self.sm_real._process_state_in_progress.assert_called_once_with(mock.ANY)
# verify that processing functions were not called
self.assertTrue(self.sm_real._StateMachineDiscrete__process_finalizable_job.call_args_list == [])
self.assertTrue(self.sm_real._StateMachineDiscrete__process_non_finalizable_job.call_args_list == [])
def test_future_timeperiod_state_in_progress(self):
""" method tests timetable records in STATE_IN_PROGRESS state"""
when(self.time_table_mocked).can_finalize_job_record(any(str), any(Job)).thenReturn(True)
uow_dao_mock = mock(UnitOfWorkDao)
when(uow_dao_mock).get_one(any()).thenReturn(create_unit_of_work(PROCESS_UNIT_TEST, 0, 1, None))
self.pipeline_real.uow_dao = uow_dao_mock
self.pipeline_real.insert_uow = then_raise
pipeline = spy(self.pipeline_real)
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_FUTURE_TIMEPERIOD, PROCESS_SITE_HOURLY)
pipeline.manage_pipeline_for_process(job_record.process_name, job_record)
verify(self.time_table_mocked, times=0). \
update_job_record(any(str), any(Job), any(UnitOfWork), any(str))
def test_preset_timeperiod_state_in_progress(self):
""" method tests job records in STATE_IN_PROGRESS state"""
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=True)
returns = [
create_unit_of_work(PROCESS_SITE_HOURLY, 1, 1, TEST_ACTUAL_TIMEPERIOD, unit_of_work.STATE_REQUESTED),
create_unit_of_work(PROCESS_SITE_HOURLY, 1, 1, TEST_ACTUAL_TIMEPERIOD, unit_of_work.STATE_PROCESSED)
]
def side_effects(*args):
return returns.pop(0)
self.uow_dao_mocked.get_one = mock.MagicMock(side_effect=side_effects)
self.ds_mocked.highest_primary_key = mock.MagicMock(return_value=1)
self.ds_mocked.lowest_primary_key = mock.MagicMock(return_value=0)
self.sm_real.insert_and_publish_uow = then_return_uow
job_record = get_job_record(job.STATE_IN_PROGRESS, TEST_PRESET_TIMEPERIOD, PROCESS_SITE_HOURLY)
self.sm_real.manage_job(job_record)
self.assertEqual(len(self.sm_real.update_job.call_args_list), 1)
self.assertEqual(len(self.sm_real._compute_and_transfer_to_final_run.call_args_list), 1)
self.assertEqual(len(self.sm_real._process_state_final_run.call_args_list), 0)
def test_cancelled_state_final_run(self):
"""method tests job records in STATE_FINAL_RUN state"""
uow_dao_mock = mock(UnitOfWorkDao)
when(uow_dao_mock).get_one(any()).thenReturn(
create_unit_of_work(PROCESS_UNIT_TEST, 1, 1, None, unit_of_work.STATE_CANCELED))
self.pipeline_real.uow_dao = uow_dao_mock
pipeline = spy(self.pipeline_real)
job_record = get_job_record(job.STATE_FINAL_RUN,
TEST_PRESET_TIMEPERIOD,
PROCESS_SITE_HOURLY)
pipeline.manage_pipeline_for_process(job_record.process_name, job_record)
verify(self.time_table_mocked, times=1). \
update_job_record(any(str), any(Job), any(UnitOfWork), any(str))
verify(self.time_table_mocked, times=0).get_tree(any(str))
def test_non_finalizable_job(self):
""" coverage scope: valid timeperiod with non_finalizable job => call non_finalizable method"""
for timeperiod in [TEST_PAST_TIMEPERIOD, TEST_ACTUAL_TIMEPERIOD]:
job_record = get_job_record(job.STATE_IN_PROGRESS, timeperiod, PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1, None)
self.uow_dao_mocked.get_one = mock.MagicMock(return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(return_value=False)
self.sm_real.manage_job(job_record)
self.sm_real._process_state_in_progress.assert_called_once_with(mock.ANY)
self.sm_real._StateMachineSimpleDiscrete__process_non_finalizable_job.\
assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, mock.ANY)
self.assertTrue(self.sm_real._StateMachineSimpleDiscrete__process_finalizable_job.call_args_list == [])
# reset mock call count
self.sm_real._process_state_in_progress.reset_mock()
self.sm_real._StateMachineSimpleDiscrete__process_non_finalizable_job.reset_mock()
def test_retry_state_in_progress(self):
""" method tests timetable records in STATE_IN_PROGRESS state"""
when(self.time_table_mocked).can_finalize_job_record(any(str), any(Job)).thenReturn(True)
uow_dao_mock = mock(UnitOfWorkDao)
when(uow_dao_mock).get_one(any()).thenReturn(
create_unit_of_work(PROCESS_UNIT_TEST, 1, 1, None, unit_of_work.STATE_PROCESSED))
self.pipeline_real.uow_dao = uow_dao_mock
self.pipeline_real.insert_uow = then_raise
self.pipeline_real.recover_from_duplicatekeyerror = override_recover_function
pipeline = spy(self.pipeline_real)
job_record = get_job_record(job.STATE_IN_PROGRESS,
TEST_PRESET_TIMEPERIOD,
PROCESS_SITE_HOURLY)
pipeline.manage_pipeline_for_process(job_record.process_name, job_record)
verify(self.time_table_mocked, times=1). \
update_job_record(any(str), any(Job), any(UnitOfWork), any(str))
def test_future_timeperiod(self):
""" coverage scope: make sure no actual processing methods are called for the Future timeperiod"""
job_record = get_job_record(job.STATE_IN_PROGRESS,
TEST_FUTURE_TIMEPERIOD,
PROCESS_SITE_HOURLY)
manual_uow = create_unit_of_work(PROCESS_SITE_HOURLY, 0, 1,
TEST_ACTUAL_TIMEPERIOD)
self.uow_dao_mocked.get_one = mock.MagicMock(return_value=manual_uow)
self.time_table_mocked.is_job_record_finalizable = mock.MagicMock(
return_value=True)
self.sm_real.manage_job(job_record)
self.sm_real._process_state_in_progress.assert_called_once_with(
mock.ANY)
# verify that processing functions were not called
self.assertTrue(
self.sm_real._StateMachineDiscrete__process_finalizable_job.
call_args_list == [])
self.assertTrue(
self.sm_real._StateMachineDiscrete__process_non_finalizable_job.
call_args_list == [])
def mock_insert_uow_return_uow(process_name, timeperiod, start_timeperiod, end_timeperiod, start_id, end_id):
"""mocks AbstractStateMachine._insert_uow"""
return create_unit_of_work(process_name, start_id, end_id, timeperiod, uow_id='a_uow_id')
def override_recover_function(e):
return create_unit_of_work(PROCESS_UNIT_TEST, 0, 1, None)
def then_return_duplicate_uow(job_record, start_id, end_id):
"""mocks AbstractStateMachine.insert_and_publish_uow"""
return create_unit_of_work(job_record.process_name, start_id, end_id, job_record.timeperiod, uow_id='a_uow_id'), \
True
def then_return_uow(process_name, start_timeperiod, end_timeperiod, job_record):
return create_unit_of_work(PROCESS_UNIT_TEST, 0, 1, None)
def then_return_duplicate_uow(job_record, start_id, end_id):
"""mocks AbstractStateMachine.insert_and_publish_uow"""
return create_unit_of_work(job_record.process_name, start_id, end_id, job_record.timeperiod, uow_id='a_uow_id'), \
True