def test_harvest_upload_undo_sliced(self, mock_subprocess):
mock_subprocess.Popen().wait.side_effect = (
HARVEST_SUCCESS, # zzz_my_test_harvester, event 1, slice 1
HARVEST_SUCCESS, # aaa_my_test_harvester, event 1, slice 1
HARVEST_SUCCESS, # aaa_my_test_harvester, event 2, slice 1
HARVEST_SUCCESS, # mmm_my_test_harvester, event 1, slice 1
HARVEST_FAIL, # failure zzz_my_test_harvester, event 1, slice 2
HARVEST_SUCCESS, # undo zzz_my_test_harvester, event 1, slice 2
HARVEST_SUCCESS, # undo zzz_my_test_harvester, event 1, slice 1
HARVEST_SUCCESS, # undo aaa_my_test_harvester, event 1, slice 1
HARVEST_SUCCESS, # undo aaa_my_test_harvester, event 2, slice 1
HARVEST_SUCCESS) # undo mmm_my_test_harvester, event 1, slice 1
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(with_store=True)
harvester_runner = TalendHarvesterRunner(self.uploader,
{'slice_size': 1},
TESTDATA_DIR, self.config,
self.test_logger)
with self.assertRaises(SystemCommandFailedError):
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_call_count(1)
harvester_runner.storage_broker.assert_delete_call_count(1)
success_slice, fail_slice, pending_slice = collection.get_slices(
harvester_runner.slice_size)
self.assertTrue(all(f.is_harvested for f in success_slice))
self.assertTrue(all(f.is_uploaded for f in success_slice))
self.assertTrue(all(f.is_harvest_undone
for f in success_slice)) # *should* be undone
self.assertTrue(all(f.is_upload_undone
for f in success_slice)) # *should* be undone
self.assertFalse(all(f.is_harvested for f in fail_slice))
self.assertFalse(all(f.is_uploaded for f in fail_slice))
self.assertTrue(all(f.is_harvest_undone
for f in fail_slice)) # *should* be undone
self.assertFalse(all(
f.is_upload_undone
for f in fail_slice)) # should *not* be undone, since never 'done'
self.assertFalse(all(f.is_harvested for f in pending_slice))
self.assertFalse(all(f.is_uploaded for f in pending_slice))
self.assertFalse(
all(f.is_harvest_undone for f in
pending_slice)) # should *not* be undone, since never 'done'
self.assertFalse(
all(f.is_upload_undone for f in
pending_slice)) # should *not* be undone, since never 'done'
def test_harvest_upload_undo_only_current_slice(self, mock_subprocess):
mock_subprocess.Popen().wait.side_effect = (
HARVEST_SUCCESS, # slice 1, zzz_my_test_harvester, event 1
HARVEST_SUCCESS, # slice 1, aaa_my_test_harvester, event 1
HARVEST_SUCCESS, # slice 1, aaa_my_test_harvester, event 2
HARVEST_SUCCESS, # slice 1, mmm_my_test_harvester, event 1
HARVEST_FAIL, # failure slice 2, zzz_my_test_harvester, event 1
HARVEST_SUCCESS) # undo slice 2, zzz_my_test_harvester, event 1
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(with_store=True)
harvester_runner = TalendHarvesterRunner(self.uploader, {
'slice_size': 1,
'undo_previous_slices': False
}, TESTDATA_DIR, self.config, self.test_logger)
with self.assertRaises(SystemCommandFailedError):
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_call_count(1)
# no delete call expected, since fail slice failed during harvesting, and did not call upload
harvester_runner.storage_broker.assert_delete_call_count(0)
success_slice, fail_slice, pending_slice = collection.get_slices(
harvester_runner.slice_size)
self.assertTrue(all(f.is_harvested for f in success_slice))
self.assertTrue(all(f.is_uploaded for f in success_slice))
self.assertFalse(any(
f.is_harvest_undone
for f in success_slice)) # should *not* be undone, due to param
self.assertFalse(any(
f.is_upload_undone
for f in success_slice)) # should *not* be undone, due to param
self.assertFalse(all(f.is_harvested for f in fail_slice))
self.assertFalse(any(f.is_uploaded for f in fail_slice))
self.assertTrue(all(f.is_harvest_undone
for f in fail_slice)) # *should* be undone
self.assertFalse(any(
f.is_upload_undone
for f in fail_slice)) # should *not* be undone, since never 'done'
self.assertFalse(any(f.is_harvested for f in pending_slice))
self.assertFalse(any(f.is_uploaded for f in pending_slice))
self.assertFalse(
any(f.is_harvest_undone for f in
pending_slice)) # should *not* be undone, since never 'done'
self.assertFalse(
any(f.is_upload_undone for f in
pending_slice)) # should *not* be undone, since never 'done'
def test_harvest_only_success(self, mock_subprocess):
mock_subprocess.Popen().wait.return_value = HARVEST_SUCCESS
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection()
harvester_runner = TalendHarvesterRunner(self.uploader, None,
TESTDATA_DIR, self.config,
self.test_logger)
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_not_called()
harvester_runner.storage_broker.assert_delete_not_called()
self.assertTrue(all(f.is_harvested for f in collection))
self.assertFalse(any(f.is_uploaded for f in collection))
def test_harvest_late_deletion_not_run_with_addition_error(
self, mock_subprocess):
mock_subprocess.Popen().wait.side_effect = (
HARVEST_SUCCESS, # slice 1, zzz_my_test_harvester, event 1
HARVEST_SUCCESS, # slice 1, aaa_my_test_harvester, event 1
HARVEST_SUCCESS, # slice 1, aaa_my_test_harvester, event 2
HARVEST_SUCCESS, # slice 1, mmm_my_test_harvester, event 1
HARVEST_FAIL, # failure slice 2, zzz_my_test_harvester, event 1
HARVEST_SUCCESS, # undo slice 1, zzz_my_test_harvester, event 1
HARVEST_SUCCESS, # undo slice 1, zzz_my_test_harvester, event 1
HARVEST_SUCCESS, # undo slice 1, aaa_my_test_harvester, event 1
HARVEST_SUCCESS, # undo slice 1, aaa_my_test_harvester, event 2
HARVEST_SUCCESS) # undo slice 1, mmm_my_test_harvester, event 1
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(with_store=True)
collection[0]._is_deletion = True
collection[0]._late_deletion = False
collection[2]._is_deletion = True
collection[2]._late_deletion = True
harvester_runner = TalendHarvesterRunner(self.uploader,
{'slice_size': 1},
TESTDATA_DIR, self.config,
self.test_logger)
with self.assertRaises(SystemCommandFailedError):
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_call_count(0)
harvester_runner.storage_broker.assert_delete_call_count(1)
# early deletion should have been triggered (i.e. run before additions)
self.assertTrue(
all((collection[0].is_deletion, collection[0].is_harvested,
collection[0].is_stored)))
# addition causes an error
self.assertTrue(
all((not collection[1].is_deletion, not collection[1].is_harvested,
not collection[1].is_stored)))
# late deletion should *not* have been triggered due to addition error
self.assertTrue(
all((collection[2].is_deletion, not collection[2].is_harvested,
not collection[2].is_stored)))
def test_harvest_only_undo(self, mock_subprocess):
mock_subprocess.Popen().wait.side_effect = (1, 0)
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection()
harvester_runner = TalendHarvesterRunner(self.uploader, None,
TESTDATA_DIR, self.config,
self.test_logger)
with self.assertRaises(SystemCommandFailedError):
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_not_called()
harvester_runner.storage_broker.assert_delete_not_called()
self.assertTrue(all(f.is_harvest_undone
for f in collection)) # *should* be undone
def test_harvest_upload_fail(self, mock_subprocess):
mock_subprocess.Popen().wait.return_value = HARVEST_FAIL
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(with_store=True)
harvester_runner = TalendHarvesterRunner(self.uploader, None,
TESTDATA_DIR, self.config,
self.test_logger)
with self.assertRaises(SystemCommandFailedError):
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_not_called()
harvester_runner.storage_broker.assert_delete_not_called()
self.assertFalse(any(f.is_harvested for f in collection))
self.assertFalse(any(f.is_uploaded for f in collection))
def test_harvest_upload_deletion(self, mock_subprocess):
mock_subprocess.Popen().wait.return_value = HARVEST_SUCCESS
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(delete=True, with_store=True)
harvester_runner = TalendHarvesterRunner(self.uploader, None,
TESTDATA_DIR, self.config,
self.test_logger)
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_call_count(0)
harvester_runner.storage_broker.assert_delete_call_count(1)
self.assertTrue(all(f.is_deletion for f in collection))
self.assertTrue(all(f.is_harvested for f in collection))
self.assertTrue(all(f.is_deleted for f in collection))
def test_harvest_upload_success_sliced(self, mock_subprocess):
mock_subprocess.Popen().wait.return_value = HARVEST_SUCCESS
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection(with_store=True)
harvester_runner = TalendHarvesterRunner(self.uploader,
{'slice_size': 1},
TESTDATA_DIR, self.config,
self.test_logger)
harvester_runner.run(collection)
harvester_runner.storage_broker.assert_upload_call_count(3)
harvester_runner.storage_broker.assert_delete_call_count(0)
self.assertTrue(all(f.is_harvested for f in collection))
self.assertTrue(all(f.is_uploaded for f in collection))
self.assertFalse(any(f.is_harvest_undone for f in collection))
self.assertFalse(any(f.is_upload_undone for f in collection))
def test_extra_params(self, mock_subprocess):
mock_subprocess.Popen().wait.return_value = HARVEST_SUCCESS
mock_subprocess.Popen().communicate.return_value = ('mocked stdout',
'mocked stderr')
collection = get_harvest_collection()
harvester_runner = TalendHarvesterRunner(self.uploader, None,
TESTDATA_DIR, self.config,
self.test_logger)
harvester_runner.run(collection)
expected_extra_params = "--collection my_test_collection"
self.assertEqual(
expected_extra_params,
harvester_runner.harvested_file_map.map['aaa_my_test_harvester']
[0].extra_params)
called_commands = [
c[1][0] for c in mock_subprocess.Popen.mock_calls if c[1]
]
self.assertTrue(
called_commands[0].startswith('echo zzz_my_test_harvester '))
self.assertFalse(called_commands[0].endswith(expected_extra_params))
self.assertTrue(
called_commands[1].startswith('echo aaa_my_test_harvester '))
self.assertTrue(called_commands[1].endswith(expected_extra_params))
self.assertTrue(
called_commands[2].startswith('echo aaa_my_test_harvester '))
self.assertFalse(called_commands[2].endswith(expected_extra_params))
self.assertTrue(
called_commands[3].startswith('echo mmm_my_test_harvester '))
self.assertFalse(called_commands[3].endswith(expected_extra_params))