def test_handle_async_errors(self):
# If we the upload process has already raised an error, we immediately
# exit without doing anything
interface = CloudInterface(url='s3://bucket/path/to/dir',
encryption=None)
interface.error = "test"
interface.errors_queue = None # If get called raises AttributeError
interface._handle_async_errors()
# There is no error and the process haven't already errored out
interface.error = None
interface.errors_queue = Queue()
interface._handle_async_errors()
assert interface.error is None
# There is an error in the queue
interface.error = None
interface.errors_queue.put("Test error")
with pytest.raises(CloudUploadingError):
interface._handle_async_errors()
def test_worker_process_main(self, worker_process_execute_job_mock):
job_collection = [
{
"job_id": 1,
"job_type": "upload_part"
},
{
"job_id": 2,
"job_type": "upload_part"
},
{
"job_id": 3,
"job_type": "upload_part"
},
None,
]
interface = CloudInterface(url='s3://bucket/path/to/dir',
encryption=None)
interface.queue = mock.MagicMock()
interface.errors_queue = Queue()
interface.queue.get.side_effect = job_collection
interface.worker_process_main(0)
# Jobs are been grabbed from queue, and the queue itself has been
# notified of tasks being done
assert interface.queue.get.call_count == 4
# worker_process_execute_job is executed only 3 times, because it's
# not called for the process stop marker
assert worker_process_execute_job_mock.call_count == 3
assert interface.queue.task_done.call_count == 4
assert interface.errors_queue.empty()
# If during an execution a job an exception is raised, the worker
# process must put the error in the appropriate queue.
def execute_mock(job, process_number):
if job["job_id"] == 2:
raise Boto3Error("Something is gone wrong")
interface.queue.reset_mock()
worker_process_execute_job_mock.reset_mock()
worker_process_execute_job_mock.side_effect = execute_mock
interface.queue.get.side_effect = job_collection
interface.worker_process_main(0)
assert interface.queue.get.call_count == 4
# worker_process_execute_job is executed only 3 times, because it's
# not called for the process stop marker
assert worker_process_execute_job_mock.call_count == 3
assert interface.queue.task_done.call_count == 4
assert interface.errors_queue.get() == "Something is gone wrong"
assert interface.errors_queue.empty()
def test_retrieve_results(self):
interface = CloudInterface(url='s3://bucket/path/to/dir',
encryption=None)
interface.queue = Queue()
interface.done_queue = Queue()
interface.result_queue = Queue()
interface.errors_queue = Queue()
# With an empty queue, the parts DB is empty
interface._retrieve_results()
assert len(interface.parts_db) == 0
# Preset the upload statistics, to avoid a random start_date
for name in ["test/file", "test/another_file"]:
interface.upload_stats[name] = FileUploadStatistics(
status='uploading',
start_time=datetime.datetime(2016, 3, 30, 17, 1, 0),
)
# Fill the result queue with mock results, and assert that after
# the refresh the result queue is empty and the parts_db full with
# ordered results
interface.result_queue.put({
"key":
"test/file",
"part_number":
2,
"end_time":
datetime.datetime(2016, 3, 30, 17, 2, 20),
"part": {
"ETag": "becb2f30c11b6a2b5c069f3c8a5b798c",
"PartNumber": "2"
}
})
interface.result_queue.put({
"key":
"test/file",
"part_number":
1,
"end_time":
datetime.datetime(2016, 3, 30, 17, 1, 20),
"part": {
"ETag": "27960aa8b7b851eb0277f0f3f5d15d68",
"PartNumber": "1"
}
})
interface.result_queue.put({
"key":
"test/file",
"part_number":
3,
"end_time":
datetime.datetime(2016, 3, 30, 17, 3, 20),
"part": {
"ETag": "724a0685c99b457d4ddd93814c2d3e2b",
"PartNumber": "3"
}
})
interface.result_queue.put({
"key":
"test/another_file",
"part_number":
1,
"end_time":
datetime.datetime(2016, 3, 30, 17, 5, 20),
"part": {
"ETag": "89d4f0341d9091aa21ddf67d3b32c34a",
"PartNumber": "1"
}
})
interface._retrieve_results()
assert interface.result_queue.empty()
assert interface.parts_db == {
"test/file": [{
"ETag": "27960aa8b7b851eb0277f0f3f5d15d68",
"PartNumber": "1"
}, {
"ETag": "becb2f30c11b6a2b5c069f3c8a5b798c",
"PartNumber": "2"
}, {
"ETag": "724a0685c99b457d4ddd93814c2d3e2b",
"PartNumber": "3"
}],
"test/another_file": [{
"ETag": "89d4f0341d9091aa21ddf67d3b32c34a",
"PartNumber": "1"
}]
}
assert interface.upload_stats == {
'test/another_file': {
'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0),
'status': 'uploading',
'parts': {
1: {
'end_time': datetime.datetime(2016, 3, 30, 17, 5, 20),
'part_number': 1,
},
},
},
'test/file': {
'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0),
'status': 'uploading',
'parts': {
1: {
'end_time': datetime.datetime(2016, 3, 30, 17, 1, 20),
'part_number': 1,
},
2: {
'end_time': datetime.datetime(2016, 3, 30, 17, 2, 20),
'part_number': 2,
},
3: {
'end_time': datetime.datetime(2016, 3, 30, 17, 3, 20),
'part_number': 3,
},
},
},
}