def test_worker_process_execute_job(self, datetime_mock, upload_part_mock,
complete_multipart_upload_mock,
open_mock, unlink_mock):
# Unknown job type, no boto functions are being called and
# an exception is being raised
interface = CloudInterface(url='s3://bucket/path/to/dir',
encryption=None)
interface.result_queue = Queue()
interface.done_queue = Queue()
with pytest.raises(ValueError):
interface.worker_process_execute_job({"job_type": "error"}, 1)
assert upload_part_mock.call_count == 0
assert complete_multipart_upload_mock.call_count == 0
assert interface.result_queue.empty()
# upload_part job, a file with the passed name is opened, uploaded
# and them deleted
part_result = {
"ETag": "89d4f0341d9091aa21ddf67d3b32c34a",
"PartNumber": "10"
}
upload_part_mock.return_value = part_result
interface.worker_process_execute_job(
{
"job_type": "upload_part",
"mpu": "mpu",
"part_number": 10,
"key": "this/key",
"body": "body"
}, 0)
upload_part_mock.assert_called_once_with(
"mpu", "this/key", open_mock.return_value.__enter__.return_value,
10)
assert not interface.result_queue.empty()
assert interface.result_queue.get() == {
"end_time": datetime_mock.now.return_value,
"key": "this/key",
"part": part_result,
"part_number": 10,
}
assert unlink_mock.call_count == 1
# complete_multipart_upload, an S3 call to create a key in the bucket
# with the right parts is called
interface.worker_process_execute_job(
{
"job_type": "complete_multipart_upload",
"mpu": "mpu",
"key": "this/key",
"parts": ["parts", "list"]
}, 0)
complete_multipart_upload_mock.assert_called_once_with(
"mpu", "this/key", ["parts", "list"])
assert not interface.done_queue.empty()
assert interface.done_queue.get() == {
"end_time": datetime_mock.now.return_value,
"key": "this/key",
"status": "done",
}
def test_retrieve_results(self):
interface = CloudInterface(url='s3://bucket/path/to/dir',
encryption=None)
interface.queue = Queue()
interface.done_queue = Queue()
interface.result_queue = Queue()
interface.errors_queue = Queue()
# With an empty queue, the parts DB is empty
interface._retrieve_results()
assert len(interface.parts_db) == 0
# Preset the upload statistics, to avoid a random start_date
for name in ["test/file", "test/another_file"]:
interface.upload_stats[name] = FileUploadStatistics(
status='uploading',
start_time=datetime.datetime(2016, 3, 30, 17, 1, 0),
)
# Fill the result queue with mock results, and assert that after
# the refresh the result queue is empty and the parts_db full with
# ordered results
interface.result_queue.put({
"key":
"test/file",
"part_number":
2,
"end_time":
datetime.datetime(2016, 3, 30, 17, 2, 20),
"part": {
"ETag": "becb2f30c11b6a2b5c069f3c8a5b798c",
"PartNumber": "2"
}
})
interface.result_queue.put({
"key":
"test/file",
"part_number":
1,
"end_time":
datetime.datetime(2016, 3, 30, 17, 1, 20),
"part": {
"ETag": "27960aa8b7b851eb0277f0f3f5d15d68",
"PartNumber": "1"
}
})
interface.result_queue.put({
"key":
"test/file",
"part_number":
3,
"end_time":
datetime.datetime(2016, 3, 30, 17, 3, 20),
"part": {
"ETag": "724a0685c99b457d4ddd93814c2d3e2b",
"PartNumber": "3"
}
})
interface.result_queue.put({
"key":
"test/another_file",
"part_number":
1,
"end_time":
datetime.datetime(2016, 3, 30, 17, 5, 20),
"part": {
"ETag": "89d4f0341d9091aa21ddf67d3b32c34a",
"PartNumber": "1"
}
})
interface._retrieve_results()
assert interface.result_queue.empty()
assert interface.parts_db == {
"test/file": [{
"ETag": "27960aa8b7b851eb0277f0f3f5d15d68",
"PartNumber": "1"
}, {
"ETag": "becb2f30c11b6a2b5c069f3c8a5b798c",
"PartNumber": "2"
}, {
"ETag": "724a0685c99b457d4ddd93814c2d3e2b",
"PartNumber": "3"
}],
"test/another_file": [{
"ETag": "89d4f0341d9091aa21ddf67d3b32c34a",
"PartNumber": "1"
}]
}
assert interface.upload_stats == {
'test/another_file': {
'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0),
'status': 'uploading',
'parts': {
1: {
'end_time': datetime.datetime(2016, 3, 30, 17, 5, 20),
'part_number': 1,
},
},
},
'test/file': {
'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0),
'status': 'uploading',
'parts': {
1: {
'end_time': datetime.datetime(2016, 3, 30, 17, 1, 20),
'part_number': 1,
},
2: {
'end_time': datetime.datetime(2016, 3, 30, 17, 2, 20),
'part_number': 2,
},
3: {
'end_time': datetime.datetime(2016, 3, 30, 17, 3, 20),
'part_number': 3,
},
},
},
}
