def test_worker_process_execute_job(self, datetime_mock, upload_part_mock, complete_multipart_upload_mock, open_mock, unlink_mock): # Unknown job type, no boto functions are being called and # an exception is being raised interface = CloudInterface(url='s3://bucket/path/to/dir', encryption=None) interface.result_queue = Queue() interface.done_queue = Queue() with pytest.raises(ValueError): interface.worker_process_execute_job({"job_type": "error"}, 1) assert upload_part_mock.call_count == 0 assert complete_multipart_upload_mock.call_count == 0 assert interface.result_queue.empty() # upload_part job, a file with the passed name is opened, uploaded # and them deleted part_result = { "ETag": "89d4f0341d9091aa21ddf67d3b32c34a", "PartNumber": "10" } upload_part_mock.return_value = part_result interface.worker_process_execute_job( { "job_type": "upload_part", "mpu": "mpu", "part_number": 10, "key": "this/key", "body": "body" }, 0) upload_part_mock.assert_called_once_with( "mpu", "this/key", open_mock.return_value.__enter__.return_value, 10) assert not interface.result_queue.empty() assert interface.result_queue.get() == { "end_time": datetime_mock.now.return_value, "key": "this/key", "part": part_result, "part_number": 10, } assert unlink_mock.call_count == 1 # complete_multipart_upload, an S3 call to create a key in the bucket # with the right parts is called interface.worker_process_execute_job( { "job_type": "complete_multipart_upload", "mpu": "mpu", "key": "this/key", "parts": ["parts", "list"] }, 0) complete_multipart_upload_mock.assert_called_once_with( "mpu", "this/key", ["parts", "list"]) assert not interface.done_queue.empty() assert interface.done_queue.get() == { "end_time": datetime_mock.now.return_value, "key": "this/key", "status": "done", }
def test_retrieve_results(self): interface = CloudInterface(url='s3://bucket/path/to/dir', encryption=None) interface.queue = Queue() interface.done_queue = Queue() interface.result_queue = Queue() interface.errors_queue = Queue() # With an empty queue, the parts DB is empty interface._retrieve_results() assert len(interface.parts_db) == 0 # Preset the upload statistics, to avoid a random start_date for name in ["test/file", "test/another_file"]: interface.upload_stats[name] = FileUploadStatistics( status='uploading', start_time=datetime.datetime(2016, 3, 30, 17, 1, 0), ) # Fill the result queue with mock results, and assert that after # the refresh the result queue is empty and the parts_db full with # ordered results interface.result_queue.put({ "key": "test/file", "part_number": 2, "end_time": datetime.datetime(2016, 3, 30, 17, 2, 20), "part": { "ETag": "becb2f30c11b6a2b5c069f3c8a5b798c", "PartNumber": "2" } }) interface.result_queue.put({ "key": "test/file", "part_number": 1, "end_time": datetime.datetime(2016, 3, 30, 17, 1, 20), "part": { "ETag": "27960aa8b7b851eb0277f0f3f5d15d68", "PartNumber": "1" } }) interface.result_queue.put({ "key": "test/file", "part_number": 3, "end_time": datetime.datetime(2016, 3, 30, 17, 3, 20), "part": { "ETag": "724a0685c99b457d4ddd93814c2d3e2b", "PartNumber": "3" } }) interface.result_queue.put({ "key": "test/another_file", "part_number": 1, "end_time": datetime.datetime(2016, 3, 30, 17, 5, 20), "part": { "ETag": "89d4f0341d9091aa21ddf67d3b32c34a", "PartNumber": "1" } }) interface._retrieve_results() assert interface.result_queue.empty() assert interface.parts_db == { "test/file": [{ "ETag": "27960aa8b7b851eb0277f0f3f5d15d68", "PartNumber": "1" }, { "ETag": "becb2f30c11b6a2b5c069f3c8a5b798c", "PartNumber": "2" }, { "ETag": "724a0685c99b457d4ddd93814c2d3e2b", "PartNumber": "3" }], "test/another_file": [{ "ETag": "89d4f0341d9091aa21ddf67d3b32c34a", "PartNumber": "1" }] } assert interface.upload_stats == { 'test/another_file': { 'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0), 'status': 'uploading', 'parts': { 1: { 'end_time': datetime.datetime(2016, 3, 30, 17, 5, 20), 'part_number': 1, }, }, }, 'test/file': { 'start_time': datetime.datetime(2016, 3, 30, 17, 1, 0), 'status': 'uploading', 'parts': { 1: { 'end_time': datetime.datetime(2016, 3, 30, 17, 1, 20), 'part_number': 1, }, 2: { 'end_time': datetime.datetime(2016, 3, 30, 17, 2, 20), 'part_number': 2, }, 3: { 'end_time': datetime.datetime(2016, 3, 30, 17, 3, 20), 'part_number': 3, }, }, }, }