def test_job_generator(self):
# the simplest case (str str)
obj = cloudfs.FileTransfer(src=self.path1, dst=self.path2)
res = [job for job in obj._job_generator()]
assert res == [(self.path1, self.path2, 0, -1)], res
# iter str multipart
src_gen = self.make_generator(self.path0, self.path1, self.path0)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2, multipart=True)
res = [job for job in obj._job_generator()]
assert res == [
(self.path0, self.path2, 0, 0),
(self.path1, self.path2, 0, 1),
(self.path0, self.path2, 0, 2),
], res
# iter str not multipart
src_gen = self.make_generator(self.path0, self.path1)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2)
res = [job for job in obj._job_generator()]
assert res == [
(self.path0, self.path2, 0, -1),
(self.path1, self.path2, 0, -1),
], res
# iter iter
src_gen = self.make_generator(self.path0, self.path1, self.path2)
dst_list = [self.path2, self.path2, self.path1]
obj = cloudfs.FileTransfer(src=src_gen, dst=dst_list)
res = [job for job in obj._job_generator()]
assert res == [
(self.path0, self.path2, 0, -1),
(self.path1, self.path2, 0, -1),
(self.path2, self.path1, 0, -1),
], res
def test_job_generator_with_retries(self):
# str str, first retry fail
obj = cloudfs.FileTransfer(src=self.path1, dst=self.path2)
res = []
generator = obj._job_generator()
src, dst, retry, chunk_num = generator.next()
res.append((src, dst, retry, chunk_num))
obj._retries_queue.put((src, dst, retry + 1, chunk_num))
res.append(generator.next())
assert_raises(StopIteration, generator.next)
assert res == [
(self.path1, self.path2, 0, -1),
(self.path1, self.path2, 1, -1),
], res
# iter str multipart
src_gen = self.make_generator(self.path0, self.path1, self.path0)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2, multipart=True)
res = []
generator = obj._job_generator()
res.append(generator.next())
src, dst, retry, chunk_num = generator.next()
res.append((src, dst, retry, chunk_num))
obj._retries_queue.put((src, dst, retry + 1, chunk_num))
res.append(generator.next())
res.append(generator.next())
assert_raises(StopIteration, generator.next)
assert res == [
(self.path0, self.path2, 0, 0),
(self.path1, self.path2, 0, 1),
(self.path1, self.path2, 1, 1),
(self.path0, self.path2, 0, 2),
], res
def test_worker_assertion_error(self, getsize, isfile):
# both remote
isfile.return_value = False
obj = cloudfs.FileTransfer(src=self.path2, dst=self.path2)
obj.fire = mock.MagicMock()
ret = obj.run()
assert ('transfer_error', self.path2, self.path2, 0, -1) in \
[list(x)[0][:5] for x in obj.fire.call_args_list]
assert not ret["completed"]
assert not ret["failed"]
# both local
isfile.reset_mock()
obj = cloudfs.FileTransfer(src=self.path1, dst=self.path1)
obj.fire = mock.MagicMock()
ret = obj.run()
assert ('transfer_error', self.path1, self.path1, 0, -1) in\
[list(x)[0][:5] for x in obj.fire.call_args_list]
assert not ret["completed"]
assert not ret["failed"]
# uploading non-existent file
isfile.return_value = False
obj = cloudfs.FileTransfer(src=self.path1, dst=self.path2)
obj.fire = mock.MagicMock()
ret = obj.run()
assert ('transfer_error', self.path1, self.path2, 0, -1) in\
[list(x)[0][:5] for x in obj.fire.call_args_list]
assert not ret["completed"]
assert not ret["failed"]
def test_worker_upload_multipart(self, getsize, isfile):
src_gen = self.make_generator(self.path0, self.path1, self.path0)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2, multipart=True)
ret = obj.run()
driver = cloudfs.cloudfs.return_value
upload_id = driver.multipart_init.return_value
assert all([
call == mock.call("s3") for call in cloudfs.cloudfs.call_args_list
])
driver.multipart_init.assert_called_once_with(self.path2,
getsize.return_value)
driver.multipart_put.assert_any_call(upload_id, 0, self.path0)
driver.multipart_put.assert_any_call(upload_id, 1, self.path1)
driver.multipart_put.assert_any_call(upload_id, 2, self.path0)
assert not driver.put.called
assert not driver.get.called
assert not driver.multipart_abort.called
driver.multipart_complete.assert_called_once_with(upload_id)
completed = [
{
"src": self.path0,
"dst": self.path2,
"chunk_num": 0,
"size": getsize.return_value
},
{
"src": self.path1,
"dst": self.path2,
"chunk_num": 1,
"size": getsize.return_value
},
{
"src": self.path0,
"dst": self.path2,
"chunk_num": 2,
"size": getsize.return_value
},
]
assert len(ret["completed"]) == len(completed)
for job in completed:
assert job in ret["completed"], str(job) + ' not in ' + str(
ret["completed"])
assert ret["failed"] == []
def test_worker_download(self, getsize, isfile):
# str str
obj = cloudfs.FileTransfer(src=self.path2, dst=self.path1)
ret = obj.run()
driver = cloudfs.cloudfs.return_value
assert all([
call == mock.call("s3") for call in cloudfs.cloudfs.call_args_list
])
driver.get.assert_called_once_with(self.path2,
self.path1,
report_to=mock.ANY)
assert not driver.multipart_init.called
assert not driver.multipart_put.called
assert not driver.put.called
assert not driver.multipart_abort.called
assert not driver.multipart_complete.called
assert ret["completed"] == [
{
"src": self.path2,
"dst": self.path1,
"size": getsize.return_value
},
]
assert ret["failed"] == []
# iter iter
cloudfs.cloudfs.reset_mock()
dst_gen = self.make_generator(self.path0, self.path1, self.path0)
dst2 = self.path2 + 'one/'
dst3 = self.path2 + 'two/'
src_gen = self.make_generator(self.path2, dst2, dst3)
obj = cloudfs.FileTransfer(src=src_gen, dst=dst_gen)
ret = obj.run()
driver = cloudfs.cloudfs.return_value
assert all([
call == mock.call("s3") for call in cloudfs.cloudfs.call_args_list
])
driver.get.assert_any_call(self.path2, self.path0, report_to=mock.ANY)
driver.get.assert_any_call(dst2, self.path1, report_to=mock.ANY)
driver.get.assert_any_call(dst3, self.path0, report_to=mock.ANY)
assert not driver.multipart_init.called
assert not driver.multipart_put.called
assert not driver.put.called
assert not driver.multipart_abort.called
assert not driver.multipart_complete.called
completed = [
{
"src": self.path2,
"dst": self.path0,
"size": getsize.return_value
},
{
"src": dst2,
"dst": self.path1,
"size": getsize.return_value
},
{
"src": dst3,
"dst": self.path0,
"size": getsize.return_value
},
]
assert len(ret["completed"]) == len(completed)
for job in completed:
assert job in ret["completed"]
assert ret["failed"] == []
def test_worker_upload_retry(self, getsize, isfile):
# basic retry queue interaction
src_gen = self.make_generator(self.path0, self.path1)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2)
driver = cloudfs.cloudfs.return_value
driver.put.side_effect = Exception("test exception")
obj._retries_queue = mock.MagicMock()
obj._retries_queue.get_nowait.side_effect = Empty("test exc")
ret = obj.run()
assert len(obj._retries_queue.put.call_args_list) == 2
obj._retries_queue.put.assert_any_call((self.path0, self.path2, 1, -1))
obj._retries_queue.put.assert_any_call((self.path1, self.path2, 1, -1))
assert obj._retries_queue.get_nowait.call_args_list == [
mock.call()
] * 18, obj._retries_queue.get_nowait.call_args_list
# iter str multipart
cloudfs.cloudfs.reset_mock()
def side_effect(upload_id, chunk_num, src, firsttime=[True]):
"""
raise exception once on src=self.path1;
for driver.multipart_put(self._upload_id, chunk_num, src)
"""
if src == self.path1 and firsttime[0]:
firsttime[0] = False
raise Exception("test exception")
src_gen = self.make_generator(self.path0, self.path1, self.path0)
obj = cloudfs.FileTransfer(src=src_gen, dst=self.path2, multipart=True)
driver = cloudfs.cloudfs.return_value
driver.multipart_put.side_effect = side_effect
ret = obj.run()
upload_id = driver.multipart_init.return_value
assert all([
call == mock.call("s3") for call in cloudfs.cloudfs.call_args_list
])
driver.multipart_init.assert_called_once_with(self.path2,
getsize.return_value)
# check for the 4 calls 2 of which are equal
assert len(driver.multipart_put.call_args_list) == 4
driver.multipart_put.assert_any_call(upload_id, 0, self.path0)
driver.multipart_put.assert_any_call(upload_id, 2, self.path0)
driver.multipart_put.assert_any_call(upload_id, 1, self.path1)
driver.multipart_put.call_args_list.remove(
mock.call(upload_id, 1, self.path1))
driver.multipart_put.assert_any_call(upload_id, 1, self.path1)
assert not driver.put.called
assert not driver.get.called
assert not driver.multipart_abort.called
driver.multipart_complete.assert_called_once_with(upload_id)
completed = [
{
"src": self.path0,
"dst": self.path2,
"chunk_num": 0,
"size": getsize.return_value
},
{
"src": self.path1,
"dst": self.path2,
"chunk_num": 1,
"size": getsize.return_value
},
{
"src": self.path0,
"dst": self.path2,
"chunk_num": 2,
"size": getsize.return_value
},
]
assert len(ret["completed"]) == len(completed)
for job in completed:
assert job in ret["completed"], str(job) + ' not in ' + str(
ret["completed"])
assert ret["failed"] == []
