def testSeekAndReturn(self):
"""Tests seeking to the end of the wrapper (simulates getting size)."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
self.test_data_file_len,
mock_api,
download_chunk_size=self.test_data_file_len)
with open(upload_file, 'wb') as upload_stream:
current_position = 0
daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
daisy_chain_wrapper.seek(current_position)
while True:
data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
current_position += len(data)
daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
daisy_chain_wrapper.seek(current_position)
if not data:
break
upload_stream.write(data)
self.assertEquals(mock_api.get_calls, 1)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testInvalidSeek(self):
"""Tests that seeking fails for unsupported seek arguments."""
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, self.MockDownloadCloudApi([]))
try:
# SEEK_CUR is invalid.
daisy_chain_wrapper.seek(0, whence=os.SEEK_CUR)
self.fail('Expected exception')
except IOError, e:
self.assertIn('does not support seek mode', str(e))
def testInvalidSeek(self):
"""Tests that seeking fails for unsupported seek arguments."""
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, self.MockDownloadCloudApi([]))
try:
# SEEK_CUR is invalid.
daisy_chain_wrapper.seek(0, whence=os.SEEK_CUR)
self.fail('Expected exception')
except IOError, e:
self.assertIn('does not support seek mode', str(e))
def testDownloadWithDifferentChunkSize(self):
"""Tests multiple calls to GetObjectMedia."""
upload_file = self.CreateTempFile()
write_values = []
with open(self.test_data_file, 'rb') as stream:
# Use an arbitrary size greater than TRANSFER_BUFFER_SIZE for writing
# data to the buffer. For reading from the buffer
# WriteFromWrapperToFile will use TRANSFER_BUFFER_SIZE.
buffer_write_size = TRANSFER_BUFFER_SIZE * 2 + 10
while True:
# Write data with size.
data = stream.read(buffer_write_size)
if not data:
break
write_values.append(data)
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
self.test_data_file_len,
mock_api,
download_chunk_size=TRANSFER_BUFFER_SIZE)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
num_expected_calls = math.ceil(self.test_data_file_len /
TRANSFER_BUFFER_SIZE)
# Since the chunk size is < the file size, multiple calls to GetObjectMedia
# should be made.
self.assertEqual(mock_api.get_calls, num_expected_calls)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testDownloadMultiChunk(self):
"""Tests multiple calls to GetObjectMedia."""
upload_file = self.CreateTempFile()
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
self.test_data_file_len,
mock_api,
download_chunk_size=TRANSFER_BUFFER_SIZE)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
num_expected_calls = self.test_data_file_len // TRANSFER_BUFFER_SIZE
if self.test_data_file_len % TRANSFER_BUFFER_SIZE:
num_expected_calls += 1
# Since the chunk size is < the file size, multiple calls to GetObjectMedia
# should be made.
self.assertEqual(mock_api.get_calls, num_expected_calls)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testDownloadSingleChunk(self):
"""Tests a single call to GetObjectMedia."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
# Test for a single call even if the chunk size is larger than the data.
for chunk_size in (self.test_data_file_len,
self.test_data_file_len + 1):
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
self.test_data_file_len,
mock_api,
download_chunk_size=chunk_size)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
# Since the chunk size is >= the file size, only a single GetObjectMedia
# call should be made.
self.assertEquals(mock_api.get_calls, 1)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(),
download_stream.read())
def testRestartDownloadThread(self):
"""Tests seek to non-stored position; this restarts the download thread."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, mock_api,
download_chunk_size=self.test_data_file_len)
daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
daisy_chain_wrapper.seek(0)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
self.assertEquals(mock_api.get_calls, 2)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testDownloadWithPartialWrite(self):
"""Tests unaligned writes to the download stream from GetObjectMedia."""
with open(self.test_data_file, 'rb') as stream:
chunk = stream.read(TRANSFER_BUFFER_SIZE)
# Though it may seem equivalent, the `:1` is actually necessary, without
# it in python 3, `one_byte` would be int(77) and with it, `one_byte` is
# the expected value of b'M' (using case where start of chunk is b'MJoTM...')
one_byte = chunk[0:1]
chunk_minus_one_byte = chunk[1:TRANSFER_BUFFER_SIZE]
half_chunk = chunk[0:TRANSFER_BUFFER_SIZE // 2]
write_values_dict = {
'First byte first chunk unaligned':
(one_byte, chunk_minus_one_byte, chunk, chunk),
'Last byte first chunk unaligned':
(chunk_minus_one_byte, chunk, chunk),
'First byte second chunk unaligned': (chunk, one_byte,
chunk_minus_one_byte, chunk),
'Last byte second chunk unaligned': (chunk, chunk_minus_one_byte,
one_byte, chunk),
'First byte final chunk unaligned': (chunk, chunk, one_byte,
chunk_minus_one_byte),
'Last byte final chunk unaligned':
(chunk, chunk, chunk_minus_one_byte, one_byte),
'Half chunks': (half_chunk, half_chunk, half_chunk),
'Many unaligned':
(one_byte, half_chunk, one_byte, half_chunk, chunk,
chunk_minus_one_byte, chunk, one_byte, half_chunk, one_byte)
}
upload_file = self.CreateTempFile()
for case_name, write_values in six.iteritems(write_values_dict):
expected_contents = b''
for write_value in write_values:
expected_contents += write_value
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
len(expected_contents),
mock_api,
download_chunk_size=self.test_data_file_len)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
with open(upload_file, 'rb') as upload_stream:
self.assertEqual(
upload_stream.read(), expected_contents,
'Uploaded file contents for case %s did not match' %
case_name)
def testDownloadThreadException(self):
"""Tests that an exception is propagated via the upload thread."""
class DownloadException(Exception):
pass
write_values = [b'a', b'b',
DownloadException('Download thread forces failure')]
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, mock_api,
download_chunk_size=self.test_data_file_len)
try:
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
self.fail('Expected exception')
except DownloadException, e:
self.assertIn('Download thread forces failure', str(e))
def testDownloadWithPartialWrite(self):
"""Tests unaligned writes to the download stream from GetObjectMedia."""
with open(self.test_data_file, 'rb') as stream:
chunk = stream.read(TRANSFER_BUFFER_SIZE)
one_byte = chunk[0]
chunk_minus_one_byte = chunk[1:TRANSFER_BUFFER_SIZE]
half_chunk = chunk[0:TRANSFER_BUFFER_SIZE / 2]
write_values_dict = {
'First byte first chunk unaligned':
(one_byte, chunk_minus_one_byte, chunk, chunk),
'Last byte first chunk unaligned':
(chunk_minus_one_byte, chunk, chunk),
'First byte second chunk unaligned': (chunk, one_byte,
chunk_minus_one_byte, chunk),
'Last byte second chunk unaligned': (chunk, chunk_minus_one_byte,
one_byte, chunk),
'First byte final chunk unaligned': (chunk, chunk, one_byte,
chunk_minus_one_byte),
'Last byte final chunk unaligned':
(chunk, chunk, chunk_minus_one_byte, one_byte),
'Half chunks': (half_chunk, half_chunk, half_chunk),
'Many unaligned':
(one_byte, half_chunk, one_byte, half_chunk, chunk,
chunk_minus_one_byte, chunk, one_byte, half_chunk, one_byte)
}
upload_file = self.CreateTempFile()
for case_name, write_values in write_values_dict.iteritems():
expected_contents = b''
for write_value in write_values:
expected_contents += write_value
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url,
len(expected_contents),
mock_api,
download_chunk_size=self.test_data_file_len)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
with open(upload_file, 'rb') as upload_stream:
self.assertEqual(
upload_stream.read(), expected_contents,
'Uploaded file contents for case %s did not match' %
case_name)
def testDownloadWithZeroWrites(self):
"""Tests 0-byte writes to the download stream from GetObjectMedia."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
write_values.append(b'')
data = stream.read(TRANSFER_BUFFER_SIZE)
write_values.append(b'')
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, mock_api,
download_chunk_size=self.test_data_file_len)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
self.assertEquals(mock_api.get_calls, 1)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testRestartDownloadThread(self):
"""Tests seek to non-stored position; this restarts the download thread."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, mock_api,
download_chunk_size=self.test_data_file_len)
daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
daisy_chain_wrapper.seek(0)
self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
self.assertEquals(mock_api.get_calls, 2)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())
def testSeekAndReturn(self):
"""Tests seeking to the end of the wrapper (simulates getting size)."""
write_values = []
with open(self.test_data_file, 'rb') as stream:
while True:
data = stream.read(TRANSFER_BUFFER_SIZE)
if not data:
break
write_values.append(data)
upload_file = self.CreateTempFile()
mock_api = self.MockDownloadCloudApi(write_values)
daisy_chain_wrapper = DaisyChainWrapper(
self._dummy_url, self.test_data_file_len, mock_api,
download_chunk_size=self.test_data_file_len)
with open(upload_file, 'wb') as upload_stream:
current_position = 0
daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
daisy_chain_wrapper.seek(current_position)
while True:
data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
current_position += len(data)
daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
daisy_chain_wrapper.seek(current_position)
if not data:
break
upload_stream.write(data)
self.assertEquals(mock_api.get_calls, 1)
with open(upload_file, 'rb') as upload_stream:
with open(self.test_data_file, 'rb') as download_stream:
self.assertEqual(upload_stream.read(), download_stream.read())