def testSeekAndReturn(self):
   """Tests seeking to the end of the wrapper (simulates getting size)."""
   write_values = []
   with open(self.test_data_file, 'rb') as stream:
     while True:
       data = stream.read(TRANSFER_BUFFER_SIZE)
       if not data:
         break
       write_values.append(data)
   upload_file = self.CreateTempFile()
   mock_api = self.MockDownloadCloudApi(write_values)
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url,
       self.test_data_file_len,
       mock_api,
       download_chunk_size=self.test_data_file_len)
   with open(upload_file, 'wb') as upload_stream:
     current_position = 0
     daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
     daisy_chain_wrapper.seek(current_position)
     while True:
       data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
       current_position += len(data)
       daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
       daisy_chain_wrapper.seek(current_position)
       if not data:
         break
       upload_stream.write(data)
   self.assertEquals(mock_api.get_calls, 1)
   with open(upload_file, 'rb') as upload_stream:
     with open(self.test_data_file, 'rb') as download_stream:
       self.assertEqual(upload_stream.read(), download_stream.read())
 def testInvalidSeek(self):
   """Tests that seeking fails for unsupported seek arguments."""
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, self.MockDownloadCloudApi([]))
   try:
     # SEEK_CUR is invalid.
     daisy_chain_wrapper.seek(0, whence=os.SEEK_CUR)
     self.fail('Expected exception')
   except IOError, e:
     self.assertIn('does not support seek mode', str(e))
示例#3
0
 def testInvalidSeek(self):
   """Tests that seeking fails for unsupported seek arguments."""
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, self.MockDownloadCloudApi([]))
   try:
     # SEEK_CUR is invalid.
     daisy_chain_wrapper.seek(0, whence=os.SEEK_CUR)
     self.fail('Expected exception')
   except IOError, e:
     self.assertIn('does not support seek mode', str(e))
 def testDownloadWithDifferentChunkSize(self):
     """Tests multiple calls to GetObjectMedia."""
     upload_file = self.CreateTempFile()
     write_values = []
     with open(self.test_data_file, 'rb') as stream:
         # Use an arbitrary size greater than TRANSFER_BUFFER_SIZE for writing
         # data to the buffer. For reading from the buffer
         # WriteFromWrapperToFile will use TRANSFER_BUFFER_SIZE.
         buffer_write_size = TRANSFER_BUFFER_SIZE * 2 + 10
         while True:
             # Write data with size.
             data = stream.read(buffer_write_size)
             if not data:
                 break
             write_values.append(data)
     mock_api = self.MockDownloadCloudApi(write_values)
     daisy_chain_wrapper = DaisyChainWrapper(
         self._dummy_url,
         self.test_data_file_len,
         mock_api,
         download_chunk_size=TRANSFER_BUFFER_SIZE)
     self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
     num_expected_calls = math.ceil(self.test_data_file_len /
                                    TRANSFER_BUFFER_SIZE)
     # Since the chunk size is < the file size, multiple calls to GetObjectMedia
     # should be made.
     self.assertEqual(mock_api.get_calls, num_expected_calls)
     with open(upload_file, 'rb') as upload_stream:
         with open(self.test_data_file, 'rb') as download_stream:
             self.assertEqual(upload_stream.read(), download_stream.read())
 def testDownloadMultiChunk(self):
     """Tests multiple calls to GetObjectMedia."""
     upload_file = self.CreateTempFile()
     write_values = []
     with open(self.test_data_file, 'rb') as stream:
         while True:
             data = stream.read(TRANSFER_BUFFER_SIZE)
             if not data:
                 break
             write_values.append(data)
     mock_api = self.MockDownloadCloudApi(write_values)
     daisy_chain_wrapper = DaisyChainWrapper(
         self._dummy_url,
         self.test_data_file_len,
         mock_api,
         download_chunk_size=TRANSFER_BUFFER_SIZE)
     self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
     num_expected_calls = self.test_data_file_len // TRANSFER_BUFFER_SIZE
     if self.test_data_file_len % TRANSFER_BUFFER_SIZE:
         num_expected_calls += 1
     # Since the chunk size is < the file size, multiple calls to GetObjectMedia
     # should be made.
     self.assertEqual(mock_api.get_calls, num_expected_calls)
     with open(upload_file, 'rb') as upload_stream:
         with open(self.test_data_file, 'rb') as download_stream:
             self.assertEqual(upload_stream.read(), download_stream.read())
 def testDownloadSingleChunk(self):
     """Tests a single call to GetObjectMedia."""
     write_values = []
     with open(self.test_data_file, 'rb') as stream:
         while True:
             data = stream.read(TRANSFER_BUFFER_SIZE)
             if not data:
                 break
             write_values.append(data)
     upload_file = self.CreateTempFile()
     # Test for a single call even if the chunk size is larger than the data.
     for chunk_size in (self.test_data_file_len,
                        self.test_data_file_len + 1):
         mock_api = self.MockDownloadCloudApi(write_values)
         daisy_chain_wrapper = DaisyChainWrapper(
             self._dummy_url,
             self.test_data_file_len,
             mock_api,
             download_chunk_size=chunk_size)
         self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
         # Since the chunk size is >= the file size, only a single GetObjectMedia
         # call should be made.
         self.assertEquals(mock_api.get_calls, 1)
         with open(upload_file, 'rb') as upload_stream:
             with open(self.test_data_file, 'rb') as download_stream:
                 self.assertEqual(upload_stream.read(),
                                  download_stream.read())
示例#7
0
 def testRestartDownloadThread(self):
   """Tests seek to non-stored position; this restarts the download thread."""
   write_values = []
   with open(self.test_data_file, 'rb') as stream:
     while True:
       data = stream.read(TRANSFER_BUFFER_SIZE)
       if not data:
         break
       write_values.append(data)
   upload_file = self.CreateTempFile()
   mock_api = self.MockDownloadCloudApi(write_values)
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, mock_api,
       download_chunk_size=self.test_data_file_len)
   daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
   daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
   daisy_chain_wrapper.seek(0)
   self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
   self.assertEquals(mock_api.get_calls, 2)
   with open(upload_file, 'rb') as upload_stream:
     with open(self.test_data_file, 'rb') as download_stream:
       self.assertEqual(upload_stream.read(), download_stream.read())
    def testDownloadWithPartialWrite(self):
        """Tests unaligned writes to the download stream from GetObjectMedia."""
        with open(self.test_data_file, 'rb') as stream:
            chunk = stream.read(TRANSFER_BUFFER_SIZE)
        # Though it may seem equivalent, the `:1` is actually necessary, without
        # it in python 3, `one_byte` would be int(77) and with it, `one_byte` is
        # the expected value of b'M' (using case where start of chunk is b'MJoTM...')
        one_byte = chunk[0:1]
        chunk_minus_one_byte = chunk[1:TRANSFER_BUFFER_SIZE]
        half_chunk = chunk[0:TRANSFER_BUFFER_SIZE // 2]

        write_values_dict = {
            'First byte first chunk unaligned':
            (one_byte, chunk_minus_one_byte, chunk, chunk),
            'Last byte first chunk unaligned':
            (chunk_minus_one_byte, chunk, chunk),
            'First byte second chunk unaligned': (chunk, one_byte,
                                                  chunk_minus_one_byte, chunk),
            'Last byte second chunk unaligned': (chunk, chunk_minus_one_byte,
                                                 one_byte, chunk),
            'First byte final chunk unaligned': (chunk, chunk, one_byte,
                                                 chunk_minus_one_byte),
            'Last byte final chunk unaligned':
            (chunk, chunk, chunk_minus_one_byte, one_byte),
            'Half chunks': (half_chunk, half_chunk, half_chunk),
            'Many unaligned':
            (one_byte, half_chunk, one_byte, half_chunk, chunk,
             chunk_minus_one_byte, chunk, one_byte, half_chunk, one_byte)
        }
        upload_file = self.CreateTempFile()
        for case_name, write_values in six.iteritems(write_values_dict):
            expected_contents = b''
            for write_value in write_values:
                expected_contents += write_value
            mock_api = self.MockDownloadCloudApi(write_values)
            daisy_chain_wrapper = DaisyChainWrapper(
                self._dummy_url,
                len(expected_contents),
                mock_api,
                download_chunk_size=self.test_data_file_len)
            self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
            with open(upload_file, 'rb') as upload_stream:
                self.assertEqual(
                    upload_stream.read(), expected_contents,
                    'Uploaded file contents for case %s did not match' %
                    case_name)
  def testDownloadThreadException(self):
    """Tests that an exception is propagated via the upload thread."""

    class DownloadException(Exception):
      pass

    write_values = [b'a', b'b',
                    DownloadException('Download thread forces failure')]
    upload_file = self.CreateTempFile()
    mock_api = self.MockDownloadCloudApi(write_values)
    daisy_chain_wrapper = DaisyChainWrapper(
        self._dummy_url, self.test_data_file_len, mock_api,
        download_chunk_size=self.test_data_file_len)
    try:
      self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
      self.fail('Expected exception')
    except DownloadException, e:
      self.assertIn('Download thread forces failure', str(e))
    def testDownloadWithPartialWrite(self):
        """Tests unaligned writes to the download stream from GetObjectMedia."""
        with open(self.test_data_file, 'rb') as stream:
            chunk = stream.read(TRANSFER_BUFFER_SIZE)
        one_byte = chunk[0]
        chunk_minus_one_byte = chunk[1:TRANSFER_BUFFER_SIZE]
        half_chunk = chunk[0:TRANSFER_BUFFER_SIZE / 2]

        write_values_dict = {
            'First byte first chunk unaligned':
            (one_byte, chunk_minus_one_byte, chunk, chunk),
            'Last byte first chunk unaligned':
            (chunk_minus_one_byte, chunk, chunk),
            'First byte second chunk unaligned': (chunk, one_byte,
                                                  chunk_minus_one_byte, chunk),
            'Last byte second chunk unaligned': (chunk, chunk_minus_one_byte,
                                                 one_byte, chunk),
            'First byte final chunk unaligned': (chunk, chunk, one_byte,
                                                 chunk_minus_one_byte),
            'Last byte final chunk unaligned':
            (chunk, chunk, chunk_minus_one_byte, one_byte),
            'Half chunks': (half_chunk, half_chunk, half_chunk),
            'Many unaligned':
            (one_byte, half_chunk, one_byte, half_chunk, chunk,
             chunk_minus_one_byte, chunk, one_byte, half_chunk, one_byte)
        }
        upload_file = self.CreateTempFile()
        for case_name, write_values in write_values_dict.iteritems():
            expected_contents = b''
            for write_value in write_values:
                expected_contents += write_value
            mock_api = self.MockDownloadCloudApi(write_values)
            daisy_chain_wrapper = DaisyChainWrapper(
                self._dummy_url,
                len(expected_contents),
                mock_api,
                download_chunk_size=self.test_data_file_len)
            self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
            with open(upload_file, 'rb') as upload_stream:
                self.assertEqual(
                    upload_stream.read(), expected_contents,
                    'Uploaded file contents for case %s did not match' %
                    case_name)
 def testDownloadWithZeroWrites(self):
   """Tests 0-byte writes to the download stream from GetObjectMedia."""
   write_values = []
   with open(self.test_data_file, 'rb') as stream:
     while True:
       write_values.append(b'')
       data = stream.read(TRANSFER_BUFFER_SIZE)
       write_values.append(b'')
       if not data:
         break
       write_values.append(data)
   upload_file = self.CreateTempFile()
   mock_api = self.MockDownloadCloudApi(write_values)
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, mock_api,
       download_chunk_size=self.test_data_file_len)
   self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
   self.assertEquals(mock_api.get_calls, 1)
   with open(upload_file, 'rb') as upload_stream:
     with open(self.test_data_file, 'rb') as download_stream:
       self.assertEqual(upload_stream.read(), download_stream.read())
 def testRestartDownloadThread(self):
   """Tests seek to non-stored position; this restarts the download thread."""
   write_values = []
   with open(self.test_data_file, 'rb') as stream:
     while True:
       data = stream.read(TRANSFER_BUFFER_SIZE)
       if not data:
         break
       write_values.append(data)
   upload_file = self.CreateTempFile()
   mock_api = self.MockDownloadCloudApi(write_values)
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, mock_api,
       download_chunk_size=self.test_data_file_len)
   daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
   daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
   daisy_chain_wrapper.seek(0)
   self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file)
   self.assertEquals(mock_api.get_calls, 2)
   with open(upload_file, 'rb') as upload_stream:
     with open(self.test_data_file, 'rb') as download_stream:
       self.assertEqual(upload_stream.read(), download_stream.read())
 def testSeekAndReturn(self):
   """Tests seeking to the end of the wrapper (simulates getting size)."""
   write_values = []
   with open(self.test_data_file, 'rb') as stream:
     while True:
       data = stream.read(TRANSFER_BUFFER_SIZE)
       if not data:
         break
       write_values.append(data)
   upload_file = self.CreateTempFile()
   mock_api = self.MockDownloadCloudApi(write_values)
   daisy_chain_wrapper = DaisyChainWrapper(
       self._dummy_url, self.test_data_file_len, mock_api,
       download_chunk_size=self.test_data_file_len)
   with open(upload_file, 'wb') as upload_stream:
     current_position = 0
     daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
     daisy_chain_wrapper.seek(current_position)
     while True:
       data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE)
       current_position += len(data)
       daisy_chain_wrapper.seek(0, whence=os.SEEK_END)
       daisy_chain_wrapper.seek(current_position)
       if not data:
         break
       upload_stream.write(data)
   self.assertEquals(mock_api.get_calls, 1)
   with open(upload_file, 'rb') as upload_stream:
     with open(self.test_data_file, 'rb') as download_stream:
       self.assertEqual(upload_stream.read(), download_stream.read())