def test_zero_length_object_download(self): """ Tests downloading a zero-length object (exercises boundary conditions). """ res_download_handler = ResumableDownloadHandler() self.empty_src_key.get_contents_to_file(self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(0, get_cur_file_size(self.dst_fp))
def test_non_resumable_download(self): """ Tests that non-resumable downloads work """ self.small_src_key.get_contents_to_file(self.dst_fp) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_download_with_inconsistent_etag_in_tracker(self): """ Tests resumable download with an inconsistent etag in tracker file """ tmp_dir = self._MakeTempDir() dst_fp = self.make_dst_fp(tmp_dir) small_src_key_as_string, small_src_key = self.make_small_key() inconsistent_etag_tracker_file_name = os.path.join(tmp_dir, 'inconsistent_etag_tracker') f = open(inconsistent_etag_tracker_file_name, 'w') good_etag = small_src_key.etag.strip('"\'') new_val_as_list = [] for c in reversed(good_etag): new_val_as_list.append(c) f.write('%s\n' % ''.join(new_val_as_list)) f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=inconsistent_etag_tracker_file_name) # An error should be printed about the expired tracker, but then it # should run the update successfully. small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string())
def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harness to fail download after several hundred KB so download # server will have saved something before we retry. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2) larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE) larger_src_key = self._MakeKey(data=larger_src_key_as_string) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1)
def test_download_without_persistent_tracker(self): """ Tests a single resumable download, with no tracker persistence """ res_download_handler = ResumableDownloadHandler() self.small_src_key.get_contents_to_file(self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_multiple_in_process_failures_then_succeed(self): """ Tests resumable download that fails twice in one process, then completes """ res_download_handler = ResumableDownloadHandler(num_retries=3) self.small_src_key.get_contents_to_file(self.dst_fp, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_zero_length_object_download(self): """ Tests downloading a zero-length object (exercises boundary conditions). """ res_download_handler = ResumableDownloadHandler() dst_fp = self.make_dst_fp() k = self._MakeKey() k.get_contents_to_file(dst_fp, res_download_handler=res_download_handler) self.assertEqual(0, get_cur_file_size(dst_fp))
def test_broken_pipe_recovery(self): """ Tests handling of a Broken Pipe (which interacts with an httplib bug) """ exception = IOError(errno.EPIPE, "Broken pipe") harnass = CallbackTestHarnass(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) self.small_src_key.get_contents_to_file(self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_non_resumable_download(self): """ Tests that non-resumable downloads work """ dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file(dst_fp) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string())
def test_retryable_exception_recovery(self): """ Tests handling of a retryable exception """ # Test one of the RETRYABLE_EXCEPTIONS. exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0] harnass = CallbackTestHarnass(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) self.small_src_key.get_contents_to_file(self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_failed_and_restarted_download_with_persistent_tracker(self): """ Tests resumable download that fails once and then completes, with tracker file """ harnass = CallbackTestHarnass() res_download_handler = ResumableDownloadHandler(tracker_file_name=self.tracker_file_name, num_retries=1) self.small_src_key.get_contents_to_file(self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string()) # Ensure tracker file deleted. self.assertFalse(os.path.exists(self.tracker_file_name))
def test_download_without_persistent_tracker(self): """ Tests a single resumable download, with no tracker persistence """ res_download_handler = ResumableDownloadHandler() dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string())
def test_download_with_invalid_tracker_etag(self): """ Tests resumable download with a tracker file containing an invalid etag """ invalid_etag_tracker_file_name = "%s%sinvalid_etag_tracker" % (self.tmp_dir, os.sep) f = open(invalid_etag_tracker_file_name, "w") f.write("3.14159\n") f.close() res_download_handler = ResumableDownloadHandler(tracker_file_name=invalid_etag_tracker_file_name) # An error should be printed about the invalid tracker, but then it # should run the update successfully. self.small_src_key.get_contents_to_file(self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_retryable_exception_recovery(self): """ Tests handling of a retryable exception """ # Test one of the RETRYABLE_EXCEPTIONS. exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0] harness = CallbackTestHarness(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string())
def test_broken_pipe_recovery(self): """ Tests handling of a Broken Pipe (which interacts with an httplib bug) """ exception = IOError(errno.EPIPE, "Broken pipe") harnass = CallbackTestHarnass(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) self.small_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_failed_and_restarted_download_with_persistent_tracker(self): """ Tests resumable download that fails once and then completes, with tracker file """ harnass = CallbackTestHarnass() res_download_handler = ResumableDownloadHandler( tracker_file_name=self.tracker_file_name, num_retries=1) self.small_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string()) # Ensure tracker file deleted. self.assertFalse(os.path.exists(self.tracker_file_name))
def test_download_with_inconsistent_etag_in_tracker(self): """ Tests resumable download with an inconsistent etag in tracker file """ inconsistent_etag_tracker_file_name = "%s%sinconsistent_etag_tracker" % (self.tmp_dir, os.sep) f = open(inconsistent_etag_tracker_file_name, "w") good_etag = self.small_src_key.etag.strip("\"'") new_val_as_list = [] for c in reversed(good_etag): new_val_as_list.append(c) f.write("%s\n" % "".join(new_val_as_list)) f.close() res_download_handler = ResumableDownloadHandler(tracker_file_name=inconsistent_etag_tracker_file_name) # An error should be printed about the expired tracker, but then it # should run the update successfully. self.small_src_key.get_contents_to_file(self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_download_with_invalid_tracker_etag(self): """ Tests resumable download with a tracker file containing an invalid etag """ invalid_etag_tracker_file_name = ( '%s%sinvalid_etag_tracker' % (self.tmp_dir, os.sep)) f = open(invalid_etag_tracker_file_name, 'w') f.write('3.14159\n') f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=invalid_etag_tracker_file_name) # An error should be printed about the invalid tracker, but then it # should run the update successfully. self.small_src_key.get_contents_to_file( self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
def test_failed_and_restarted_download_with_persistent_tracker(self): """ Tests resumable download that fails once and then completes, with tracker file """ harness = CallbackTestHarness() tmpdir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmpdir) dst_fp = self.make_dst_fp(tmpdir) small_src_key_as_string, small_src_key = self.make_small_key() res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name, num_retries=1) small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) # Ensure tracker file deleted. self.assertFalse(os.path.exists(tracker_file_name))
def test_download_with_invalid_tracker_etag(self): """ Tests resumable download with a tracker file containing an invalid etag """ tmp_dir = self._MakeTempDir() dst_fp = self.make_dst_fp(tmp_dir) small_src_key_as_string, small_src_key = self.make_small_key() invalid_etag_tracker_file_name = os.path.join(tmp_dir, 'invalid_etag_tracker') f = open(invalid_etag_tracker_file_name, 'w') f.write('3.14159\n') f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=invalid_etag_tracker_file_name) # An error should be printed about the invalid tracker, but then it # should run the update successfully. small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string())
def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): """ Tests resumable download that fails completely in one process, then when restarted completes, using a tracker file """ # Set up test harness that causes more failures than a single # ResumableDownloadHandler instance will handle, writing enough data # before the first failure that some of it survives that process run. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2, num_times_to_fail=2) larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE) larger_src_key = self._MakeKey(data=larger_src_key_as_string) tmpdir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmpdir) dst_fp = self.make_dst_fp(tmpdir) res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name, num_retries=0) try: larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it one more time; this time should succeed. larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1)
def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harnass to fail download after several hundred KB so download # server will have saved something before we retry. harnass = CallbackTestHarnass( fail_after_n_bytes=self.larger_src_key_size/2) res_download_handler = ResumableDownloadHandler(num_retries=1) self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(self.larger_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.larger_src_key_as_string, self.larger_src_key.get_contents_as_string()) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harnass.transferred_seq_before_first_failure) > 1 and len(harnass.transferred_seq_after_first_failure) > 1)
def test_download_with_inconsistent_etag_in_tracker(self): """ Tests resumable download with an inconsistent etag in tracker file """ inconsistent_etag_tracker_file_name = ( '%s%sinconsistent_etag_tracker' % (self.tmp_dir, os.sep)) f = open(inconsistent_etag_tracker_file_name, 'w') good_etag = self.small_src_key.etag.strip('"\'') new_val_as_list = [] for c in reversed(good_etag): new_val_as_list.append(c) f.write('%s\n' % ''.join(new_val_as_list)) f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=inconsistent_etag_tracker_file_name) # An error should be printed about the expired tracker, but then it # should run the update successfully. self.small_src_key.get_contents_to_file( self.dst_fp, res_download_handler=res_download_handler) self.assertEqual(self.small_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.small_src_key_as_string, self.small_src_key.get_contents_as_string())
self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException, e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(self.tracker_file_name)) # Try it one more time; this time should succeed. self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) self.assertEqual(self.larger_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.larger_src_key_as_string, self.larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(self.tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harnass.transferred_seq_before_first_failure) > 1 and len(harnass.transferred_seq_after_first_failure) > 1) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harnass to fail download after several hundred KB so download # server will have saved something before we retry.
harnass = CallbackTestHarnass(fail_after_n_bytes=self.larger_src_key_size / 2, num_times_to_fail=2) res_download_handler = ResumableDownloadHandler(tracker_file_name=self.tracker_file_name, num_retries=0) try: self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler ) self.fail("Did not get expected ResumableDownloadException") except ResumableDownloadException, e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(self.tracker_file_name)) # Try it one more time; this time should succeed. self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler ) self.assertEqual(self.larger_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.larger_src_key_as_string, self.larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(self.tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harnass.transferred_seq_before_first_failure) > 1 and len(harnass.transferred_seq_after_first_failure) > 1 ) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harnass to fail download after several hundred KB so download # server will have saved something before we retry.
tracker_file_name=tracker_file_name, num_retries=0) try: larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException, e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it one more time; this time should succeed. larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harness to fail download after several hundred KB so download # server will have saved something before we retry.
try: self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException, e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(self.tracker_file_name)) # Try it one more time; this time should succeed. self.larger_src_key.get_contents_to_file( self.dst_fp, cb=harnass.call, res_download_handler=res_download_handler) self.assertEqual(self.larger_src_key_size, get_cur_file_size(self.dst_fp)) self.assertEqual(self.larger_src_key_as_string, self.larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(self.tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harnass.transferred_seq_before_first_failure) > 1 and len(harnass.transferred_seq_after_first_failure) > 1) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harnass to fail download after several hundred KB so download # server will have saved something before we retry.
try: larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException, e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it one more time; this time should succeed. larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harness to fail download after several hundred KB so download # server will have saved something before we retry.