class MultipartUploadTestCase(TestCase): def setUp(self): super(MultipartUploadTestCase, self).setUp() self.log = FakeLog() self.clock = FakeClock() def test_str(self): upload = MultipartUpload(None, None, None, None, None, Deferred()) upload.retry_strategy.clock = self.clock r = MultipartInitiationResponse('', '', '1234567890') upload.init_response = r self.assertEqual(str(upload), "MultipartUpload upload_id=1234567890") def _assertPartsCompleted(self, parts_generator, part_handler, received, task, client): expected = [('aaaaaaaaaa', 1), ('bbbbbbbbbb', 2), ('cccccccccc', 3), ('dddddddddd', 4), ('eeeeeeeeee', 5), ('ffffffffff', 6), ('gggggggggg', 7), ('hhhhhhhhhh', 8), ('iiiiiiiiii', 9), ('jjjjjjjjjj', 10)] self.assertEqual(parts_generator.generated, expected) self.assertEqual(part_handler.handled, expected) expected = [ ('e09c80c42fda55f9d992e59ca6b3307d', 1), ('82136b4240d6ce4ea7d03e51469a393b', 2), ('e9b3390206d8dfc5ffc9b09284c0bbde', 3), ('a9e49c7aefe022f0a8540361cce7575c', 4), ('c5ba867d9056b7cecf87f8ce88af90f8', 5), ('66952c6203ae23242590c0061675234d', 6), ('fdc68ea4cf2763996cf215451b291c63', 7), ('70270ca63a3de2d8905a9181a0245e58', 8), ('d98bcb28df1de541e95a6722c5e983ea', 9), ('c71a8da22bf4053760a604897627474c', 10)] self.assertEqual(received, expected) self.assertIsInstance(task.init_response, MultipartInitiationResponse) self.assertIsInstance(task.completion_response, MultipartCompletionResponse) self.assertEqual(client.calls, [ ('init_multipart_upload', 'mybucket', 'mykey', '', {}, {'acl': 'public-read'})]) def test_upload(self): client = FakeS3Client() parts_generator = DummyPartsGenerator() part_handler = DummyPartHandler() counter = PartsTransferredCounter('?') d = Deferred() received = [] amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock def check(task): self.assertIdentical(task, upload) self._assertPartsCompleted(parts_generator, part_handler, received, task, client) d.addCallback(check) received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) return d def test_upload_error_recovery(self): client = FakeS3Client() parts_generator = DummyPartsGenerator() part_handler = ErroringPartHandler() counter = PartsTransferredCounter('?') d = Deferred() received = [] amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock def check(task): self.flushLoggedErrors() self.assertEqual(len(self.clock.calls), 30) self.assertIdentical(task, upload) self._assertPartsCompleted(parts_generator, part_handler, received, task, client) d.addCallback(check) received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) return d def test_upload_error_timeout_finally(self): client = FakeS3Client() parts_generator = DummyPartsGenerator() part_handler = ErroringPartHandler(100) counter = PartsTransferredCounter('?') d = Deferred() received = [] amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) def eb(why): self.flushLoggedErrors() self.assertEquals(len(self.clock.calls), 110) return why d.addErrback(eb) return self.assertFailure(d, ValueError) def test_upload_on_complete_error_recovery(self): client = ErroringFakeS3Client() parts_generator = DummyPartsGenerator() part_handler = DummyPartHandler() counter = PartsTransferredCounter('?') d = Deferred() amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock def check(task): self.flushLoggedErrors() self.assertEqual(len(self.clock.calls), 3) self.assertIdentical(task, upload) self._assertPartsCompleted(parts_generator, part_handler, received, task, client) d.addCallback(check) received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) return d def test_upload_on_complete_error_timeout_finally(self): client = ErroringFakeS3Client(100) parts_generator = DummyPartsGenerator() part_handler = DummyPartHandler() counter = PartsTransferredCounter('?') d = Deferred() amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) def eb(why): self.flushLoggedErrors() self.assertEquals(len(self.clock.calls), 11) return why d.addErrback(eb) return self.assertFailure(d, ValueError) def test_upload_with_throughput_counter(self): client = FakeS3Client() parts_generator = DummyPartsGenerator() part_handler = DummyPartHandler() counter = PartsTransferredCounter('?') stats = SlidingStats(self.clock.seconds(), size=100) throughput_counter = ThroughputCounter(clock=self.clock, stats=stats) d = Deferred() amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock upload.throughput_counter = throughput_counter def check(task): self.assertIdentical(task, upload) self._assertPartsCompleted(parts_generator, part_handler, received, task, client) self.assertEquals(throughput_counter.read()[-1], (0, 100)) d.addCallback(check) received = [] upload.on_part_generated = received.append upload.upload('mybucket', 'mykey', '', {}, amz_headers) return d def test_upload_with_throughput_counter_and_error(self): client = FakeS3Client() parts_generator = DummyPartsGenerator() part_handler = ErroringPartHandler(100) counter = PartsTransferredCounter('?') stats = SlidingStats(self.clock.seconds(), size=100) throughput_counter = ThroughputCounter(clock=self.clock, stats=stats) d = Deferred() amz_headers = {'acl': 'public-read'} upload = MultipartUpload(client, None, parts_generator, part_handler, counter, d, self.log) upload.retry_strategy.clock = self.clock upload.throughput_counter = throughput_counter received = [] upload.on_part_generated = received.append def eb(why): self.flushLoggedErrors() self.assertEquals(len(self.clock.calls), 110) self.assertEquals(throughput_counter.read()[-1], (0, 0)) return why upload.upload('mybucket', 'mykey', '', {}, amz_headers) d.addErrback(eb) return self.assertFailure(d, ValueError)
def setUp(self): super(MultipartUploadTestCase, self).setUp() self.log = FakeLog() self.clock = FakeClock()
def test_initialization(self): clock = FakeClock() clock.tick(200) counter = ThroughputCounter(clock) last = counter.stats.slots[-1][0] self.assertEquals(last, 200)