def test_normal_non_threaded(self): # The context object is pretty straightforward. # This shows the non threaded usage of this object. context = MultipartUploadContext(expected_parts=3) # First you can announce an upload id. context.announce_upload_id('my_upload_id') # Then a thread that was waiting on the id would be notified. self.assertEqual(context.wait_for_upload_id(), 'my_upload_id') # Then thread would chug away at the parts. context.announce_finished_part(etag='etag1', part_number=1) context.announce_finished_part(etag='etag2', part_number=2) context.announce_finished_part(etag='etag3', part_number=3) # Then a thread that was waiting for all the parts to finish # would be notified. self.assertEqual(context.wait_for_parts_to_finish(), [{ 'ETag': 'etag1', 'PartNumber': 1 }, { 'ETag': 'etag2', 'PartNumber': 2 }, { 'ETag': 'etag3', 'PartNumber': 3 }]) context.announce_completed() # This will return right away since we've already announced completion. self.assertIsNone(context.wait_for_completion())
def test_streaming_threaded_parts(self): # This is similar to the basic threaded parts test but instead # the thread has to wait to know exactly how many parts are # expected from the stream. This is indicated when the expected # parts of the context changes from ... to an integer. self.context = MultipartUploadContext(expected_parts='...') upload_part_thread = threading.Thread(target=self.upload_part, args=(1, )) # Once this thread starts it will immediately block. self.start_thread(upload_part_thread) # Also, let's start the thread that will do the complete # multipart upload. It will also block because it needs all # the parts so it's blocked up the upload_part_thread. It also # needs the upload_id so it's blocked on that as well. complete_upload_thread = threading.Thread(target=self.complete_upload) self.start_thread(complete_upload_thread) # Then finally the CreateMultipartUpload completes and we # announce the upload id. self.create_upload('my_upload_id') # The complete upload thread should still be waiting for an expect # parts number. with self.call_lock: was_completed = (len(self.calls) > 2) # The upload_part thread can now proceed as well as the complete # multipart upload thread. self.context.announce_total_parts(1) self.join_threads() self.assertIsNone(self.caught_exception) # Make sure that the completed task was never called since it was # waiting to announce the parts. self.assertFalse(was_completed) # We can verify that the invariants still hold. self.assertEqual(len(self.calls), 3) # First there should be three calls, create, upload, complete. self.assertEqual(self.calls[0][0], 'create_multipart_upload') self.assertEqual(self.calls[1][0], 'upload_part') self.assertEqual(self.calls[2][0], 'complete_upload') # Verify the correct args were used. self.assertEqual(self.calls[0][1], 'my_upload_id') self.assertEqual(self.calls[1][1:], (1, 'my_upload_id')) self.assertEqual(self.calls[2][1:], ('my_upload_id', [{ 'ETag': 'etag1', 'PartNumber': 1 }]))
def test_randomized_stress_test(self): # Now given that we've verified the functionality from # the two tests above, we randomize the threading to ensure # that the order doesn't actually matter. The invariant that # the CreateMultipartUpload is called first, then UploadPart # operations are called with the appropriate upload_id, then # CompleteMultipartUpload with the appropriate upload_id and # parts list should hold true regardless of how the threads # are ordered. # I've run this with much larger values, but 100 is a good # tradeoff with coverage vs. execution time. for i in range(100): expected_parts = random.randint(2, 50) self.context = MultipartUploadContext( expected_parts=expected_parts) self.threads = [] self.calls = [] all_threads = [ threading.Thread(target=self.complete_upload), threading.Thread(target=self.create_upload, args=('my_upload_id', )), threading.Thread(target=self.wait_for_upload_complete), ] for i in range(1, expected_parts + 1): all_threads.append( threading.Thread(target=self.upload_part, args=(i, ))) random.shuffle(all_threads) for thread in all_threads: self.start_thread(thread) self.join_threads() self.assertEqual(self.calls[0][0], 'create_multipart_upload') self.assertEqual(self.calls[-1][0], 'arbitrary_post_complete_operation') self.assertEqual(self.calls[-2][0], 'complete_upload') parts = set() for call in self.calls[1:-2]: self.assertEqual(call[0], 'upload_part') self.assertEqual(call[2], 'my_upload_id') parts.add(call[1]) self.assertEqual(len(parts), expected_parts)
def setUp(self): self.context = MultipartUploadContext(expected_parts=1) self.calls = [] self.threads = [] self.call_lock = threading.Lock() self.caught_exception = None