def test_overlap_cause_small_copy_remainder_3_intent_case(self): self.assertGreater(self.min_size, MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) source_copy3 = CopySource(copy_size) write_intents = [ WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), WriteIntent(source_copy3, destination_offset=2 * copy_overlap_offset), ] # 12345678901234 # sc1: *****X # sc2: X***** # sc3: XX**** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part([ (source_copy3, 2 * MEGABYTE, copy_overlap_offset), # this means: download and then upload ]), ], )
def test_overlap_copy_and_small_copy_remainder_and_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) self.assertGreater(self.recommended_size, self.min_size + MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(self.min_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] # 12345678901 # su: XXXXXX*****(...) # sc1: ****** # sc2: XXXXX self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, copy_size), part(source_upload, copy_size, self.recommended_size - copy_size), ], )
def test_overlap_protected_copy_and_upload(self): self.assertGreater(self.min_size, MEGABYTE) self.assertGreater(self.recommended_size, 2 * self.min_size) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] upload_offset = copy_overlap_offset + copy_size # 123456789012 # su: XXXXXXXXXX**(...) # sc1: *****X # sc2: X***** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part(source_upload, upload_offset, self.recommended_size - upload_offset), ], )
def test_basic_local_overlap(self): source1 = UploadSource(self.recommended_size * 2) source2 = UploadSource(self.recommended_size * 2) write_intents = [ WriteIntent(source1), WriteIntent(source2, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source1, 0, self.recommended_size)] + self.split_source_to_part_defs(source2, [self.recommended_size] * 2), )
def test_local_remote_overlap_end(self): source_upload = UploadSource(self.recommended_size * 2) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload, 0, self.recommended_size), part(source_copy), ], )
def test_local_stairs_overlap(self): """ intent 0 #### intent 1 #### intent 2 #### intent 3 #### """ self.assertEqual(self.recommended_size % 4, 0) shift = int(self.recommended_size / 4) sources = [UploadSource(self.recommended_size) for i in range(4)] write_intents = [ WriteIntent(source, destination_offset=i * shift) for i, source in enumerate(sources) ] three_quarters = int(3 * self.recommended_size / 4) # 1234567 # su1: **** # su2: XXXX # su3: XXXX # su4: X*** self.verify_emerge_plan_for_write_intents( write_intents, [ part([ (sources[0], 0, three_quarters), (sources[-1], 0, shift) ]), part(sources[-1], shift, three_quarters), ], )
def test_single_multipart_copy(self): source = CopySource(5 * self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, [self.max_size] * 5) )
def test_single_part_copy(self): source = CopySource(self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], )
def test_single_part_upload(self): source = UploadSource(self.recommended_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], )
def test_upload_multiple_sources(self): self.assertEqual(self.recommended_size % 8, 0) unit_part_size = int(self.recommended_size / 8) uneven_part_size = 3 * unit_part_size sources = [ UploadSource(uneven_part_size) for i in range(8) ] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [ part([ sources[0], sources[1], (sources[2], 0, 2 * unit_part_size), ]), part([ (sources[2], 2 * unit_part_size, unit_part_size), sources[3], sources[4], (sources[5], 0, unit_part_size), ]), part([ (sources[5], unit_part_size, 2 * unit_part_size), sources[6], sources[7], ]), ], )
def test_single_small_copy(self): source = CopySource(self.min_size - 1) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [ part([source]), # this means: download and then upload ], )
def test_local_small_copy_overlap(self): self.assertGreater(self.recommended_size, self.min_size * 3 - 3) source_upload = UploadSource(self.recommended_size) small_size = self.min_size - 1 source_copy1 = CopySource(small_size) source_copy2 = CopySource(small_size) source_copy3 = CopySource(small_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=small_size), WriteIntent(source_copy3, destination_offset=2 * small_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source_upload)], )
def test_raise_on_hole(self): source_upload1 = UploadSource(self.recommended_size) source_upload2 = UploadSource(self.recommended_size) source_copy1 = CopySource(self.recommended_size) source_copy2 = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload1), WriteIntent(source_upload2, destination_offset=self.recommended_size + 2 * MEGABYTE), WriteIntent(source_copy1, destination_offset=MEGABYTE), WriteIntent(source_copy2, destination_offset=self.recommended_size + 3 * MEGABYTE), ] hole_msg = ('Cannot emerge file with holes. ' 'Found hole range: ({}, {})'.format( write_intents[2].destination_end_offset, write_intents[1].destination_offset, )) with self.assertRaises(ValueError, hole_msg): self.planner.get_emerge_plan(write_intents)
def test_single_multipart_upload(self): self.assertGreater(self.recommended_size, 2 * self.min_size) remainder = 2 * self.min_size source = UploadSource(self.recommended_size * 5 + remainder) expected_part_sizes = [self.recommended_size] * 5 + [remainder] self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), )
def test_single_small_copy(self): source = CopySource(self.min_size - 1) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [ # single small copy should be processed using `copy_file` # which does not have minimum file size limit part(source), ], )
def test_copy_then_small_copy(self): source_copy = CopySource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) write_intents = WriteIntent.wrap_sources_iterator([source_copy, source_small_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy), part([source_small_copy]), # this means: download and then upload ], )
def test_simple_concatenate(self): sources = [ CopySource(self.recommended_size), UploadSource(self.recommended_size), CopySource(self.recommended_size), UploadSource(self.recommended_size), ] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [part(source) for source in sources], )
def test_single_multipart_copy_remainder(self): self.assertGreaterEqual(self.min_size, 2) source = CopySource(5 * self.max_size + int(self.min_size / 2)) expected_part_count = 7 base_part_size = int(source.get_content_length() / expected_part_count) size_remainder = source.get_content_length() % expected_part_count expected_part_sizes = ( [base_part_size + 1] * size_remainder + [base_part_size] * (expected_part_count - size_remainder) ) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), )
def test_upload_small_copy_then_copy(self): source_upload = UploadSource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator([source_upload, source_small_copy, source_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_upload, source_small_copy, ]), part(source_copy), ] )
def test_small_copy_then_copy(self): self.assertGreater(self.min_size, MEGABYTE) source_small_copy = CopySource(self.min_size - MEGABYTE) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator([source_small_copy, source_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_small_copy, (source_copy, 0, MEGABYTE), ]), part(source_copy, MEGABYTE, self.recommended_size - MEGABYTE) ], )
def test_small_upload_not_enough_copy_then_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) source_small_upload = UploadSource(self.min_size - 2 * MEGABYTE) source_copy = CopySource(self.min_size + MEGABYTE) source_upload = UploadSource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_small_upload, source_copy, source_upload] ) small_parts_len = source_small_upload.get_content_length() + source_copy.get_content_length() source_upload_split_offset = self.recommended_size - small_parts_len self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_small_upload, source_copy, (source_upload, 0, source_upload_split_offset), ]), part(source_upload, source_upload_split_offset, small_parts_len), ], )
def test_empty_upload(self): source_upload = UploadSource(0) self.verify_emerge_plan_for_write_intents( [WriteIntent(source_upload)], [part(source_upload)], )