def testSyntheticStepSplitProvider(self): provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, 2, False, False, None) self.assertEquals(list(provider.split('ab', (2, 15))), [(2, 8), (8, 15)]) self.assertEquals(list(provider.split('ab', (0, 8))), [(0, 4), (4, 8)]) self.assertEquals(list(provider.split('ab', (0, 0))), []) self.assertEquals(list(provider.split('ab', (2, 3))), [(2, 3)]) provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 10, 1, False, False, None) self.assertEquals(list(provider.split('ab', (1, 10))), [(1, 10)]) self.assertEquals(provider.restriction_size('ab', (1, 10)), 9 * 2) provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 10, 3, False, False, None) self.assertEquals(list(provider.split('ab', (1, 10))), [(1, 4), (4, 7), (7, 10)]) self.assertEquals(provider.initial_restriction('a'), (0, 10)) provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 10, 3, False, False, 45) self.assertEquals(provider.restriction_size('ab', (1, 3)), 45) tracker = provider.create_tracker((1, 6)) tracker.try_claim(1) # Claim to allow splitting. self.assertEquals(tracker.try_split(.5), ((1, 3), (3, 6)))
def testSyntheticStepSplitProviderNoLiquidSharding(self): # Verify Liquid Sharding Works provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, 5, True, False, None) tracker = provider.create_tracker((1, 6)) tracker.try_claim(2) self.assertEquals(tracker.try_split(.5), ((1, 4), (4, 6))) # Verify No Liquid Sharding provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, 5, True, True, None) tracker = provider.create_tracker((1, 6)) tracker.try_claim(2) self.assertEquals(tracker.try_split(3), None)
def test_synthetic_step_split_provider_no_liquid_sharding(self): # Verify Liquid Sharding Works provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, 5, True, False, None) tracker = provider.create_tracker(OffsetRange(1, 6)) tracker.try_claim(2) self.assertEqual(tracker.try_split(.5), (OffsetRange(1, 4), OffsetRange(4, 6))) # Verify No Liquid Sharding provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, 5, True, True, None) tracker = provider.create_tracker(OffsetRange(1, 6)) tracker.try_claim(2) self.assertEqual(tracker.try_split(3), None)
def testSyntheticStepSplitProviderUnevenChunks(self): bundles = 4 provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider( 5, bundles, True, False, None) self.verify_random_splits(provider, 4, 10, bundles) self.verify_random_splits(provider, 4, 4, 0) self.verify_random_splits(provider, 0, 1, 1) self.verify_random_splits(provider, 0, bundles - 2, bundles)