def testSyntheticStepSplitProvider(self):
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, 2, False, False, None)
self.assertEquals(list(provider.split('ab', (2, 15))), [(2, 8),
(8, 15)])
self.assertEquals(list(provider.split('ab', (0, 8))), [(0, 4), (4, 8)])
self.assertEquals(list(provider.split('ab', (0, 0))), [])
self.assertEquals(list(provider.split('ab', (2, 3))), [(2, 3)])
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
10, 1, False, False, None)
self.assertEquals(list(provider.split('ab', (1, 10))), [(1, 10)])
self.assertEquals(provider.restriction_size('ab', (1, 10)), 9 * 2)
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
10, 3, False, False, None)
self.assertEquals(list(provider.split('ab', (1, 10))), [(1, 4), (4, 7),
(7, 10)])
self.assertEquals(provider.initial_restriction('a'), (0, 10))
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
10, 3, False, False, 45)
self.assertEquals(provider.restriction_size('ab', (1, 3)), 45)
tracker = provider.create_tracker((1, 6))
tracker.try_claim(1) # Claim to allow splitting.
self.assertEquals(tracker.try_split(.5), ((1, 3), (3, 6)))
def testSyntheticStepSplitProviderNoLiquidSharding(self):
# Verify Liquid Sharding Works
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, 5, True, False, None)
tracker = provider.create_tracker((1, 6))
tracker.try_claim(2)
self.assertEquals(tracker.try_split(.5), ((1, 4), (4, 6)))
# Verify No Liquid Sharding
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, 5, True, True, None)
tracker = provider.create_tracker((1, 6))
tracker.try_claim(2)
self.assertEquals(tracker.try_split(3), None)
def test_synthetic_step_split_provider_no_liquid_sharding(self):
# Verify Liquid Sharding Works
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, 5, True, False, None)
tracker = provider.create_tracker(OffsetRange(1, 6))
tracker.try_claim(2)
self.assertEqual(tracker.try_split(.5),
(OffsetRange(1, 4), OffsetRange(4, 6)))
# Verify No Liquid Sharding
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, 5, True, True, None)
tracker = provider.create_tracker(OffsetRange(1, 6))
tracker.try_claim(2)
self.assertEqual(tracker.try_split(3), None)
def testSyntheticStepSplitProviderUnevenChunks(self):
bundles = 4
provider = synthetic_pipeline.SyntheticSDFStepRestrictionProvider(
5, bundles, True, False, None)
self.verify_random_splits(provider, 4, 10, bundles)
self.verify_random_splits(provider, 4, 4, 0)
self.verify_random_splits(provider, 0, 1, 1)
self.verify_random_splits(provider, 0, bundles - 2, bundles)
