def test_cacheable_key_with_version_map(self): p = beam.Pipeline(interactive_runner.InteractiveRunner()) ie.current_env().set_cache_manager(InMemoryCache(), p) # pylint: disable=range-builtin-not-iterating init_pcoll = p | 'Init Create' >> beam.Create(range(10)) # It's normal that when executing, the pipeline object is a different # but equivalent instance from what user has built. The pipeline instrument # should be able to identify if the original instance has changed in an # interactive env while mutating the other instance for execution. The # version map can be used to figure out what the PCollection instances are # in the original instance and if the evaluation has changed since last # execution. p2 = beam.Pipeline(interactive_runner.InteractiveRunner()) ie.current_env().set_cache_manager(InMemoryCache(), p2) # pylint: disable=range-builtin-not-iterating init_pcoll_2 = p2 | 'Init Create' >> beam.Create(range(10)) _, ctx = p2.to_runner_api(return_context=True) # The cacheable_key should use id(init_pcoll) as prefix even when # init_pcoll_2 is supplied as long as the version map is given. self.assertEqual( instr.cacheable_key( init_pcoll_2, instr.pcolls_to_pcoll_id(p2, ctx), {'ref_PCollection_PCollection_8': str(id(init_pcoll))}), str(id(init_pcoll)) + '_ref_PCollection_PCollection_8')
def test_cacheable_key_without_version_map(self): p = beam.Pipeline(interactive_runner.InteractiveRunner()) # pylint: disable=range-builtin-not-iterating init_pcoll = p | 'Init Create' >> beam.Create(range(10)) _, ctx = p.to_runner_api(use_fake_coders=True, return_context=True) self.assertEqual( instr.cacheable_key(init_pcoll, instr.pcolls_to_pcoll_id(p, ctx)), str(id(init_pcoll)) + '_ref_PCollection_PCollection_10')
def test_cacheable_key_without_version_map(self): p = beam.Pipeline(interactive_runner.InteractiveRunner()) ie.current_env().set_cache_manager(InMemoryCache(), p) # pylint: disable=range-builtin-not-iterating init_pcoll = p | 'Init Create' >> beam.Create(range(10)) _, ctx = p.to_runner_api(return_context=True) self.assertEqual( instr.cacheable_key(init_pcoll, instr.pcolls_to_pcoll_id(p, ctx)), str(id(init_pcoll)) + '_ref_PCollection_PCollection_8')