def cleanup(self, pipeline=None): """Cleans up cached states for the given pipeline. Noop if the given pipeline is absent from the environment. Cleans up for all pipelines if no pipeline is specified.""" if pipeline: from apache_beam.runners.interactive import background_caching_job as bcj bcj.attempt_to_cancel_background_caching_job(pipeline) bcj.attempt_to_stop_test_stream_service(pipeline) cache_manager = self.get_cache_manager(pipeline) if cache_manager: cache_manager.cleanup() else: for _, job in self._background_caching_jobs.items(): if job: job.cancel() for _, controller in self._test_stream_service_controllers.items(): if controller: controller.stop() for _, cache_manager in self._cache_managers.items(): if cache_manager: cache_manager.cleanup() self.evict_recording_manager(pipeline) self.evict_background_caching_job(pipeline) self.evict_test_stream_service_controller(pipeline) self.evict_computed_pcollections(pipeline) self.evict_cached_source_signature(pipeline) self.evict_pipeline_result(pipeline)
def cancel(self): # type: (None) -> None """Cancels the current background recording job.""" bcj.attempt_to_cancel_background_caching_job(self.user_pipeline) for r in self._recordings: r.wait_until_finish() self._recordings = set()
def cleanup_pipeline(self, pipeline): from apache_beam.runners.interactive import background_caching_job as bcj bcj.attempt_to_cancel_background_caching_job(pipeline) bcj.attempt_to_stop_test_stream_service(pipeline) cache_manager = self.get_cache_manager(pipeline) # Recording manager performs cache manager cleanup during eviction, so we # don't need to clean it up here. if cache_manager and self.get_recording_manager(pipeline) is None: cache_manager.cleanup() self.clusters.cleanup(pipeline)
def cancel(self): # type: (None) -> None """Cancels the current background recording job.""" bcj.attempt_to_cancel_background_caching_job(self.user_pipeline) for r in self._recordings: r.wait_until_finish() self._recordings = set() # The recordings rely on a reference to the BCJ to correctly finish. So we # evict the BCJ after they complete. ie.current_env().evict_background_caching_job(self.user_pipeline)
def evict_captured_data(): """Evicts all deterministic replayable data that have been captured by Interactive Beam. In future PCollection evaluation/visualization and pipeline runs, Interactive Beam will capture fresh data.""" if ie.current_env().options.enable_capture_replay: _LOGGER.info( 'You have requested Interactive Beam to evict all captured ' 'data that could be deterministically replayed among multiple ' 'pipeline runs.') ie.current_env().track_user_pipelines() for user_pipeline in ie.current_env().tracked_user_pipelines: bcj.attempt_to_cancel_background_caching_job(user_pipeline) bcj.attempt_to_stop_test_stream_service(user_pipeline) ie.current_env().cleanup()