def __init__(self,
underlying_runner=None,
cache_dir=None,
cache_format='text',
render_option=None,
skip_display=False):
"""Constructor of InteractiveRunner.
Args:
underlying_runner: (runner.PipelineRunner)
cache_dir: (str) the directory where PCollection caches are kept
cache_format: (str) the file format that should be used for saving
PCollection caches. Available options are 'text' and 'tfrecord'.
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
skip_display: (bool) whether to skip display operations when running the
pipeline. Useful if running large pipelines when display is not
needed.
"""
self._underlying_runner = (underlying_runner
or direct_runner.DirectRunner())
self._cache_manager = cache.FileBasedCacheManager(
cache_dir, cache_format)
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
self._in_session = False
self._skip_display = skip_display
def set_render_option(self, render_option):
"""Sets the rendering option.
Args:
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
"""
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
def set_render_option(self, render_option):
"""Sets the rendering option.
Args:
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
"""
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
def __init__(self, underlying_runner=None, cache_dir=None,
render_option=None):
"""Constructor of InteractiveRunner.
Args:
underlying_runner: (runner.PipelineRunner)
cache_dir: (str) the directory where PCollection caches are kept
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
"""
self._underlying_runner = (underlying_runner
or direct_runner.DirectRunner())
self._cache_manager = cache.FileBasedCacheManager(cache_dir)
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
self._in_session = False
def __init__(self, underlying_runner=None, cache_dir=None,
render_option=None):
"""Constructor of InteractiveRunner.
Args:
underlying_runner: (runner.PipelineRunner)
cache_dir: (str) the directory where PCollection caches are kept
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
"""
self._underlying_runner = (underlying_runner
or direct_runner.DirectRunner())
self._cache_manager = cache.FileBasedCacheManager(cache_dir)
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
self._in_session = False
def __init__(
self,
pipeline, # type: Union[beam_runner_api_pb2.Pipeline, beam.Pipeline]
default_vertex_attrs={'shape': 'box'},
default_edge_attrs=None,
render_option=None):
"""Constructor of PipelineGraph.
Examples:
graph = pipeline_graph.PipelineGraph(pipeline_proto)
graph.get_dot()
or
graph = pipeline_graph.PipelineGraph(pipeline)
graph.get_dot()
Args:
pipeline: (Pipeline proto) or (Pipeline) pipeline to be rendered.
default_vertex_attrs: (Dict[str, str]) a dict of default vertex attributes
default_edge_attrs: (Dict[str, str]) a dict of default edge attributes
render_option: (str) this parameter decides how the pipeline graph is
rendered. See display.pipeline_graph_renderer for available options.
"""
self._lock = threading.Lock()
self._graph = None # type: pydot.Dot
self._pipeline_instrument = None
if isinstance(pipeline, beam.Pipeline):
self._pipeline_instrument = inst.PipelineInstrument(pipeline)
# The pre-process links user pipeline to runner pipeline through analysis
# but without mutating runner pipeline.
self._pipeline_instrument.preprocess()
if isinstance(pipeline, beam_runner_api_pb2.Pipeline):
self._pipeline_proto = pipeline
elif isinstance(pipeline, beam.Pipeline):
self._pipeline_proto = pipeline.to_runner_api()
else:
raise TypeError(
'pipeline should either be a %s or %s, while %s is given' %
(beam_runner_api_pb2.Pipeline, beam.Pipeline, type(pipeline)))
# A dict from PCollection ID to a list of its consuming Transform IDs
self._consumers = collections.defaultdict(
list) # type: DefaultDict[str, List[str]]
# A dict from PCollection ID to its producing Transform ID
self._producers = {} # type: Dict[str, str]
for transform_id, transform_proto in self._top_level_transforms():
for pcoll_id in transform_proto.inputs.values():
self._consumers[pcoll_id].append(transform_id)
for pcoll_id in transform_proto.outputs.values():
self._producers[pcoll_id] = transform_id
default_vertex_attrs = default_vertex_attrs or {'shape': 'box'}
if 'color' not in default_vertex_attrs:
default_vertex_attrs['color'] = 'blue'
if 'fontcolor' not in default_vertex_attrs:
default_vertex_attrs['fontcolor'] = 'blue'
vertex_dict, edge_dict = self._generate_graph_dicts()
self._construct_graph(vertex_dict, edge_dict, default_vertex_attrs,
default_edge_attrs)
self._renderer = pipeline_graph_renderer.get_renderer(render_option)
