def main(argv=None): config = get_app_config() pipeline = get_pipeline_for_configuration_and_args(config, argv=argv) args = parse_args(pipeline, config, argv) run(args, config=config, pipeline=pipeline)
def run(argv=None): config = get_app_config() pipeline = get_pipeline_for_configuration_and_args(config, argv=argv) args = parse_args(pipeline, config, argv) # We use the save_main_session option because one or more DoFn's in this # workflow rely on global context (e.g., a module imported at module level). pipeline_options = PipelineOptions.from_dictionary(vars(args)) pipeline_options.view_as(SetupOptions).save_main_session = True with beam.Pipeline(args.runner, options=pipeline_options) as p: configure_pipeline(p, args, pipeline, config)
def create_simple_pipeline_runner_from_config(config, args): pipeline = get_pipeline_for_configuration_and_args(config, args=args) return create_simple_pipeline_runner_from_pipeline(pipeline, config, args)
def add_arguments(parser, config, argv=None): add_pipeline_args(parser) pipeline = get_pipeline_for_configuration_and_args(config, argv=argv) pipeline.add_arguments(parser, config, argv=argv)