Python get_pipeline_for_configuration_and_args示例

示例#1

def main(argv=None):
    config = get_app_config()

    pipeline = get_pipeline_for_configuration_and_args(config, argv=argv)

    args = parse_args(pipeline, config, argv)
    run(args, config=config, pipeline=pipeline)

示例#2

def run(argv=None):
    config = get_app_config()

    pipeline = get_pipeline_for_configuration_and_args(config, argv=argv)

    args = parse_args(pipeline, config, argv)

    # We use the save_main_session option because one or more DoFn's in this
    # workflow rely on global context (e.g., a module imported at module level).
    pipeline_options = PipelineOptions.from_dictionary(vars(args))
    pipeline_options.view_as(SetupOptions).save_main_session = True

    with beam.Pipeline(args.runner, options=pipeline_options) as p:
        configure_pipeline(p, args, pipeline, config)

示例#3

def create_simple_pipeline_runner_from_config(config, args):
    pipeline = get_pipeline_for_configuration_and_args(config, args=args)
    return create_simple_pipeline_runner_from_pipeline(pipeline, config, args)

示例#4

def add_arguments(parser, config, argv=None):
    add_pipeline_args(parser)

    pipeline = get_pipeline_for_configuration_and_args(config, argv=argv)
    pipeline.add_arguments(parser, config, argv=argv)