def main(argv=None):
config = get_app_config()
pipeline = get_pipeline_for_configuration_and_args(config, argv=argv)
args = parse_args(pipeline, config, argv)
run(args, config=config, pipeline=pipeline)
def run(argv=None):
config = get_app_config()
pipeline = get_pipeline_for_configuration_and_args(config, argv=argv)
args = parse_args(pipeline, config, argv)
# We use the save_main_session option because one or more DoFn's in this
# workflow rely on global context (e.g., a module imported at module level).
pipeline_options = PipelineOptions.from_dictionary(vars(args))
pipeline_options.view_as(SetupOptions).save_main_session = True
with beam.Pipeline(args.runner, options=pipeline_options) as p:
configure_pipeline(p, args, pipeline, config)
def _get_default_config():
config = DEFAULT_CONFIGURATION
app_config = get_app_config()
process_timeout = os.environ.get(
EnvironmentVariables.DOC_CONVERT_PROCESS_TIMEOUT)
if not process_timeout:
process_timeout = app_config.getint(
DOC_CONVERT_SECTION_NAME,
AppConfigOptions.PROCESS_TIMEOUT,
fallback=DEFAULT_DOC_CONVERT_PROCESS_TIMEOUT)
config = {
**config, 'process_timeout':
int(process_timeout),
'stop_listener_on_error':
app_config.getboolean(DOC_CONVERT_SECTION_NAME,
AppConfigOptions.STOP_LISTENER_ON_ERROR),
'enable_debug':
app_config.getboolean(DOC_CONVERT_SECTION_NAME,
AppConfigOptions.ENABLE_DEBUG)
}
return config
def main(argv=None):
config = get_app_config()
args = parse_args(config, argv)
app = create_app(config, args)
app.run(port=args.port, host=args.host, threaded=True)