parser.add_value_provider_argument(
"--user_project_id",
dest="user_project_id",
help=("User's CloudML project id. It is not the project id of the "
"Dataflow job. The logs are sent to user job project in "
"Stackdriver with job id as its label."))
parser.add_value_provider_argument(
"--user_job_id",
dest="user_job_id",
help=
("User's CloudML job id. It is not the job id of the Dataflow job."
" The logs are sent to user job project in Stackdriver with job"
" id as its label."))
if __name__ == "__main__":
logging.getLogger().setLevel(logging.INFO)
dataflow_pipeline_options = PipelineOptions()
logging.info("Dataflow option: %s",
dataflow_pipeline_options.get_all_options())
# Create the pipeline
p = beam.Pipeline(options=dataflow_pipeline_options)
# Create a dict of aggregators.
aggregator_dict = aggregators.CreateAggregatorsDict()
# Actually start the pipeline
result = batch_prediction_pipeline.run(
p, dataflow_pipeline_options.view_as(BatchPredictionOptions),
aggregator_dict)
"--user_job_id",
dest="user_job_id",
help=(
"User's CloudML job id. It is not the job id of the Dataflow job. "
"The logs are sent to user job project with job id as its label."))
known_args, pipeline_args = parser.parse_known_args(sys.argv[1:])
pipeline_options = PipelineOptions(flags=pipeline_args)
return known_args, pipeline_options
if __name__ == "__main__":
logging.getLogger().setLevel(logging.INFO)
dataflow_args, dataflow_pipeline_options = _parse_args()
logging.info("Dataflow option:%s",
dataflow_pipeline_options.get_all_options())
# Create the pipeline
p = beam.Pipeline(options=dataflow_pipeline_options)
# Create a dict of aggregators.
aggregator_dict = aggregators.CreateAggregatorsDict()
# Create a cloud logging client.
cloud_logger = None
if dataflow_args.user_project_id and dataflow_args.user_job_id:
cloud_logger = cloud_logging_client.MLCloudLoggingClient.create(
dataflow_args.user_project_id, dataflow_args.user_job_id, LOG_NAME,
"jsonPayload")
# Actually start the pipeline
batch_prediction_pipeline.run(p, dataflow_args, aggregator_dict,
cloud_logger)