def ConfigureJob(self, job, args, files_by_type):
messages = self.context['dataproc_messages']
hive_job = messages.HiveJob(continueOnFailure=args.continue_on_failure,
jarFileUris=files_by_type['jars'],
queryFileUri=files_by_type['file'])
if args.queries:
hive_job.queryList = messages.QueryList(queries=args.queries)
if args.params:
hive_job.scriptVariables = apitools_base.DictToMessage(
args.params, messages.HiveJob.ScriptVariablesValue)
if args.properties:
hive_job.properties = apitools_base.DictToMessage(
args.properties, messages.HiveJob.PropertiesValue)
job.hiveJob = hive_job
def ConfigureJob(self, job, args, files_by_type):
messages = self.context['dataproc_messages']
log_config = self.BuildLoggingConfiguration(args.driver_log_levels)
pig_job = messages.PigJob(continueOnFailure=args.continue_on_failure,
jarFileUris=files_by_type['jars'],
queryFileUri=files_by_type['file'],
loggingConfiguration=log_config)
if args.queries:
pig_job.queryList = messages.QueryList(queries=args.queries)
if args.params:
pig_job.scriptVariables = apitools_base.DictToMessage(
args.params, messages.PigJob.ScriptVariablesValue)
if args.properties:
pig_job.properties = apitools_base.DictToMessage(
args.properties, messages.PigJob.PropertiesValue)
job.pigJob = pig_job
def ConfigureJob(self, job, args, files_by_type):
messages = self.context['dataproc_messages']
log_config = self.BuildLoggingConfiguration(args.driver_log_levels)
spark_sql_job = messages.SparkSqlJob(
jarFileUris=files_by_type['jars'],
queryFileUri=files_by_type['file'],
scriptVariables=args.params,
loggingConfiguration=log_config)
if args.queries:
spark_sql_job.queryList = messages.QueryList(queries=args.queries)
if args.params:
spark_sql_job.scriptVariables = apitools_base.DictToMessage(
args.params, messages.SparkSqlJob.ScriptVariablesValue)
if args.properties:
spark_sql_job.properties = apitools_base.DictToMessage(
args.properties, messages.SparkSqlJob.PropertiesValue)
job.sparkSqlJob = spark_sql_job
def BuildLoggingConfiguration(self, driver_logging):
"""Build LoggingConfiguration from parameters."""
if not driver_logging:
return None
messages = self.context['dataproc_messages']
return messages.LoggingConfiguration(
driverLogLevels=apitools_base.DictToMessage(
driver_logging,
messages.LoggingConfiguration.DriverLogLevelsValue))
def ConfigureJob(self, job, args, files_by_type):
messages = self.context['dataproc_messages']
log_config = self.BuildLoggingConfiguration(args.driver_log_levels)
pyspark_job = messages.PySparkJob(
args=args.job_args,
archiveUris=files_by_type['archives'],
fileUris=files_by_type['files'],
pythonFileUris=files_by_type['py_files'],
mainPythonFileUri=files_by_type['py_file'],
loggingConfiguration=log_config)
if args.properties:
pyspark_job.properties = apitools_base.DictToMessage(
args.properties, messages.PySparkJob.PropertiesValue)
job.pysparkJob = pyspark_job
def ConfigureJob(self, job, args, files_by_type):
messages = self.context['dataproc_messages']
log_config = self.BuildLoggingConfiguration(args.driver_log_levels)
hadoop_job = messages.HadoopJob(
args=args.job_args,
archiveUris=files_by_type['archives'],
fileUris=files_by_type['files'],
jarFileUris=files_by_type['jars'],
mainClass=args.main_class,
mainJarFileUri=files_by_type['main_jar'],
loggingConfiguration=log_config)
if args.properties:
hadoop_job.properties = apitools_base.DictToMessage(
args.properties, messages.HadoopJob.PropertiesValue)
job.hadoopJob = hadoop_job
