def ConfigureJob(self, job, args, files_by_type): messages = self.context['dataproc_messages'] hive_job = messages.HiveJob(continueOnFailure=args.continue_on_failure, jarFileUris=files_by_type['jars'], queryFileUri=files_by_type['file']) if args.queries: hive_job.queryList = messages.QueryList(queries=args.queries) if args.params: hive_job.scriptVariables = apitools_base.DictToMessage( args.params, messages.HiveJob.ScriptVariablesValue) if args.properties: hive_job.properties = apitools_base.DictToMessage( args.properties, messages.HiveJob.PropertiesValue) job.hiveJob = hive_job
def ConfigureJob(self, job, args, files_by_type): messages = self.context['dataproc_messages'] log_config = self.BuildLoggingConfiguration(args.driver_log_levels) pig_job = messages.PigJob(continueOnFailure=args.continue_on_failure, jarFileUris=files_by_type['jars'], queryFileUri=files_by_type['file'], loggingConfiguration=log_config) if args.queries: pig_job.queryList = messages.QueryList(queries=args.queries) if args.params: pig_job.scriptVariables = apitools_base.DictToMessage( args.params, messages.PigJob.ScriptVariablesValue) if args.properties: pig_job.properties = apitools_base.DictToMessage( args.properties, messages.PigJob.PropertiesValue) job.pigJob = pig_job
def ConfigureJob(self, job, args, files_by_type): messages = self.context['dataproc_messages'] log_config = self.BuildLoggingConfiguration(args.driver_log_levels) spark_sql_job = messages.SparkSqlJob( jarFileUris=files_by_type['jars'], queryFileUri=files_by_type['file'], scriptVariables=args.params, loggingConfiguration=log_config) if args.queries: spark_sql_job.queryList = messages.QueryList(queries=args.queries) if args.params: spark_sql_job.scriptVariables = apitools_base.DictToMessage( args.params, messages.SparkSqlJob.ScriptVariablesValue) if args.properties: spark_sql_job.properties = apitools_base.DictToMessage( args.properties, messages.SparkSqlJob.PropertiesValue) job.sparkSqlJob = spark_sql_job
def BuildLoggingConfiguration(self, driver_logging): """Build LoggingConfiguration from parameters.""" if not driver_logging: return None messages = self.context['dataproc_messages'] return messages.LoggingConfiguration( driverLogLevels=apitools_base.DictToMessage( driver_logging, messages.LoggingConfiguration.DriverLogLevelsValue))
def ConfigureJob(self, job, args, files_by_type): messages = self.context['dataproc_messages'] log_config = self.BuildLoggingConfiguration(args.driver_log_levels) pyspark_job = messages.PySparkJob( args=args.job_args, archiveUris=files_by_type['archives'], fileUris=files_by_type['files'], pythonFileUris=files_by_type['py_files'], mainPythonFileUri=files_by_type['py_file'], loggingConfiguration=log_config) if args.properties: pyspark_job.properties = apitools_base.DictToMessage( args.properties, messages.PySparkJob.PropertiesValue) job.pysparkJob = pyspark_job
def ConfigureJob(self, job, args, files_by_type): messages = self.context['dataproc_messages'] log_config = self.BuildLoggingConfiguration(args.driver_log_levels) hadoop_job = messages.HadoopJob( args=args.job_args, archiveUris=files_by_type['archives'], fileUris=files_by_type['files'], jarFileUris=files_by_type['jars'], mainClass=args.main_class, mainJarFileUri=files_by_type['main_jar'], loggingConfiguration=log_config) if args.properties: hadoop_job.properties = apitools_base.DictToMessage( args.properties, messages.HadoopJob.PropertiesValue) job.hadoopJob = hadoop_job