def Run(self, args): region = dataflow_util.GetRegion(args) if args.sql_launcher_template: gcs_location = args.sql_launcher_template else: gcs_location = 'gs://dataflow-sql-templates-{}/latest/sql_launcher_template'.format( region) if args.parameters_file: query_parameters = sql_query_parameters.ParseParametersFile( args.parameters_file) elif args.parameter: query_parameters = sql_query_parameters.ParseParametersList( args.parameter) else: query_parameters = '[]' template_parameters = { 'dryRun': 'true' if args.dry_run else 'false', 'outputs': sql_util.ExtractOutputs(args), 'queryParameters': query_parameters, 'queryString': args.query, } arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.GetOrFail(), region_id=region, job_name=args.job_name, gcs_location=gcs_location, zone=args.worker_zone, max_workers=args.max_workers, disable_public_ips=properties.VALUES.dataflow.disable_public_ips. GetBool(), parameters=template_parameters, service_account_email=args.service_account_email) return apis.Templates.LaunchDynamicTemplate(arguments)
def _CommonRun(args): """Runs the command. Args: args: The arguments that were provided to this command invocation. Returns: A Job message. """ arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.Get(required=True), region_id=dataflow_util.GetRegion(args), job_name=args.job_name, gcs_location=args.gcs_location, zone=args.zone, max_workers=args.max_workers, num_workers=args.num_workers, network=args.network, subnetwork=args.subnetwork, worker_machine_type=args.worker_machine_type, staging_location=args.staging_location, kms_key_name=args.dataflow_kms_key, disable_public_ips=properties.VALUES.dataflow.disable_public_ips. GetBool(), parameters=args.parameters, service_account_email=args.service_account_email, worker_region=args.worker_region, worker_zone=args.worker_zone) return apis.Templates.Create(arguments)
def Run(self, args): """Runs the command. Args: args: The arguments that were provided to this command invocation. Returns: A Job message. """ arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.Get(required=True), region_id=dataflow_util.GetRegion(args), job_name=args.job_name, gcs_location=args.template_file_gcs_location, max_workers=args.max_workers, num_workers=args.num_workers, network=args.network, subnetwork=args.subnetwork, worker_machine_type=args.worker_machine_type, kms_key_name=args.dataflow_kms_key, staging_location=args.staging_location, disable_public_ips=properties.VALUES.dataflow.disable_public_ips. GetBool(), service_account_email=args.service_account_email, worker_region=args.worker_region, worker_zone=args.worker_zone, enable_streaming_engine=properties.VALUES.dataflow. enable_streaming_engine.GetBool(), additional_experiments=args.additional_experiments, additional_user_labels=args.additional_user_labels, parameters=args.parameters) return apis.Templates.CreateJobFromFlexTemplate(arguments)
def Run(self, args): """Runs the command. Args: args: The arguments that were provided to this command invocation. Returns: A Snapshot message. """ return apis.Snapshots.List( job_id=args.job_id, project_id=properties.VALUES.core.project.GetOrFail(), region_id=dataflow_util.GetRegion(args))
def _CommonRun(args): """Runs the command. Args: args: The arguments that were provided to this command invocation. Returns: A Job message. """ arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.Get(required=True), region_id=dataflow_util.GetRegion(args), job_name=args.job_name, gcs_location=args.template_file_gcs_location, parameters=args.parameters) return apis.Templates.CreateJobFromFlexTemplate(arguments)
def Run(self, args): use_dynamic_engine = (args.sql_launcher_template_engine == 'dynamic') region = dataflow_util.GetRegion(args) if args.sql_launcher_template: gcs_location = args.sql_launcher_template else: if use_dynamic_engine: suffix = 'sql_launcher_template' else: suffix = 'sql_launcher_flex_template' gcs_location = 'gs://dataflow-sql-templates-{}/latest/{}'.format( region, suffix) if args.parameters_file: query_parameters = sql_query_parameters.ParseParametersFile( args.parameters_file) elif args.parameter: query_parameters = sql_query_parameters.ParseParametersList( args.parameter) else: query_parameters = '[]' template_parameters = collections.OrderedDict([ ('dryRun', 'true' if args.dry_run else 'false'), ('outputs', sql_util.ExtractOutputs(args)), ('queryParameters', query_parameters), ('queryString', args.query), ]) arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.GetOrFail(), region_id=region, job_name=args.job_name, gcs_location=gcs_location, zone=args.zone, max_workers=args.max_workers, disable_public_ips=properties.VALUES.dataflow.disable_public_ips. GetBool(), parameters=template_parameters, service_account_email=args.service_account_email, kms_key_name=args.dataflow_kms_key, num_workers=args.num_workers, network=args.network, subnetwork=args.subnetwork, worker_machine_type=args.worker_machine_type, worker_region=args.worker_region, worker_zone=args.worker_zone) if use_dynamic_engine: return apis.Templates.LaunchDynamicTemplate(arguments) return apis.Templates.CreateJobFromFlexTemplate(arguments)
def ExtractJobRef(args): """Extract the Job Ref for a command. Used with ArgsForJobRef. Args: args: The command line arguments. Returns: A Job resource. """ job = args.job region = dataflow_util.GetRegion(args) return resources.REGISTRY.Parse( job, params={ 'projectId': properties.VALUES.core.project.GetOrFail, 'location': region }, collection='dataflow.projects.locations.jobs')
def _CommonRun(args, support_beta_features=False): """Runs the command. Args: args: The arguments that were provided to this command invocation. support_beta_features: True, if using the beta command. Returns: A Job message. """ num_workers = None worker_machine_type = None network = None subnetwork = None dataflow_kms_key = None if support_beta_features: num_workers = args.num_workers worker_machine_type = args.worker_machine_type network = args.network subnetwork = args.subnetwork dataflow_kms_key = args.dataflow_kms_key arguments = apis.TemplateArguments( project_id=properties.VALUES.core.project.Get(required=True), region_id=dataflow_util.GetRegion(args), job_name=args.job_name, gcs_location=args.gcs_location, zone=args.zone, max_workers=args.max_workers, num_workers=num_workers, network=network, subnetwork=subnetwork, worker_machine_type=worker_machine_type, staging_location=args.staging_location, kms_key_name=dataflow_kms_key, disable_public_ips=properties.VALUES.dataflow.disable_public_ips. GetBool(), parameters=args.parameters, service_account_email=args.service_account_email) flex_template = properties.VALUES.dataflow.flex_template.GetBool() if flex_template: return apis.Templates.CreateJobFromFlexTemplate(arguments) else: return apis.Templates.Create(arguments)
def ExtractJobRefs(args): """Extract the Job Refs for a command. Used with ArgsForJobRefs. Args: args: The command line arguments that were provided to this invocation. Returns: A list of job resources. """ jobs = args.jobs region = dataflow_util.GetRegion(args) return [ resources.REGISTRY.Parse( job, params={ 'projectId': properties.VALUES.core.project.GetOrFail, 'location': region }, collection='dataflow.projects.locations.jobs') for job in jobs ]
def _CommonRun(args, support_beta_features=False): """Runs the command. Args: args: The arguments that were provided to this command invocation. support_beta_features: True, if using the beta command. Returns: A Job message. """ num_workers = None worker_machine_type = None network = None subnetwork = None dataflow_kms_key = None if support_beta_features: num_workers = args.num_workers worker_machine_type = args.worker_machine_type network = args.network subnetwork = args.subnetwork dataflow_kms_key = args.dataflow_kms_key job = apis.Templates.Create( project_id=properties.VALUES.core.project.Get(required=True), region_id=dataflow_util.GetRegion(args), gcs_location=args.gcs_location, staging_location=args.staging_location, job_name=args.job_name, parameters=args.parameters, service_account_email=args.service_account_email, zone=args.zone, max_workers=args.max_workers, num_workers=num_workers, worker_machine_type=worker_machine_type, network=network, subnetwork=subnetwork, dataflow_kms_key=dataflow_kms_key) return job
def testGetRegionEmpty(self): args = self.parser.parse_args(['--region=']) self.assertEqual('us-central1', dataflow_util.GetRegion(args))