def ValidateFlagsAddedInComposer2(self, args, is_composer_v1, release_track): """Raises InputError if flags from Composer v2 are used when creating v1.""" if args.environment_size and is_composer_v1: raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V1_ERROR_MSG.format( opt='environment-size')) composer_v2_flag_used = (args.scheduler_cpu or args.worker_cpu or args.web_server_cpu or args.scheduler_memory or args.worker_memory or args.web_server_memory or args.scheduler_storage or args.worker_storage or args.web_server_storage or args.min_workers or args.max_workers) if composer_v2_flag_used and is_composer_v1: raise command_util.InvalidUserInputError( 'Workloads Config flags introduced in Composer 2.X' ' cannot be used when creating Composer 1.X environments.') if args.connection_subnetwork and is_composer_v1: raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V1_ERROR_MSG.format( opt='connection-subnetwork')) if args.connection_subnetwork and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='connection-subnetwork'))
def ParseIpAliasConfigOptions(self, args): """Parses the options for VPC-native configuration.""" if args.enable_private_environment and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='enable-private-environment')) if args.cluster_ipv4_cidr and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-ipv4-cidr')) if args.cluster_secondary_range_name and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-secondary-range-name')) if args.services_ipv4_cidr and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-ipv4-cidr')) if args.services_secondary_range_name and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-secondary-range-name'))
def ParsePrivateEnvironmentWebServerCloudSqlRanges(self, args, image_version, release_track): if (args.web_server_ipv4_cidr and not image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format( opt='web-server-ipv4-cidr')) if args.web_server_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='web-server-ipv4-cidr')) if args.cloud_sql_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='cloud-sql-ipv4-cidr')) if (args.composer_network_ipv4_cidr and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V1_ERROR_MSG.format( opt='composer-network-ipv4-cidr')) if (args.composer_network_ipv4_cidr and not args.enable_private_environment): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='composer-network-ipv4-cidr'))
def ParsePrivateEnvironmentConfigOptions(self, args, image_version): """Parses the options for Private Environment configuration.""" if (args.enable_private_environment and not args.enable_ip_alias and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='enable-private-environment')) if args.enable_private_endpoint and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='enable-private-endpoint')) if (args.enable_privately_used_public_ips and not args.enable_private_environment): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='enable-privately-used-public-ips')) if args.master_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='master-ipv4-cidr'))
def ParseIpAliasConfigOptions(self, args): """Parses the options for VPC-native configuration.""" if args.cluster_ipv4_cidr and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-ipv4-cidr')) if args.cluster_secondary_range_name and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-secondary-range-name')) if args.services_ipv4_cidr and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-ipv4-cidr')) if args.services_secondary_range_name and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-secondary-range-name')) if self._support_max_pods_per_node and args.max_pods_per_node \ and not args.enable_ip_alias: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='max-pods-per-node'))
def ParsePrivateEnvironmentWebServerCloudSqlRanges(self, args): if args.web_server_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='web-server-ipv4-cidr')) if args.cloud_sql_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='cloud-sql-ipv4-cidr'))
def ValidateComposer1ExclusiveFlags(self, args, is_composer_v1, release_track): """Raises InputError if flags from Composer v2 are used when creating v1.""" # Composer 2 flags are currently unavailable in GA release track. if args.python_version and not is_composer_v1: raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format(opt='python-version')) if args.disk_size and not is_composer_v1: raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format(opt='disk-size')) if args.machine_type and not is_composer_v1: raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format(opt='machine-type'))
def ParsePrivateEnvironmentConfigOptions(self, args): """Parses the options for Private Environment configuration.""" if args.enable_private_endpoint and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='enable-private-endpoint')) if args.main_ipv4_cidr and not args.enable_private_environment: raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-private-environment', opt='main-ipv4-cidr'))
def Run(self, args): env_ref = args.CONCEPTS.environment.Parse() if args.airflow_version: # Converts airflow_version arg to image_version arg args.image_version = ( image_versions_command_util.ImageVersionFromAirflowVersion( args.airflow_version)) # Checks validity of image_version upgrade request. if (args.image_version and not image_versions_command_util.IsValidImageVersionUpgrade( env_ref, args.image_version, self.ReleaseTrack())): raise command_util.InvalidUserInputError( 'Invalid environment upgrade. [Requested: {}]'.format( args.image_version)) # Checks validity of update_web_server_allow_ip if (self.ReleaseTrack() == base.ReleaseTrack.BETA and args.update_web_server_allow_ip): flags.ValidateIpRanges( [acl['ip_range'] for acl in args.update_web_server_allow_ip]) field_mask, patch = self._ConstructPatch( env_ref, args, UpdateBeta.support_environment_upgrades, UpdateBeta.support_web_server_access_control, True, True) return patch_util.Patch( env_ref, field_mask, patch, args.async_, release_track=self.ReleaseTrack())
def Run(self, args): env_ref = args.CONCEPTS.environment.Parse() if args.airflow_version: # Converts airflow_version arg to image_version arg args.image_version = ( image_versions_command_util.ImageVersionFromAirflowVersion( args.airflow_version)) # Checks validity of image_version upgrade request. if (args.image_version and not image_versions_command_util.IsValidImageVersionUpgrade( env_ref, args.image_version, self.ReleaseTrack())): raise command_util.InvalidUserInputError( 'Invalid environment upgrade. [Requested: {}]'.format( args.image_version)) field_mask, patch = self._ConstructPatch( env_ref, args, UpdateBeta.support_environment_upgrades) return patch_util.Patch(env_ref, field_mask, patch, args.async_, release_track=self.ReleaseTrack())
def Run(self, args): self.ParseIpAliasConfigOptions(args) self.ParsePrivateEnvironmentConfigOptions(args) if self._support_web_server_cloud_sql_private_ip_ranges: self.ParsePrivateEnvironmentWebServerCloudSqlRanges(args) flags.ValidateDiskSize('--disk-size', args.disk_size) self.env_ref = args.CONCEPTS.environment.Parse() env_name = self.env_ref.Name() if not command_util.IsValidEnvironmentName(env_name): raise command_util.InvalidUserInputError( 'Invalid environment name: [{}]. Must match pattern: {}'. format(env_name, command_util.ENVIRONMENT_NAME_PATTERN.pattern)) self.zone_ref = parsers.ParseZone(args.zone) if args.zone else None self.zone = self.zone_ref.RelativeName() if self.zone_ref else None self.machine_type = None self.network = None self.subnetwork = None if args.machine_type: self.machine_type = parsers.ParseMachineType( args.machine_type, fallback_zone=self.zone_ref.Name() if self.zone_ref else None).RelativeName() if args.network: self.network = parsers.ParseNetwork(args.network).RelativeName() if args.subnetwork: self.subnetwork = parsers.ParseSubnetwork( args.subnetwork, fallback_region=self.env_ref.Parent().Name()).RelativeName() self.image_version = None if args.airflow_version: self.image_version = image_versions_util.ImageVersionFromAirflowVersion( args.airflow_version) elif args.image_version: self.image_version = args.image_version operation = self.GetOperationMessage(args) details = 'with operation [{0}]'.format(operation.name) if args.async_: log.CreatedResource(self.env_ref.RelativeName(), kind='environment', is_async=True, details=details) return operation else: try: operations_api_util.WaitForOperation( operation, 'Waiting for [{}] to be created with [{}]'.format( self.env_ref.RelativeName(), operation.name), release_track=self.ReleaseTrack()) except command_util.OperationError as e: raise command_util.EnvironmentCreateError( 'Error creating [{}]: {}'.format( self.env_ref.RelativeName(), six.text_type(e)))
def Run(self, args): flags.ValidateDiskSize('--disk-size', args.disk_size) env_ref = args.CONCEPTS.environment.Parse() env_name = env_ref.Name() if not command_util.IsValidEnvironmentName(env_name): raise command_util.InvalidUserInputError( 'Invalid environment name: [{}]. Must match pattern: {}'. format(env_name, command_util.ENVIRONMENT_NAME_PATTERN.pattern)) zone_ref = parsers.ParseZone(args.zone) if args.zone else None zone = zone_ref.RelativeName() if zone_ref else None machine_type = None network = None subnetwork = None if args.machine_type: machine_type = parsers.ParseMachineType( args.machine_type, fallback_zone=zone_ref.Name() if zone_ref else None).RelativeName() if args.network: network = parsers.ParseNetwork(args.network).RelativeName() if args.subnetwork: subnetwork = parsers.ParseSubnetwork( args.subnetwork, fallback_region=env_ref.Parent().Name()).RelativeName() operation = environments_api_util.Create( env_ref, args.node_count, labels=args.labels, location=zone, machine_type=machine_type, network=network, subnetwork=subnetwork, env_variables=args.env_variables, airflow_config_overrides=args.airflow_configs, service_account=args.service_account, oauth_scopes=args.oauth_scopes, tags=args.tags, disk_size_gb=args.disk_size >> 30, release_track=self.ReleaseTrack()) details = 'with operation [{0}]'.format(operation.name) if args. async: log.CreatedResource(env_ref.RelativeName(), kind='environment', is_async=True, details=details) return operation else: try: operations_api_util.WaitForOperation( operation, 'Waiting for [{}] to be created with [{}]'.format( env_ref.RelativeName(), operation.name), release_track=self.ReleaseTrack()) except command_util.OperationError as e: raise command_util.EnvironmentCreateError( 'Error creating [{}]: {}'.format(env_ref.RelativeName(), str(e)))
def ParseMasterAuthorizedNetworksConfigOptions(self, args, release_track): if release_track == base.ReleaseTrack.GA: return if args.enable_master_authorized_networks: self.enable_master_authorized_networks = args.enable_master_authorized_networks elif args.master_authorized_networks: raise command_util.InvalidUserInputError( 'Cannot specify --master-authorized-networks without ' + '--enable-master-authorized-networks.') command_util.ValidateMasterAuthorizedNetworks( args.master_authorized_networks) self.master_authorized_networks = args.master_authorized_networks
def ParseWebServerAccessControlConfigOptions(self, args): if (args.enable_private_environment and not args.web_server_allow_ip and not args.web_server_allow_all and not args.web_server_deny_all): raise command_util.InvalidUserInputError( 'Cannot specify --enable-private-environment without one of: ' + '--web-server-allow-ip, --web-server-allow-all ' + 'or --web-server-deny-all') # Default to allow all if no flag is specified. self.web_server_access_control = ( environments_api_util.BuildWebServerAllowedIps( args.web_server_allow_ip, args.web_server_allow_all or not args.web_server_allow_ip, args.web_server_deny_all)) flags.ValidateIpRanges( [acl['ip_range'] for acl in self.web_server_access_control])
def ValidateIpRanges(ip_ranges): """Validates list of IP ranges. Raises exception when any of the given strings is not a valid IPv4 or IPv6 network IP range. Args: ip_ranges: [string], list of IP ranges to validate """ for ip_range in ip_ranges: if six.PY2: ip_range = ip_range.decode() try: ipaddress.ip_network(ip_range) except: raise command_util.InvalidUserInputError( 'Invalid IP range: [{}].'.format(ip_range))
def Run(self, args): flags.ValidateDiskSize('--disk-size', args.disk_size) self.env_ref = args.CONCEPTS.environment.Parse() env_name = self.env_ref.Name() if not command_util.IsValidEnvironmentName(env_name): raise command_util.InvalidUserInputError( 'Invalid environment name: [{}]. Must match pattern: {}'. format(env_name, command_util.ENVIRONMENT_NAME_PATTERN.pattern)) self.zone_ref = parsers.ParseZone(args.zone) if args.zone else None self.zone = self.zone_ref.RelativeName() if self.zone_ref else None self.machine_type = None self.network = None self.subnetwork = None if args.machine_type: self.machine_type = parsers.ParseMachineType( args.machine_type, fallback_zone=self.zone_ref.Name() if self.zone_ref else None).RelativeName() if args.network: self.network = parsers.ParseNetwork(args.network).RelativeName() if args.subnetwork: self.subnetwork = parsers.ParseSubnetwork( args.subnetwork, fallback_region=self.env_ref.Parent().Name()).RelativeName() operation = self.GetOperationMessage(args) details = 'with operation [{0}]'.format(operation.name) if args. async: log.CreatedResource(self.env_ref.RelativeName(), kind='environment', is_async=True, details=details) return operation else: try: operations_api_util.WaitForOperation( operation, 'Waiting for [{}] to be created with [{}]'.format( self.env_ref.RelativeName(), operation.name), release_track=self.ReleaseTrack()) except command_util.OperationError as e: raise command_util.EnvironmentCreateError( 'Error creating [{}]: {}'.format( self.env_ref.RelativeName(), str(e)))
def ParseIpAliasConfigOptions(self, args, image_version): """Parses the options for VPC-native configuration.""" if (args.enable_ip_alias and not image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format(opt='enable-ip-alias')) if (args.cluster_ipv4_cidr and not args.enable_ip_alias and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-ipv4-cidr')) if (args.cluster_secondary_range_name and not args.enable_ip_alias and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='cluster-secondary-range-name')) if (args.services_ipv4_cidr and not args.enable_ip_alias and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-ipv4-cidr')) if (args.services_secondary_range_name and not args.enable_ip_alias and image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='services-secondary-range-name')) if (self._support_max_pods_per_node and args.max_pods_per_node and not image_versions_util.IsImageVersionStringComposerV1( image_version)): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format( opt='max-pods-per-node')) if (self._support_max_pods_per_node and args.max_pods_per_node and not args.enable_ip_alias): raise command_util.InvalidUserInputError( PREREQUISITE_OPTION_ERROR_MSG.format( prerequisite='enable-ip-alias', opt='max-pods-per-node'))
def _ConstructPatch(self, env_ref, args, support_environment_upgrades=False): env_obj = environments_api_util.Get(env_ref, release_track=self.ReleaseTrack()) is_composer_v1 = image_versions_command_util.IsImageVersionStringComposerV1( env_obj.config.softwareConfig.imageVersion) params = dict( is_composer_v1=is_composer_v1, env_ref=env_ref, node_count=args.node_count, update_pypi_packages_from_file=args.update_pypi_packages_from_file, clear_pypi_packages=args.clear_pypi_packages, remove_pypi_packages=args.remove_pypi_packages, update_pypi_packages=dict( command_util.SplitRequirementSpecifier(r) for r in args.update_pypi_package), clear_labels=args.clear_labels, remove_labels=args.remove_labels, update_labels=args.update_labels, clear_airflow_configs=args.clear_airflow_configs, remove_airflow_configs=args.remove_airflow_configs, update_airflow_configs=args.update_airflow_configs, clear_env_variables=args.clear_env_variables, remove_env_variables=args.remove_env_variables, update_env_variables=args.update_env_variables, release_track=self.ReleaseTrack()) if support_environment_upgrades: params['update_image_version'] = args.image_version params['update_web_server_access_control'] = ( environments_api_util.BuildWebServerAllowedIps( args.update_web_server_allow_ip, args.web_server_allow_all, args.web_server_deny_all)) if (args.cloud_sql_machine_type and not is_composer_v1): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format( opt='cloud-sql-machine-type')) if (args.web_server_machine_type and not is_composer_v1): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V2_ERROR_MSG.format( opt='web-server-machine-type')) params['cloud_sql_machine_type'] = args.cloud_sql_machine_type params['web_server_machine_type'] = args.web_server_machine_type params['scheduler_count'] = args.scheduler_count if self._support_environment_size: if (args.environment_size and is_composer_v1): raise command_util.InvalidUserInputError( _INVALID_OPTION_FOR_V1_ERROR_MSG.format( opt='environment-size')) if self.ReleaseTrack() == base.ReleaseTrack.GA: params[ 'environment_size'] = flags.ENVIRONMENT_SIZE_GA.GetEnumForChoice( args.environment_size) elif self.ReleaseTrack() == base.ReleaseTrack.BETA: params[ 'environment_size'] = flags.ENVIRONMENT_SIZE_BETA.GetEnumForChoice( args.environment_size) elif self.ReleaseTrack() == base.ReleaseTrack.ALPHA: params[ 'environment_size'] = flags.ENVIRONMENT_SIZE_ALPHA.GetEnumForChoice( args.environment_size) if self._support_autoscaling: if (is_composer_v1 and (args.scheduler_cpu or args.worker_cpu or args.web_server_cpu or args.scheduler_memory or args.worker_memory or args.web_server_memory or args.scheduler_storage or args.worker_storage or args.web_server_storage or args.min_workers or args.max_workers)): raise command_util.InvalidUserInputError( 'Workloads Config flags introduced in Composer 2.X' ' cannot be used when creating Composer 1.X environments.') params['scheduler_cpu'] = args.scheduler_cpu params['worker_cpu'] = args.worker_cpu params['web_server_cpu'] = args.web_server_cpu params[ 'scheduler_memory_gb'] = environments_api_util.MemorySizeBytesToGB( args.scheduler_memory) params[ 'worker_memory_gb'] = environments_api_util.MemorySizeBytesToGB( args.worker_memory) params[ 'web_server_memory_gb'] = environments_api_util.MemorySizeBytesToGB( args.web_server_memory) params[ 'scheduler_storage_gb'] = environments_api_util.MemorySizeBytesToGB( args.scheduler_storage) params[ 'worker_storage_gb'] = environments_api_util.MemorySizeBytesToGB( args.worker_storage) params[ 'web_server_storage_gb'] = environments_api_util.MemorySizeBytesToGB( args.web_server_storage) params['min_workers'] = args.min_workers params['max_workers'] = args.max_workers if self._support_maintenance_window: params['maintenance_window_start'] = args.maintenance_window_start params['maintenance_window_end'] = args.maintenance_window_end params[ 'maintenance_window_recurrence'] = args.maintenance_window_recurrence if self.ReleaseTrack() != base.ReleaseTrack.GA: if args.enable_master_authorized_networks and args.disable_master_authorized_networks: raise command_util.InvalidUserInputError( 'Cannot specify --enable-master-authorized-networks with --disable-master-authorized-networks' ) if args.disable_master_authorized_networks and args.master_authorized_networks: raise command_util.InvalidUserInputError( 'Cannot specify --disable-master-authorized-networks with --master-authorized-networks' ) if args.enable_master_authorized_networks is None and args.master_authorized_networks: raise command_util.InvalidUserInputError( 'Cannot specify --master-authorized-networks without --enable-master-authorized-networks' ) if args.enable_master_authorized_networks or args.disable_master_authorized_networks: params[ 'master_authorized_networks_enabled'] = True if args.enable_master_authorized_networks else False command_util.ValidateMasterAuthorizedNetworks( args.master_authorized_networks) params[ 'master_authorized_networks'] = args.master_authorized_networks return patch_util.ConstructPatch(**params)