def Args(parser): SnapshotDisks.disks_arg = disks_flags.MakeDiskArgZonalOrRegional( plural=True) labels_util.AddCreateLabelsFlags(parser) flags.AddStorageLocationFlag(parser, 'snapshot') _CommonArgs(parser)
def Args(parser): resource_args.AddEnvironmentResourceArg(parser, 'to create') base.ASYNC_FLAG.AddToParser(parser) parser.add_argument( '--node-count', type=int, help='The number of nodes to create to run the environment.') parser.add_argument( '--zone', help='The Compute Engine zone in which the environment will ' 'be created. For example `--zone=us-central1-a`.') parser.add_argument( '--machine-type', help='The Compute Engine machine type ' '(https://cloud.google.com/compute/docs/machine-types) to use for ' 'nodes. For example `--machine-type=n1-standard-1`.') parser.add_argument( '--disk-size', default='100GB', type=arg_parsers.BinarySize( lower_bound='20GB', upper_bound='64TB', suggested_binary_size_scales=['GB', 'TB']), help= 'The disk size for each VM node in the environment. The minimum ' 'size is 20GB, and the maximum is 64TB. Specified value must be an ' 'integer multiple of gigabytes. Cannot be updated after the ' 'environment has been created. If units are not provided, defaults to ' 'GB.') networking_group = parser.add_group( help='Virtual Private Cloud networking') networking_group.add_argument( '--network', required=True, help='The Compute Engine Network to which the environment will ' 'be connected. If a \'Custom Subnet Network\' is provided, ' '`--subnetwork` must be specified as well.') networking_group.add_argument( '--subnetwork', help='The Compute Engine subnetwork ' '(https://cloud.google.com/compute/docs/subnetworks) to which the ' 'environment will be connected.') labels_util.AddCreateLabelsFlags(parser) flags.CREATE_ENV_VARS_FLAG.AddToParser(parser) # Default is provided by API server. parser.add_argument( '--service-account', help= 'The Google Cloud Platform service account to be used by the node ' 'VMs. If a service account is not specified, the "default" Compute ' 'Engine service account for the project is used. Cannot be updated.' ) # Default is provided by API server. parser.add_argument( '--oauth-scopes', help= 'The set of Google API scopes to be made available on all of the ' 'node VMs. Defaults to ' '[\'https://www.googleapis.com/auth/cloud-platform\']. Cannot be ' 'updated.', type=arg_parsers.ArgList(), metavar='SCOPE', action=arg_parsers.UpdateAction) parser.add_argument( '--tags', help= 'The set of instance tags applied to all node VMs. Tags are used ' 'to identify valid sources or targets for network firewalls. Each tag ' 'within the list must comply with RFC 1035. Cannot be updated.', type=arg_parsers.ArgList(), metavar='TAG', action=arg_parsers.UpdateAction) # API server will validate key/value pairs. parser.add_argument('--airflow-configs', help="""\ A list of Airflow software configuration override KEY=VALUE pairs to set. For information on how to structure KEYs and VALUEs, run `$ {top_command} help composer environments update`.""", type=arg_parsers.ArgDict(), metavar='KEY=VALUE', action=arg_parsers.UpdateAction)
def _CommonArgs(parser, release_track, support_source_instance, support_local_ssd_size=False, support_kms=False, support_multi_writer=False, support_mesh=False, support_host_error_timeout_seconds=False, support_numa_node_count=False, support_visible_core_count=False, support_disk_architecture=False, support_max_run_duration=False): """Adding arguments applicable for creating instance templates.""" parser.display_info.AddFormat(instance_templates_flags.DEFAULT_LIST_FORMAT) metadata_utils.AddMetadataArgs(parser) instances_flags.AddDiskArgs(parser, enable_kms=support_kms) instances_flags.AddCreateDiskArgs( parser, enable_kms=support_kms, support_boot=True, support_multi_writer=support_multi_writer, support_disk_architecture=support_disk_architecture) if support_local_ssd_size: instances_flags.AddLocalSsdArgsWithSize(parser) else: instances_flags.AddLocalSsdArgs(parser) instances_flags.AddCanIpForwardArgs(parser) instances_flags.AddAddressArgs(parser, instances=False) instances_flags.AddAcceleratorArgs(parser) instances_flags.AddMachineTypeArgs(parser) deprecate_maintenance_policy = release_track in [base.ReleaseTrack.ALPHA] instances_flags.AddMaintenancePolicyArgs(parser, deprecate_maintenance_policy) instances_flags.AddNoRestartOnFailureArgs(parser) instances_flags.AddPreemptibleVmArgs(parser) instances_flags.AddServiceAccountAndScopeArgs(parser, False) instances_flags.AddTagsArgs(parser) instances_flags.AddCustomMachineTypeArgs(parser) instances_flags.AddImageArgs(parser) instances_flags.AddNetworkArgs(parser) instances_flags.AddShieldedInstanceConfigArgs(parser) labels_util.AddCreateLabelsFlags(parser) instances_flags.AddNetworkTierArgs(parser, instance=True) instances_flags.AddPrivateNetworkIpArgs(parser) instances_flags.AddMinNodeCpuArg(parser) instances_flags.AddNestedVirtualizationArgs(parser) instances_flags.AddThreadsPerCoreArgs(parser) instances_flags.AddEnableUefiNetworkingArgs(parser) instances_flags.AddResourceManagerTagsArgs(parser) if support_numa_node_count: instances_flags.AddNumaNodeCountArgs(parser) instances_flags.AddStackTypeArgs(parser) instances_flags.AddIpv6NetworkTierArgs(parser) maintenance_flags.AddResourcePoliciesArgs(parser, 'added to', 'instance-template') instances_flags.AddProvisioningModelVmArgs(parser) instances_flags.AddInstanceTerminationActionVmArgs(parser) if support_max_run_duration: instances_flags.AddMaxRunDurationVmArgs(parser) instance_templates_flags.AddServiceProxyConfigArgs( parser, release_track=release_track) if support_mesh: instance_templates_flags.AddMeshArgs(parser) sole_tenancy_flags.AddNodeAffinityFlagToParser(parser) instances_flags.AddLocationHintArg(parser) if support_visible_core_count: instances_flags.AddVisibleCoreCountArgs(parser) instances_flags.AddNetworkPerformanceConfigsArgs(parser) flags.AddRegionFlag( parser, resource_type='subnetwork', operation_type='attach') parser.add_argument( '--description', help='Specifies a textual description for the instance template.') Create.InstanceTemplateArg = ( instance_templates_flags.MakeInstanceTemplateArg()) Create.InstanceTemplateArg.AddArgument(parser, operation_type='create') if support_source_instance: instance_templates_flags.MakeSourceInstanceArg().AddArgument(parser) parser.add_argument( '--configure-disk', type=arg_parsers.ArgDict( spec={ 'auto-delete': arg_parsers.ArgBoolean(), 'device-name': str, 'instantiate-from': str, 'custom-image': str, },), metavar='PROPERTY=VALUE', action='append', help="""\ This option has effect only when used with `--source-instance`. It allows you to override how the source-instance's disks are defined in the template. *auto-delete*::: If `true`, this persistent disk will be automatically deleted when the instance is deleted. However, if the disk is later detached from the instance, this option won't apply. If not provided, the setting is copied from the source instance. Allowed values of the flag are: `false`, `no`, `true`, and `yes`. *device-name*::: Name of the device. *instantiate-from*::: Specifies whether to include the disk and which image to use. Valid values are: {} *custom-image*::: The custom image to use if custom-image is specified for instantiate-from. """.format(', '.join(_INSTANTIATE_FROM_VALUES)), ) instances_flags.AddReservationAffinityGroup( parser, group_text="""\ Specifies the reservation for instances created from this template. """, affinity_text="""\ The type of reservation for instances created from this template. """) parser.display_info.AddCacheUpdater(completers.InstanceTemplatesCompleter) if support_host_error_timeout_seconds: instances_flags.AddHostErrorTimeoutSecondsArgs(parser)
def _AddCreateArgs(parser): """Add common arguments for `versions create` command.""" flags.GetModelName(positional=False, required=True).AddToParser(parser) flags.GetDescriptionFlag('version').AddToParser(parser) flags.GetRegionArg().AddToParser(parser) flags.VERSION_NAME.AddToParser(parser) base.Argument( '--origin', help="""\ Location of ```model/``` "directory" (see https://cloud.google.com/ai-platform/prediction/docs/deploying-models#upload-model). This overrides `deploymentUri` in the `--config` file. If this flag is not passed, `deploymentUri` *must* be specified in the file from `--config`. Can be a Cloud Storage (`gs://`) path or local file path (no prefix). In the latter case the files will be uploaded to Cloud Storage and a `--staging-bucket` argument is required. """).AddToParser(parser) flags.RUNTIME_VERSION.AddToParser(parser) base.ASYNC_FLAG.AddToParser(parser) flags.STAGING_BUCKET.AddToParser(parser) base.Argument( '--config', help="""\ Path to a YAML configuration file containing configuration parameters for the [Version](https://cloud.google.com/ai-platform/prediction/docs/reference/rest/v1/projects.models.versions) to create. The file is in YAML format. Note that not all attributes of a version are configurable; available attributes (with example values) are: description: A free-form description of the version. deploymentUri: gs://path/to/source runtimeVersion: '2.1' # Set only one of either manualScaling or autoScaling. manualScaling: nodes: 10 # The number of nodes to allocate for this model. autoScaling: minNodes: 0 # The minimum number of nodes to allocate for this model. labels: user-defined-key: user-defined-value The name of the version must always be specified via the required VERSION argument. Only one of manualScaling or autoScaling can be specified. If both are specified in same yaml file an error will be returned. If an option is specified both in the configuration file and via command-line arguments, the command-line arguments override the configuration file. """ ).AddToParser(parser) labels_util.AddCreateLabelsFlags(parser) flags.FRAMEWORK_MAPPER.choice_arg.AddToParser(parser) flags.AddPythonVersionFlag(parser, 'when creating the version') flags.AddMachineTypeFlagToParser(parser) flags.GetAcceleratorFlag().AddToParser(parser)
def AddBetaArgs(parser): """Declare beta flag and positional arguments for this command parser.""" flags.AddExternalMasterGroup(parser) flags.AddInstanceResizeLimit(parser) flags.AddNetwork(parser) labels_util.AddCreateLabelsFlags(parser)
def Args(cls, parser): dataproc = dp.Dataproc(cls.ReleaseTrack()) labels_util.AddCreateLabelsFlags(parser) flags.AddTemplateResourceArg(parser, 'create', dataproc.api_version)
def Args(parser): SnapshotDisks.disks_arg = disks_flags.MakeDiskArg(plural=True) labels_util.AddCreateLabelsFlags(parser) _CommonArgs(parser)
def _CommonArgs(parser, release_track, support_source_instance, support_create_disk=False, support_network_tier=False, support_local_ssd_size=False, support_shielded_vms=False, support_sole_tenancy=False): """Adding arguments applicable for creating instance templates.""" parser.display_info.AddFormat(instance_templates_flags.DEFAULT_LIST_FORMAT) metadata_utils.AddMetadataArgs(parser) instances_flags.AddDiskArgs(parser) if support_create_disk: instances_flags.AddCreateDiskArgs(parser) if support_local_ssd_size: instances_flags.AddLocalSsdArgsWithSize(parser) else: instances_flags.AddLocalSsdArgs(parser) instances_flags.AddCanIpForwardArgs(parser) instances_flags.AddAddressArgs(parser, instances=False, support_network_tier=support_network_tier) instances_flags.AddAcceleratorArgs(parser) instances_flags.AddMachineTypeArgs(parser) deprecate_maintenance_policy = release_track in [base.ReleaseTrack.ALPHA] instances_flags.AddMaintenancePolicyArgs(parser, deprecate_maintenance_policy) instances_flags.AddNoRestartOnFailureArgs(parser) instances_flags.AddPreemptibleVmArgs(parser) instances_flags.AddServiceAccountAndScopeArgs(parser, False) instances_flags.AddTagsArgs(parser) instances_flags.AddCustomMachineTypeArgs(parser) instances_flags.AddImageArgs(parser) instances_flags.AddNetworkArgs(parser) if support_shielded_vms: instances_flags.AddShieldedVMConfigArgs(parser) labels_util.AddCreateLabelsFlags(parser) if support_network_tier: instances_flags.AddNetworkTierArgs(parser, instance=True) if support_sole_tenancy: sole_tenancy_flags.AddNodeAffinityFlagToParser(parser) flags.AddRegionFlag(parser, resource_type='subnetwork', operation_type='attach') parser.add_argument( '--description', help='Specifies a textual description for the instance template.') Create.InstanceTemplateArg = ( instance_templates_flags.MakeInstanceTemplateArg()) Create.InstanceTemplateArg.AddArgument(parser, operation_type='create') if support_source_instance: instance_templates_flags.MakeSourceInstanceArg().AddArgument(parser) parser.add_argument( '--configure-disk', type=arg_parsers.ArgDict(spec={ 'auto-delete': arg_parsers.ArgBoolean(), 'device-name': str, 'instantiate-from': str, 'custom-image': str, }, ), metavar='PROPERTY=VALUE', action='append', help="""\ This option has effect only when used with `--source-instance`. It allows you to override how the source-instance's disks are defined in the template. *auto-delete*::: If `true`, this persistent disk will be automatically deleted when the instance is deleted. However, if the disk is later detached from the instance, this option won't apply. If not provided, the setting is copied from the source instance. Allowed values of the flag are: `false`, `no`, `true`, and `yes`. *device-name*::: Name of the device. *instantiate-from*::: Specifies whether to include the disk and which image to use. Valid values are: {} *custom-image*::: The custom image to use if custom-image is specified for instantiate-from. """.format(', '.join(_INSTANTIATE_FROM_VALUES)), ) parser.display_info.AddCacheUpdater(completers.InstanceTemplatesCompleter)
def ArgsForClusterRef(parser, beta=False, include_deprecated=True, include_ttl_config=False): """Register flags for creating a dataproc cluster. Args: parser: The argparse.ArgParser to configure with dataproc cluster arguments. beta: whether or not this is a beta command (may affect flag visibility) include_deprecated: whether deprecated flags should be included include_ttl_config: whether to include Scheduled Delete(TTL) args """ labels_util.AddCreateLabelsFlags(parser) instances_flags.AddTagsArgs(parser) # 30m is backend timeout + 5m for safety buffer. flags.AddTimeoutFlag(parser, default='35m') flags.AddZoneFlag(parser, short_flags=include_deprecated) flags.AddComponentFlag(parser) parser.add_argument( '--metadata', type=arg_parsers.ArgDict(min_length=1), action='append', default=None, help=('Metadata to be made available to the guest operating system ' 'running on the instances'), metavar='KEY=VALUE') # Either allow creating a single node cluster (--single-node), or specifying # the number of workers in the multi-node cluster (--num-workers and # --num-preemptible-workers) node_group = parser.add_argument_group(mutex=True) # Mutually exclusive node_group.add_argument('--single-node', action='store_true', help="""\ Create a single node cluster. A single node cluster has all master and worker components. It cannot have any separate worker nodes. If this flag is not specified, a cluster with separate workers is created. """) # Not mutually exclusive worker_group = node_group.add_argument_group( help='Multi-node cluster flags') worker_group.add_argument( '--num-workers', type=int, help='The number of worker nodes in the cluster. Defaults to ' 'server-specified.') worker_group.add_argument( '--num-preemptible-workers', type=int, help='The number of preemptible worker nodes in the cluster.') parser.add_argument( '--master-machine-type', help='The type of machine to use for the master. Defaults to ' 'server-specified.') parser.add_argument( '--worker-machine-type', help='The type of machine to use for workers. Defaults to ' 'server-specified.') image_parser = parser.add_mutually_exclusive_group() # TODO(b/73291743): Add external doc link to --image image_parser.add_argument( '--image', metavar='IMAGE', help='The full custom image URI or the custom image name that ' 'will be used to create a cluster.') image_parser.add_argument( '--image-version', metavar='VERSION', help='The image version to use for the cluster. Defaults to the ' 'latest version.') parser.add_argument('--bucket', help="""\ The Google Cloud Storage bucket to use by default to stage job dependencies, miscellaneous config files, and job driver console output when using this cluster. """) netparser = parser.add_mutually_exclusive_group() netparser.add_argument('--network', help="""\ The Compute Engine network that the VM instances of the cluster will be part of. This is mutually exclusive with --subnet. If neither is specified, this defaults to the "default" network. """) netparser.add_argument('--subnet', help="""\ Specifies the subnet that the cluster will be part of. This is mutally exclusive with --network. """) parser.add_argument( '--num-worker-local-ssds', type=int, help='The number of local SSDs to attach to each worker in a cluster.') parser.add_argument( '--num-master-local-ssds', type=int, help='The number of local SSDs to attach to the master in a cluster.') parser.add_argument('--num-preemptible-worker-local-ssds', type=int, help="""\ The number of local SSDs to attach to each preemptible worker in a cluster. """) parser.add_argument( '--initialization-actions', type=arg_parsers.ArgList(min_length=1), metavar='CLOUD_STORAGE_URI', help=('A list of Google Cloud Storage URIs of ' 'executables to run on each node in the cluster.')) parser.add_argument( '--initialization-action-timeout', type=arg_parsers.Duration(), metavar='TIMEOUT', default='10m', help=('The maximum duration of each initialization action. See ' '$ gcloud topic datetimes for information on duration formats.')) parser.add_argument( '--num-masters', type=arg_parsers.CustomFunctionValidator( lambda n: int(n) in [1, 3], 'Number of masters must be 1 (Standard) or 3 (High Availability)', parser=arg_parsers.BoundedInt(1, 3)), help="""\ The number of master nodes in the cluster. Number of Masters | Cluster Mode --- | --- 1 | Standard 3 | High Availability """) parser.add_argument('--properties', type=arg_parsers.ArgDict(), action=arg_parsers.UpdateAction, default={}, metavar='PREFIX:PROPERTY=VALUE', help="""\ Specifies configuration properties for installed packages, such as Hadoop and Spark. Properties are mapped to configuration files by specifying a prefix, such as "core:io.serializations". The following are supported prefixes and their mappings: Prefix | File | Purpose of file --- | --- | --- capacity-scheduler | capacity-scheduler.xml | Hadoop YARN Capacity Scheduler configuration core | core-site.xml | Hadoop general configuration distcp | distcp-default.xml | Hadoop Distributed Copy configuration hadoop-env | hadoop-env.sh | Hadoop specific environment variables hdfs | hdfs-site.xml | Hadoop HDFS configuration hive | hive-site.xml | Hive configuration mapred | mapred-site.xml | Hadoop MapReduce configuration mapred-env | mapred-env.sh | Hadoop MapReduce specific environment variables pig | pig.properties | Pig configuration spark | spark-defaults.conf | Spark configuration spark-env | spark-env.sh | Spark specific environment variables yarn | yarn-site.xml | Hadoop YARN configuration yarn-env | yarn-env.sh | Hadoop YARN specific environment variables See https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/cluster-properties for more information. """) parser.add_argument( '--service-account', help='The Google Cloud IAM service account to be authenticated as.') parser.add_argument('--scopes', type=arg_parsers.ArgList(min_length=1), metavar='SCOPE', help="""\ Specifies scopes for the node instances. Multiple SCOPEs can be specified, separated by commas. Examples: $ {{command}} example-cluster --scopes https://www.googleapis.com/auth/bigtable.admin $ {{command}} example-cluster --scopes sqlservice,bigquery The following *minimum scopes* are necessary for the cluster to function properly and are always added, even if not explicitly specified: {minimum_scopes} If the `--scopes` flag is not specified, the following *default scopes* are also included: {additional_scopes} If you want to enable all scopes use the 'cloud-platform' scope. {scopes_help} """.format(minimum_scopes='\n '.join(constants.MINIMUM_SCOPE_URIS), additional_scopes='\n '.join( constants.ADDITIONAL_DEFAULT_SCOPE_URIS), scopes_help=compute_helpers.SCOPES_HELP)) if include_deprecated: _AddDiskArgsDeprecated(parser) else: _AddDiskArgs(parser) # --no-address is an exception to the no negative-flag style guildline to be # consistent with gcloud compute instances create --no-address parser.add_argument('--no-address', action='store_true', help="""\ If provided, the instances in the cluster will not be assigned external IP addresses. If omitted the instances in the cluster will each be assigned an ephemeral external IP address. Note: Dataproc VMs need access to the Dataproc API. This can be achieved without external IP addresses using Private Google Access (https://cloud.google.com/compute/docs/private-google-access). """) boot_disk_type_detailed_help = """\ The type of the boot disk. The value must be ``pd-standard'' or ``pd-ssd''. """ parser.add_argument('--master-boot-disk-type', help=boot_disk_type_detailed_help) parser.add_argument('--worker-boot-disk-type', help=boot_disk_type_detailed_help) parser.add_argument('--preemptible-worker-boot-disk-type', help=boot_disk_type_detailed_help) autoscaling_group = parser.add_argument_group() flags.AddAutoscalingPolicyResourceArgForCluster( autoscaling_group, api_version=('v1beta2' if beta else 'v1')) if include_ttl_config: parser.add_argument('--max-idle', type=arg_parsers.Duration(), help="""\ The duration before cluster is auto-deleted after last job completes, such as "2h" or "1d". See $ gcloud topic datetimes for information on duration formats. """) auto_delete_group = parser.add_mutually_exclusive_group() auto_delete_group.add_argument('--max-age', type=arg_parsers.Duration(), help="""\ The lifespan of the cluster before it is auto-deleted, such as "2h" or "1d". See $ gcloud topic datetimes for information on duration formats. """) auto_delete_group.add_argument('--expiration-time', type=arg_parsers.Datetime.Parse, help="""\ The time when cluster will be auto-deleted, such as "2017-08-29T18:52:51.142Z." See $ gcloud topic datetimes for information on time formats. """) AddKerberosGroup(parser) flags.AddMinCpuPlatformArgs(parser) for instance_type in ('master', 'worker', 'preemptible-worker'): help_msg = """\ Attaches accelerators (e.g. GPUs) to the {instance_type} instance(s). """.format(instance_type=instance_type) help_msg += """ *type*::: The specific type (e.g. nvidia-tesla-k80 for nVidia Tesla K80) of accelerator to attach to the instances. Use 'gcloud compute accelerator-types list' to learn about all available accelerator types. *count*::: The number of pieces of the accelerator to attach to each of the instances. The default value is 1. """ parser.add_argument('--{0}-accelerator'.format(instance_type), type=arg_parsers.ArgDict(spec={ 'type': str, 'count': int, }), metavar='type=TYPE,[count=COUNT]', help=help_msg)