def Args(parser):
SnapshotDisks.disks_arg = disks_flags.MakeDiskArgZonalOrRegional(
plural=True)
labels_util.AddCreateLabelsFlags(parser)
flags.AddStorageLocationFlag(parser, 'snapshot')
_CommonArgs(parser)
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser, 'to create')
base.ASYNC_FLAG.AddToParser(parser)
parser.add_argument(
'--node-count',
type=int,
help='The number of nodes to create to run the environment.')
parser.add_argument(
'--zone',
help='The Compute Engine zone in which the environment will '
'be created. For example `--zone=us-central1-a`.')
parser.add_argument(
'--machine-type',
help='The Compute Engine machine type '
'(https://cloud.google.com/compute/docs/machine-types) to use for '
'nodes. For example `--machine-type=n1-standard-1`.')
parser.add_argument(
'--disk-size',
default='100GB',
type=arg_parsers.BinarySize(
lower_bound='20GB',
upper_bound='64TB',
suggested_binary_size_scales=['GB', 'TB']),
help=
'The disk size for each VM node in the environment. The minimum '
'size is 20GB, and the maximum is 64TB. Specified value must be an '
'integer multiple of gigabytes. Cannot be updated after the '
'environment has been created. If units are not provided, defaults to '
'GB.')
networking_group = parser.add_group(
help='Virtual Private Cloud networking')
networking_group.add_argument(
'--network',
required=True,
help='The Compute Engine Network to which the environment will '
'be connected. If a \'Custom Subnet Network\' is provided, '
'`--subnetwork` must be specified as well.')
networking_group.add_argument(
'--subnetwork',
help='The Compute Engine subnetwork '
'(https://cloud.google.com/compute/docs/subnetworks) to which the '
'environment will be connected.')
labels_util.AddCreateLabelsFlags(parser)
flags.CREATE_ENV_VARS_FLAG.AddToParser(parser)
# Default is provided by API server.
parser.add_argument(
'--service-account',
help=
'The Google Cloud Platform service account to be used by the node '
'VMs. If a service account is not specified, the "default" Compute '
'Engine service account for the project is used. Cannot be updated.'
)
# Default is provided by API server.
parser.add_argument(
'--oauth-scopes',
help=
'The set of Google API scopes to be made available on all of the '
'node VMs. Defaults to '
'[\'https://www.googleapis.com/auth/cloud-platform\']. Cannot be '
'updated.',
type=arg_parsers.ArgList(),
metavar='SCOPE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--tags',
help=
'The set of instance tags applied to all node VMs. Tags are used '
'to identify valid sources or targets for network firewalls. Each tag '
'within the list must comply with RFC 1035. Cannot be updated.',
type=arg_parsers.ArgList(),
metavar='TAG',
action=arg_parsers.UpdateAction)
# API server will validate key/value pairs.
parser.add_argument('--airflow-configs',
help="""\
A list of Airflow software configuration override KEY=VALUE pairs to set. For
information on how to structure KEYs and VALUEs, run
`$ {top_command} help composer environments update`.""",
type=arg_parsers.ArgDict(),
metavar='KEY=VALUE',
action=arg_parsers.UpdateAction)
def _CommonArgs(parser,
release_track,
support_source_instance,
support_local_ssd_size=False,
support_kms=False,
support_multi_writer=False,
support_mesh=False,
support_host_error_timeout_seconds=False,
support_numa_node_count=False,
support_visible_core_count=False,
support_disk_architecture=False,
support_max_run_duration=False):
"""Adding arguments applicable for creating instance templates."""
parser.display_info.AddFormat(instance_templates_flags.DEFAULT_LIST_FORMAT)
metadata_utils.AddMetadataArgs(parser)
instances_flags.AddDiskArgs(parser, enable_kms=support_kms)
instances_flags.AddCreateDiskArgs(
parser,
enable_kms=support_kms,
support_boot=True,
support_multi_writer=support_multi_writer,
support_disk_architecture=support_disk_architecture)
if support_local_ssd_size:
instances_flags.AddLocalSsdArgsWithSize(parser)
else:
instances_flags.AddLocalSsdArgs(parser)
instances_flags.AddCanIpForwardArgs(parser)
instances_flags.AddAddressArgs(parser, instances=False)
instances_flags.AddAcceleratorArgs(parser)
instances_flags.AddMachineTypeArgs(parser)
deprecate_maintenance_policy = release_track in [base.ReleaseTrack.ALPHA]
instances_flags.AddMaintenancePolicyArgs(parser, deprecate_maintenance_policy)
instances_flags.AddNoRestartOnFailureArgs(parser)
instances_flags.AddPreemptibleVmArgs(parser)
instances_flags.AddServiceAccountAndScopeArgs(parser, False)
instances_flags.AddTagsArgs(parser)
instances_flags.AddCustomMachineTypeArgs(parser)
instances_flags.AddImageArgs(parser)
instances_flags.AddNetworkArgs(parser)
instances_flags.AddShieldedInstanceConfigArgs(parser)
labels_util.AddCreateLabelsFlags(parser)
instances_flags.AddNetworkTierArgs(parser, instance=True)
instances_flags.AddPrivateNetworkIpArgs(parser)
instances_flags.AddMinNodeCpuArg(parser)
instances_flags.AddNestedVirtualizationArgs(parser)
instances_flags.AddThreadsPerCoreArgs(parser)
instances_flags.AddEnableUefiNetworkingArgs(parser)
instances_flags.AddResourceManagerTagsArgs(parser)
if support_numa_node_count:
instances_flags.AddNumaNodeCountArgs(parser)
instances_flags.AddStackTypeArgs(parser)
instances_flags.AddIpv6NetworkTierArgs(parser)
maintenance_flags.AddResourcePoliciesArgs(parser, 'added to',
'instance-template')
instances_flags.AddProvisioningModelVmArgs(parser)
instances_flags.AddInstanceTerminationActionVmArgs(parser)
if support_max_run_duration:
instances_flags.AddMaxRunDurationVmArgs(parser)
instance_templates_flags.AddServiceProxyConfigArgs(
parser, release_track=release_track)
if support_mesh:
instance_templates_flags.AddMeshArgs(parser)
sole_tenancy_flags.AddNodeAffinityFlagToParser(parser)
instances_flags.AddLocationHintArg(parser)
if support_visible_core_count:
instances_flags.AddVisibleCoreCountArgs(parser)
instances_flags.AddNetworkPerformanceConfigsArgs(parser)
flags.AddRegionFlag(
parser, resource_type='subnetwork', operation_type='attach')
parser.add_argument(
'--description',
help='Specifies a textual description for the instance template.')
Create.InstanceTemplateArg = (
instance_templates_flags.MakeInstanceTemplateArg())
Create.InstanceTemplateArg.AddArgument(parser, operation_type='create')
if support_source_instance:
instance_templates_flags.MakeSourceInstanceArg().AddArgument(parser)
parser.add_argument(
'--configure-disk',
type=arg_parsers.ArgDict(
spec={
'auto-delete': arg_parsers.ArgBoolean(),
'device-name': str,
'instantiate-from': str,
'custom-image': str,
},),
metavar='PROPERTY=VALUE',
action='append',
help="""\
This option has effect only when used with `--source-instance`. It
allows you to override how the source-instance's disks are defined in
the template.
*auto-delete*::: If `true`, this persistent disk will be automatically
deleted when the instance is deleted. However, if the disk is later
detached from the instance, this option won't apply. If not provided,
the setting is copied from the source instance. Allowed values of the
flag are: `false`, `no`, `true`, and `yes`.
*device-name*::: Name of the device.
*instantiate-from*::: Specifies whether to include the disk and which
image to use. Valid values are: {}
*custom-image*::: The custom image to use if custom-image is specified
for instantiate-from.
""".format(', '.join(_INSTANTIATE_FROM_VALUES)),
)
instances_flags.AddReservationAffinityGroup(
parser,
group_text="""\
Specifies the reservation for instances created from this template.
""",
affinity_text="""\
The type of reservation for instances created from this template.
""")
parser.display_info.AddCacheUpdater(completers.InstanceTemplatesCompleter)
if support_host_error_timeout_seconds:
instances_flags.AddHostErrorTimeoutSecondsArgs(parser)
def _AddCreateArgs(parser):
"""Add common arguments for `versions create` command."""
flags.GetModelName(positional=False, required=True).AddToParser(parser)
flags.GetDescriptionFlag('version').AddToParser(parser)
flags.GetRegionArg().AddToParser(parser)
flags.VERSION_NAME.AddToParser(parser)
base.Argument(
'--origin',
help="""\
Location of ```model/``` "directory" (see
https://cloud.google.com/ai-platform/prediction/docs/deploying-models#upload-model).
This overrides `deploymentUri` in the `--config` file. If this flag is
not passed, `deploymentUri` *must* be specified in the file from
`--config`.
Can be a Cloud Storage (`gs://`) path or local file path (no
prefix). In the latter case the files will be uploaded to Cloud
Storage and a `--staging-bucket` argument is required.
""").AddToParser(parser)
flags.RUNTIME_VERSION.AddToParser(parser)
base.ASYNC_FLAG.AddToParser(parser)
flags.STAGING_BUCKET.AddToParser(parser)
base.Argument(
'--config',
help="""\
Path to a YAML configuration file containing configuration parameters
for the
[Version](https://cloud.google.com/ai-platform/prediction/docs/reference/rest/v1/projects.models.versions)
to create.
The file is in YAML format. Note that not all attributes of a version
are configurable; available attributes (with example values) are:
description: A free-form description of the version.
deploymentUri: gs://path/to/source
runtimeVersion: '2.1'
# Set only one of either manualScaling or autoScaling.
manualScaling:
nodes: 10 # The number of nodes to allocate for this model.
autoScaling:
minNodes: 0 # The minimum number of nodes to allocate for this model.
labels:
user-defined-key: user-defined-value
The name of the version must always be specified via the required
VERSION argument.
Only one of manualScaling or autoScaling can be specified. If both
are specified in same yaml file an error will be returned.
If an option is specified both in the configuration file and via
command-line arguments, the command-line arguments override the
configuration file.
"""
).AddToParser(parser)
labels_util.AddCreateLabelsFlags(parser)
flags.FRAMEWORK_MAPPER.choice_arg.AddToParser(parser)
flags.AddPythonVersionFlag(parser, 'when creating the version')
flags.AddMachineTypeFlagToParser(parser)
flags.GetAcceleratorFlag().AddToParser(parser)
def AddBetaArgs(parser):
"""Declare beta flag and positional arguments for this command parser."""
flags.AddExternalMasterGroup(parser)
flags.AddInstanceResizeLimit(parser)
flags.AddNetwork(parser)
labels_util.AddCreateLabelsFlags(parser)
def Args(cls, parser):
dataproc = dp.Dataproc(cls.ReleaseTrack())
labels_util.AddCreateLabelsFlags(parser)
flags.AddTemplateResourceArg(parser, 'create', dataproc.api_version)
def Args(parser): SnapshotDisks.disks_arg = disks_flags.MakeDiskArg(plural=True) labels_util.AddCreateLabelsFlags(parser) _CommonArgs(parser)
def _CommonArgs(parser,
release_track,
support_source_instance,
support_create_disk=False,
support_network_tier=False,
support_local_ssd_size=False,
support_shielded_vms=False,
support_sole_tenancy=False):
"""Adding arguments applicable for creating instance templates."""
parser.display_info.AddFormat(instance_templates_flags.DEFAULT_LIST_FORMAT)
metadata_utils.AddMetadataArgs(parser)
instances_flags.AddDiskArgs(parser)
if support_create_disk:
instances_flags.AddCreateDiskArgs(parser)
if support_local_ssd_size:
instances_flags.AddLocalSsdArgsWithSize(parser)
else:
instances_flags.AddLocalSsdArgs(parser)
instances_flags.AddCanIpForwardArgs(parser)
instances_flags.AddAddressArgs(parser,
instances=False,
support_network_tier=support_network_tier)
instances_flags.AddAcceleratorArgs(parser)
instances_flags.AddMachineTypeArgs(parser)
deprecate_maintenance_policy = release_track in [base.ReleaseTrack.ALPHA]
instances_flags.AddMaintenancePolicyArgs(parser,
deprecate_maintenance_policy)
instances_flags.AddNoRestartOnFailureArgs(parser)
instances_flags.AddPreemptibleVmArgs(parser)
instances_flags.AddServiceAccountAndScopeArgs(parser, False)
instances_flags.AddTagsArgs(parser)
instances_flags.AddCustomMachineTypeArgs(parser)
instances_flags.AddImageArgs(parser)
instances_flags.AddNetworkArgs(parser)
if support_shielded_vms:
instances_flags.AddShieldedVMConfigArgs(parser)
labels_util.AddCreateLabelsFlags(parser)
if support_network_tier:
instances_flags.AddNetworkTierArgs(parser, instance=True)
if support_sole_tenancy:
sole_tenancy_flags.AddNodeAffinityFlagToParser(parser)
flags.AddRegionFlag(parser,
resource_type='subnetwork',
operation_type='attach')
parser.add_argument(
'--description',
help='Specifies a textual description for the instance template.')
Create.InstanceTemplateArg = (
instance_templates_flags.MakeInstanceTemplateArg())
Create.InstanceTemplateArg.AddArgument(parser, operation_type='create')
if support_source_instance:
instance_templates_flags.MakeSourceInstanceArg().AddArgument(parser)
parser.add_argument(
'--configure-disk',
type=arg_parsers.ArgDict(spec={
'auto-delete': arg_parsers.ArgBoolean(),
'device-name': str,
'instantiate-from': str,
'custom-image': str,
}, ),
metavar='PROPERTY=VALUE',
action='append',
help="""\
This option has effect only when used with `--source-instance`. It
allows you to override how the source-instance's disks are defined in
the template.
*auto-delete*::: If `true`, this persistent disk will be automatically
deleted when the instance is deleted. However, if the disk is later
detached from the instance, this option won't apply. If not provided,
the setting is copied from the source instance. Allowed values of the
flag are: `false`, `no`, `true`, and `yes`.
*device-name*::: Name of the device.
*instantiate-from*::: Specifies whether to include the disk and which
image to use. Valid values are: {}
*custom-image*::: The custom image to use if custom-image is specified
for instantiate-from.
""".format(', '.join(_INSTANTIATE_FROM_VALUES)),
)
parser.display_info.AddCacheUpdater(completers.InstanceTemplatesCompleter)
def ArgsForClusterRef(parser,
beta=False,
include_deprecated=True,
include_ttl_config=False):
"""Register flags for creating a dataproc cluster.
Args:
parser: The argparse.ArgParser to configure with dataproc cluster arguments.
beta: whether or not this is a beta command (may affect flag visibility)
include_deprecated: whether deprecated flags should be included
include_ttl_config: whether to include Scheduled Delete(TTL) args
"""
labels_util.AddCreateLabelsFlags(parser)
instances_flags.AddTagsArgs(parser)
# 30m is backend timeout + 5m for safety buffer.
flags.AddTimeoutFlag(parser, default='35m')
flags.AddZoneFlag(parser, short_flags=include_deprecated)
flags.AddComponentFlag(parser)
parser.add_argument(
'--metadata',
type=arg_parsers.ArgDict(min_length=1),
action='append',
default=None,
help=('Metadata to be made available to the guest operating system '
'running on the instances'),
metavar='KEY=VALUE')
# Either allow creating a single node cluster (--single-node), or specifying
# the number of workers in the multi-node cluster (--num-workers and
# --num-preemptible-workers)
node_group = parser.add_argument_group(mutex=True) # Mutually exclusive
node_group.add_argument('--single-node',
action='store_true',
help="""\
Create a single node cluster.
A single node cluster has all master and worker components.
It cannot have any separate worker nodes. If this flag is not
specified, a cluster with separate workers is created.
""")
# Not mutually exclusive
worker_group = node_group.add_argument_group(
help='Multi-node cluster flags')
worker_group.add_argument(
'--num-workers',
type=int,
help='The number of worker nodes in the cluster. Defaults to '
'server-specified.')
worker_group.add_argument(
'--num-preemptible-workers',
type=int,
help='The number of preemptible worker nodes in the cluster.')
parser.add_argument(
'--master-machine-type',
help='The type of machine to use for the master. Defaults to '
'server-specified.')
parser.add_argument(
'--worker-machine-type',
help='The type of machine to use for workers. Defaults to '
'server-specified.')
image_parser = parser.add_mutually_exclusive_group()
# TODO(b/73291743): Add external doc link to --image
image_parser.add_argument(
'--image',
metavar='IMAGE',
help='The full custom image URI or the custom image name that '
'will be used to create a cluster.')
image_parser.add_argument(
'--image-version',
metavar='VERSION',
help='The image version to use for the cluster. Defaults to the '
'latest version.')
parser.add_argument('--bucket',
help="""\
The Google Cloud Storage bucket to use by default to stage job
dependencies, miscellaneous config files, and job driver console output
when using this cluster.
""")
netparser = parser.add_mutually_exclusive_group()
netparser.add_argument('--network',
help="""\
The Compute Engine network that the VM instances of the cluster will be
part of. This is mutually exclusive with --subnet. If neither is
specified, this defaults to the "default" network.
""")
netparser.add_argument('--subnet',
help="""\
Specifies the subnet that the cluster will be part of. This is mutally
exclusive with --network.
""")
parser.add_argument(
'--num-worker-local-ssds',
type=int,
help='The number of local SSDs to attach to each worker in a cluster.')
parser.add_argument(
'--num-master-local-ssds',
type=int,
help='The number of local SSDs to attach to the master in a cluster.')
parser.add_argument('--num-preemptible-worker-local-ssds',
type=int,
help="""\
The number of local SSDs to attach to each preemptible worker in
a cluster.
""")
parser.add_argument(
'--initialization-actions',
type=arg_parsers.ArgList(min_length=1),
metavar='CLOUD_STORAGE_URI',
help=('A list of Google Cloud Storage URIs of '
'executables to run on each node in the cluster.'))
parser.add_argument(
'--initialization-action-timeout',
type=arg_parsers.Duration(),
metavar='TIMEOUT',
default='10m',
help=('The maximum duration of each initialization action. See '
'$ gcloud topic datetimes for information on duration formats.'))
parser.add_argument(
'--num-masters',
type=arg_parsers.CustomFunctionValidator(
lambda n: int(n) in [1, 3],
'Number of masters must be 1 (Standard) or 3 (High Availability)',
parser=arg_parsers.BoundedInt(1, 3)),
help="""\
The number of master nodes in the cluster.
Number of Masters | Cluster Mode
--- | ---
1 | Standard
3 | High Availability
""")
parser.add_argument('--properties',
type=arg_parsers.ArgDict(),
action=arg_parsers.UpdateAction,
default={},
metavar='PREFIX:PROPERTY=VALUE',
help="""\
Specifies configuration properties for installed packages, such as Hadoop
and Spark.
Properties are mapped to configuration files by specifying a prefix, such as
"core:io.serializations". The following are supported prefixes and their
mappings:
Prefix | File | Purpose of file
--- | --- | ---
capacity-scheduler | capacity-scheduler.xml | Hadoop YARN Capacity Scheduler configuration
core | core-site.xml | Hadoop general configuration
distcp | distcp-default.xml | Hadoop Distributed Copy configuration
hadoop-env | hadoop-env.sh | Hadoop specific environment variables
hdfs | hdfs-site.xml | Hadoop HDFS configuration
hive | hive-site.xml | Hive configuration
mapred | mapred-site.xml | Hadoop MapReduce configuration
mapred-env | mapred-env.sh | Hadoop MapReduce specific environment variables
pig | pig.properties | Pig configuration
spark | spark-defaults.conf | Spark configuration
spark-env | spark-env.sh | Spark specific environment variables
yarn | yarn-site.xml | Hadoop YARN configuration
yarn-env | yarn-env.sh | Hadoop YARN specific environment variables
See https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/cluster-properties
for more information.
""")
parser.add_argument(
'--service-account',
help='The Google Cloud IAM service account to be authenticated as.')
parser.add_argument('--scopes',
type=arg_parsers.ArgList(min_length=1),
metavar='SCOPE',
help="""\
Specifies scopes for the node instances. Multiple SCOPEs can be specified,
separated by commas.
Examples:
$ {{command}} example-cluster --scopes https://www.googleapis.com/auth/bigtable.admin
$ {{command}} example-cluster --scopes sqlservice,bigquery
The following *minimum scopes* are necessary for the cluster to function
properly and are always added, even if not explicitly specified:
{minimum_scopes}
If the `--scopes` flag is not specified, the following *default scopes*
are also included:
{additional_scopes}
If you want to enable all scopes use the 'cloud-platform' scope.
{scopes_help}
""".format(minimum_scopes='\n '.join(constants.MINIMUM_SCOPE_URIS),
additional_scopes='\n '.join(
constants.ADDITIONAL_DEFAULT_SCOPE_URIS),
scopes_help=compute_helpers.SCOPES_HELP))
if include_deprecated:
_AddDiskArgsDeprecated(parser)
else:
_AddDiskArgs(parser)
# --no-address is an exception to the no negative-flag style guildline to be
# consistent with gcloud compute instances create --no-address
parser.add_argument('--no-address',
action='store_true',
help="""\
If provided, the instances in the cluster will not be assigned external
IP addresses.
If omitted the instances in the cluster will each be assigned an
ephemeral external IP address.
Note: Dataproc VMs need access to the Dataproc API. This can be achieved
without external IP addresses using Private Google Access
(https://cloud.google.com/compute/docs/private-google-access).
""")
boot_disk_type_detailed_help = """\
The type of the boot disk. The value must be ``pd-standard'' or
``pd-ssd''.
"""
parser.add_argument('--master-boot-disk-type',
help=boot_disk_type_detailed_help)
parser.add_argument('--worker-boot-disk-type',
help=boot_disk_type_detailed_help)
parser.add_argument('--preemptible-worker-boot-disk-type',
help=boot_disk_type_detailed_help)
autoscaling_group = parser.add_argument_group()
flags.AddAutoscalingPolicyResourceArgForCluster(
autoscaling_group, api_version=('v1beta2' if beta else 'v1'))
if include_ttl_config:
parser.add_argument('--max-idle',
type=arg_parsers.Duration(),
help="""\
The duration before cluster is auto-deleted after last job completes,
such as "2h" or "1d".
See $ gcloud topic datetimes for information on duration formats.
""")
auto_delete_group = parser.add_mutually_exclusive_group()
auto_delete_group.add_argument('--max-age',
type=arg_parsers.Duration(),
help="""\
The lifespan of the cluster before it is auto-deleted, such as
"2h" or "1d".
See $ gcloud topic datetimes for information on duration formats.
""")
auto_delete_group.add_argument('--expiration-time',
type=arg_parsers.Datetime.Parse,
help="""\
The time when cluster will be auto-deleted, such as
"2017-08-29T18:52:51.142Z." See $ gcloud topic datetimes for
information on time formats.
""")
AddKerberosGroup(parser)
flags.AddMinCpuPlatformArgs(parser)
for instance_type in ('master', 'worker', 'preemptible-worker'):
help_msg = """\
Attaches accelerators (e.g. GPUs) to the {instance_type}
instance(s).
""".format(instance_type=instance_type)
help_msg += """
*type*::: The specific type (e.g. nvidia-tesla-k80 for nVidia Tesla
K80) of accelerator to attach to the instances. Use 'gcloud compute
accelerator-types list' to learn about all available accelerator
types.
*count*::: The number of pieces of the accelerator to attach to each
of the instances. The default value is 1.
"""
parser.add_argument('--{0}-accelerator'.format(instance_type),
type=arg_parsers.ArgDict(spec={
'type': str,
'count': int,
}),
metavar='type=TYPE,[count=COUNT]',
help=help_msg)
