def Args(parser):
resource_args.AddEnvironmentResourceArg(
parser,
'from whose Cloud Storage bucket to export plugins.',
positional=False)
flags.AddExportSourceFlag(parser, Export.SUBDIR_BASE)
flags.AddExportDestinationFlag(parser)
def Args(parser):
resource_args.AddEnvironmentResourceArg(
parser,
'into whose Cloud Storage bucket to import DAGs',
positional=False)
flags.AddImportSourceFlag(parser, Import.SUBDIR_BASE)
flags.AddImportDestinationFlag(parser, Import.SUBDIR_BASE)
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser,
'to delete',
plural=True,
required=True,
positional=True)
base.ASYNC_FLAG.AddToParser(parser)
def Args(cls, parser):
resource_args.AddEnvironmentResourceArg(
parser, 'in which to run an Airflow command')
doc_url = 'https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html'
parser.add_argument(
'subcommand',
metavar='SUBCOMMAND',
choices=list(cls.SUBCOMMAND_ALLOWLIST.keys()),
help=('The Airflow CLI subcommand to run. Available subcommands '
'include (listed with Airflow versions that support): {} '
'(see {} for more info).').format(
', '.join(
sorted([
'{} [{}, {})'.format(cmd, r[0] or '**', r[1]
or '**')
for cmd, r in cls.SUBCOMMAND_ALLOWLIST.items()
])), doc_url))
parser.add_argument('subcommand_nested',
metavar='SUBCOMMAND_NESTED',
nargs=argparse.OPTIONAL,
help='Additional subcommand in case it is nested.')
parser.add_argument(
'cmd_args',
metavar='CMD_ARGS',
nargs=argparse.REMAINDER,
help='Command line arguments to the subcommand.',
example='{command} myenv trigger_dag -- some_dag --run_id=foo')
def Args(parser, release_track=base.ReleaseTrack.GA):
resource_args.AddEnvironmentResourceArg(parser, 'to update')
base.ASYNC_FLAG.AddToParser(parser)
Update.update_type_group = parser.add_mutually_exclusive_group(
required=True, help='The update type.')
flags.AddNodeCountUpdateFlagToGroup(Update.update_type_group)
flags.AddPypiUpdateFlagsToGroup(Update.update_type_group)
flags.AddEnvVariableUpdateFlagsToGroup(Update.update_type_group)
flags.AddAirflowConfigUpdateFlagsToGroup(Update.update_type_group)
flags.AddLabelsUpdateFlagsToGroup(Update.update_type_group)
web_server_group = Update.update_type_group.add_mutually_exclusive_group(
)
flags.UPDATE_WEB_SERVER_ALLOW_IP.AddToParser(web_server_group)
flags.WEB_SERVER_ALLOW_ALL.AddToParser(web_server_group)
flags.WEB_SERVER_DENY_ALL.AddToParser(web_server_group)
flags.CLOUD_SQL_MACHINE_TYPE.AddToParser(Update.update_type_group)
flags.WEB_SERVER_MACHINE_TYPE.AddToParser(Update.update_type_group)
flags.AddAutoscalingUpdateFlagsToGroup(Update.update_type_group,
release_track)
if release_track != base.ReleaseTrack.GA:
flags.AddMasterAuthorizedNetworksUpdateFlagsToGroup(
Update.update_type_group)
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser, 'to list upgrades')
base.URI_FLAG.RemoveFromParser(parser)
parser.display_info.AddFormat(
'table[box,title="AVAILABLE UPGRADES"]('
'imageVersionId:label="IMAGE VERSION",'
'isDefault:label="COMPOSER DEFAULT",'
'supportedPythonVersions.list():label="SUPPORTED PYTHON VERSIONS")')
def Args(cls, parser):
resource_args.AddEnvironmentResourceArg(
parser, 'in which to list PyPI modules')
parser.add_argument('--tree',
default=None,
action='store_true',
help="""\
List PyPI packages, their versions and a dependency tree, as displayed by the "python -m pipdeptree --warn" command.
""")
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser, 'to update')
base.ASYNC_FLAG.AddToParser(parser)
Update.update_type_group = parser.add_mutually_exclusive_group(
required=True, help='The update type.')
flags.AddNodeCountUpdateFlagToGroup(Update.update_type_group)
flags.AddPypiUpdateFlagsToGroup(Update.update_type_group)
flags.AddEnvVariableUpdateFlagsToGroup(Update.update_type_group)
flags.AddAirflowConfigUpdateFlagsToGroup(Update.update_type_group)
flags.AddLabelsUpdateFlagsToGroup(Update.update_type_group)
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser,
'where to load a snapshot')
base.ASYNC_FLAG.AddToParser(parser)
parser.add_argument(
'--snapshot-path',
type=str,
help='The Cloud Storage path to load the snapshot from. It must '
'start with prefix gs:// and one needs to specify a single snapshot '
'that should be loaded.',
required=True)
flags.SKIP_PYPI_PACKAGES_INSTALLATION.AddToParser(parser)
def Args(parser):
resource_args.AddEnvironmentResourceArg(
parser, 'in which to run an Airflow command')
parser.add_argument('subcommand',
metavar='SUBCOMMAND',
choices=command_util.SUBCOMMAND_WHITELIST,
help='The Airflow CLI subcommand to run. (see '
'https://airflow.incubator.apache.org/cli.html)')
parser.add_argument(
'cmd_args',
metavar='CMD_ARGS',
nargs=argparse.REMAINDER,
help='Command line arguments to the subcommand.',
example='{command} myenv trigger_dag -- some_dag --run_id=foo')
def Args(parser):
resource_args.AddEnvironmentResourceArg(
parser, 'in which to run an Airflow command')
parser.add_argument(
'subcommand',
metavar='SUBCOMMAND',
choices=command_util.SUBCOMMAND_WHITELIST,
help=('The Airflow CLI subcommand to run. Available subcommands '
'include: {} (see https://airflow.apache.org/cli.html for more '
'info). Note that delete_dag is available from Airflow 1.10.1, '
'and list_dag_runs, next_execution are available from Airflow '
'1.10.2.').format(', '.join(command_util.SUBCOMMAND_WHITELIST)))
parser.add_argument(
'cmd_args',
metavar='CMD_ARGS',
nargs=argparse.REMAINDER,
help='Command line arguments to the subcommand.',
example='{command} myenv trigger_dag -- some_dag --run_id=foo')
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser,
'whose data to delete.',
positional=False)
flags.AddDeleteTargetPositional(parser, 'data')
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser, 'to create')
base.ASYNC_FLAG.AddToParser(parser)
parser.add_argument(
'--node-count',
type=int,
help='The number of nodes to create to run the environment.')
parser.add_argument(
'--zone',
help='The Compute Engine zone in which the environment will '
'be created. For example `--zone=us-central1-a`.')
parser.add_argument(
'--machine-type',
help='The Compute Engine machine type '
'(https://cloud.google.com/compute/docs/machine-types) to use for '
'nodes. For example `--machine-type=n1-standard-1`.')
parser.add_argument(
'--disk-size',
default='100GB',
type=arg_parsers.BinarySize(
lower_bound='20GB',
upper_bound='64TB',
suggested_binary_size_scales=['GB', 'TB']),
help='The disk size for each VM node in the environment. The minimum '
'size is 20GB, and the maximum is 64TB. Specified value must be an '
'integer multiple of gigabytes. Cannot be updated after the '
'environment has been created. If units are not provided, defaults to '
'GB.')
networking_group = parser.add_group(help='Virtual Private Cloud networking')
networking_group.add_argument(
'--network',
required=True,
help='The Compute Engine Network to which the environment will '
'be connected. If a \'Custom Subnet Network\' is provided, '
'`--subnetwork` must be specified as well.')
networking_group.add_argument(
'--subnetwork',
help='The Compute Engine subnetwork '
'(https://cloud.google.com/compute/docs/subnetworks) to which the '
'environment will be connected.')
labels_util.AddCreateLabelsFlags(parser)
flags.CREATE_ENV_VARS_FLAG.AddToParser(parser)
# Default is provided by API server.
parser.add_argument(
'--service-account',
help='The Google Cloud Platform service account to be used by the node '
'VMs. If a service account is not specified, the "default" Compute '
'Engine service account for the project is used. Cannot be updated.')
# Default is provided by API server.
parser.add_argument(
'--oauth-scopes',
help='The set of Google API scopes to be made available on all of the '
'node VMs. Defaults to '
'[\'https://www.googleapis.com/auth/cloud-platform\']. Cannot be '
'updated.',
type=arg_parsers.ArgList(),
metavar='SCOPE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--tags',
help='The set of instance tags applied to all node VMs. Tags are used '
'to identify valid sources or targets for network firewalls. Each tag '
'within the list must comply with RFC 1035. Cannot be updated.',
type=arg_parsers.ArgList(),
metavar='TAG',
action=arg_parsers.UpdateAction)
# API server will validate key/value pairs.
parser.add_argument(
'--airflow-configs',
help="""\
A list of Airflow software configuration override KEY=VALUE pairs to set. For
information on how to structure KEYs and VALUEs, run
`$ {top_command} help composer environments update`.""",
type=arg_parsers.ArgDict(),
metavar='KEY=VALUE',
action=arg_parsers.UpdateAction)
def _CommonArgs(parser):
"""Common arguments that apply to all ReleaseTracks."""
resource_args.AddEnvironmentResourceArg(parser, 'to create')
base.ASYNC_FLAG.AddToParser(parser)
parser.add_argument(
'--node-count',
type=int,
help='The number of nodes to create to run the environment.')
parser.add_argument(
'--zone',
help='The Compute Engine zone in which the environment will '
'be created. For example `--zone=us-central1-a`.')
parser.add_argument(
'--machine-type',
help='The Compute Engine machine type '
'(https://cloud.google.com/compute/docs/machine-types) to use for '
'nodes. For example `--machine-type=n1-standard-1`.')
parser.add_argument(
'--disk-size',
default='100GB',
type=arg_parsers.BinarySize(lower_bound='20GB',
upper_bound='64TB',
suggested_binary_size_scales=['GB', 'TB']),
help='The disk size for each VM node in the environment. The minimum '
'size is 20GB, and the maximum is 64TB. Specified value must be an '
'integer multiple of gigabytes. Cannot be updated after the '
'environment has been created. If units are not provided, defaults to '
'GB.')
networking_group = parser.add_group(
help='Virtual Private Cloud networking')
networking_group.add_argument(
'--network',
required=True,
help='The Compute Engine Network to which the environment will '
'be connected. If a \'Custom Subnet Network\' is provided, '
'`--subnetwork` must be specified as well.')
networking_group.add_argument(
'--subnetwork',
help='The Compute Engine subnetwork '
'(https://cloud.google.com/compute/docs/subnetworks) to which the '
'environment will be connected.')
labels_util.AddCreateLabelsFlags(parser)
flags.CREATE_ENV_VARS_FLAG.AddToParser(parser)
# Default is provided by API server.
parser.add_argument(
'--service-account',
help='The Google Cloud Platform service account to be used by the node '
'VMs. If a service account is not specified, the "default" Compute '
'Engine service account for the project is used. Cannot be updated.')
# Default is provided by API server.
parser.add_argument(
'--oauth-scopes',
help='The set of Google API scopes to be made available on all of the '
'node VMs. Defaults to '
'[\'https://www.googleapis.com/auth/cloud-platform\']. Cannot be '
'updated.',
type=arg_parsers.ArgList(),
metavar='SCOPE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--tags',
help='The set of instance tags applied to all node VMs. Tags are used '
'to identify valid sources or targets for network firewalls. Each tag '
'within the list must comply with RFC 1035. Cannot be updated.',
type=arg_parsers.ArgList(),
metavar='TAG',
action=arg_parsers.UpdateAction)
# API server will validate key/value pairs.
parser.add_argument('--airflow-configs',
help="""\
A list of Airflow software configuration override KEY=VALUE pairs to set. For
information on how to structure KEYs and VALUEs, run
`$ {top_command} help composer environments update`.""",
type=arg_parsers.ArgDict(),
metavar='KEY=VALUE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--python-version',
type=str,
choices={
'2': 'Created environment will use Python 2',
'3': 'Created environment will use Python 3'
},
help='The Python version to be used within the created environment. '
'Supplied value should represent the desired major Python version. '
'Cannot be updated.')
version_group = parser.add_mutually_exclusive_group()
airflow_version_type = arg_parsers.RegexpValidator(
r'^(\d+\.\d+(?:\.\d+)?)', 'must be in the form X.Y[.Z].')
version_group.add_argument(
'--airflow-version',
type=airflow_version_type,
help="""Version of Airflow to run in the environment.
Must be of the form `X.Y[.Z]`.
The latest supported Cloud Composer version will be used within
the created environment.""")
image_version_type = arg_parsers.RegexpValidator(
r'^composer-(\d+\.\d+.\d+|latest)-airflow-(\d+\.\d+(?:\.\d+)?)',
'must be in the form \'composer-A.B.C-airflow-X.Y[.Z]\' or '
'\'latest\' can be provided in place of the Cloud Composer version '
'string. For example: \'composer-latest-airflow-1.10.0\'.')
version_group.add_argument(
'--image-version',
type=image_version_type,
help="""Version of the image to run in the environment.
The image version encapsulates the versions of both Cloud Composer
and Apache Airflow. Must be of the form `composer-A.B.C-airflow-X.Y[.Z]`.
The Cloud Composer and Airflow versions are semantic versions.
`latest` can be provided instead of an explicit Cloud Composer
version number indicating that the server will replace `latest`
with the current Cloud Composer version. For the Apache Airflow
portion, the patch version can be omitted and the current
version will be selected. The version numbers that are used will
be stored.""")
def _CommonArgs(parser, support_max_pods_per_node, release_track):
"""Common arguments that apply to all ReleaseTracks."""
resource_args.AddEnvironmentResourceArg(parser, 'to create')
base.ASYNC_FLAG.AddToParser(parser)
parser.add_argument(
'--node-count',
type=int,
help='The number of nodes to create to run the environment.')
parser.add_argument(
'--zone',
help='The Compute Engine zone in which the environment will '
'be created. For example `--zone=us-central1-a`.')
parser.add_argument(
'--machine-type',
help='The Compute Engine machine type '
'(https://cloud.google.com/compute/docs/machine-types) to use for '
'nodes. For example `--machine-type=n1-standard-1`.')
parser.add_argument(
'--disk-size',
type=arg_parsers.BinarySize(lower_bound='20GB',
upper_bound='64TB',
suggested_binary_size_scales=['GB', 'TB']),
help='The disk size for each VM node in the environment. The minimum '
'size is 20GB, and the maximum is 64TB. Specified value must be an '
'integer multiple of gigabytes. Cannot be updated after the '
'environment has been created. If units are not provided, defaults to '
'GB.',
action=flags.V1ExclusiveStoreAction)
networking_group = parser.add_group(
help='Virtual Private Cloud networking')
networking_group.add_argument(
'--network',
required=True,
help='The Compute Engine Network to which the environment will '
'be connected. If a \'Custom Subnet Network\' is provided, '
'`--subnetwork` must be specified as well.')
networking_group.add_argument(
'--subnetwork',
help='The Compute Engine subnetwork '
'(https://cloud.google.com/compute/docs/subnetworks) to which the '
'environment will be connected.')
labels_util.AddCreateLabelsFlags(parser)
flags.CREATE_ENV_VARS_FLAG.AddToParser(parser)
# Default is provided by API server.
parser.add_argument(
'--service-account',
help='The Google Cloud Platform service account to be used by the node '
'VMs. If a service account is not specified, the "default" Compute '
'Engine service account for the project is used. Cannot be updated.')
# Default is provided by API server.
parser.add_argument(
'--oauth-scopes',
help='The set of Google API scopes to be made available on all of the '
'node VMs. Defaults to '
'[\'https://www.googleapis.com/auth/cloud-platform\']. Cannot be '
'updated.',
type=arg_parsers.ArgList(),
metavar='SCOPE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--tags',
help='The set of instance tags applied to all node VMs. Tags are used '
'to identify valid sources or targets for network firewalls. Each tag '
'within the list must comply with RFC 1035. Cannot be updated.',
type=arg_parsers.ArgList(),
metavar='TAG',
action=arg_parsers.UpdateAction)
# API server will validate key/value pairs.
parser.add_argument('--airflow-configs',
help="""\
A list of Airflow software configuration override KEY=VALUE pairs to set. For
information on how to structure KEYs and VALUEs, run
`$ {top_command} help composer environments update`.""",
type=arg_parsers.ArgDict(),
metavar='KEY=VALUE',
action=arg_parsers.UpdateAction)
parser.add_argument(
'--python-version',
type=str,
choices={
'2': 'Created environment will use Python 2',
'3': 'Created environment will use Python 3'
},
action=flags.V1ExclusiveStoreAction,
help='The Python version to be used within the created environment. '
'Supplied value should represent the desired major Python version. '
'Cannot be updated.')
version_group = parser.add_mutually_exclusive_group()
airflow_version_type = arg_parsers.RegexpValidator(
r'^(\d+(?:\.\d+(?:\.\d+)?)?)', 'must be in the form X[.Y[.Z]].')
version_group.add_argument(
'--airflow-version',
type=airflow_version_type,
help="""Version of Apache Airflow to run in the environment.
Must be of the form `X[.Y[.Z]]`, where `[]` denotes optional fragments.
The current Cloud Composer version will be used within the created
environment. The Apache Airflow version is a semantic version or an alias
in the form of major or major.minor version numbers, resolved to the
latest matching Apache Airflow version supported in the current Cloud
Composer version. The resolved version is stored in the created
environment.""")
image_version_type = arg_parsers.RegexpValidator(
r'^composer-(\d+(?:\.\d+.\d+(?:-[a-z]+\.\d+)?)?|latest)-airflow-(\d+(?:\.\d+(?:\.\d+)?)?)',
'must be in the form \'composer-A[.B.C[-D.E]]-airflow-X[.Y[.Z]]\' or '
'\'latest\' can be provided in place of the Cloud Composer version '
'string. For example: \'composer-latest-airflow-1.10.0\'.')
version_group.add_argument(
'--image-version',
type=image_version_type,
help="""Version of the image to run in the environment.
The image version encapsulates the versions of both Cloud Composer
and Apache Airflow. Must be of the form
`composer-A[.B.C[-D.E]]-airflow-X[.Y[.Z]]`, where `[]` denotes optional
fragments.
The Cloud Composer portion of the image version is a semantic version or
an alias in the form of major version number or `latest`, resolved to the
current Cloud Composer version. The Apache Airflow portion of the image
version is a semantic version or an alias in the form of major or
major.minor version numbers, resolved to the latest matching Apache
Airflow version supported in the given Cloud Composer version. The
resolved versions are stored in the created environment.""")
flags.AddIpAliasEnvironmentFlags(parser, support_max_pods_per_node)
flags.AddPrivateIpEnvironmentFlags(parser)
web_server_group = parser.add_mutually_exclusive_group()
flags.WEB_SERVER_ALLOW_IP.AddToParser(web_server_group)
flags.WEB_SERVER_ALLOW_ALL.AddToParser(web_server_group)
flags.WEB_SERVER_DENY_ALL.AddToParser(web_server_group)
flags.CLOUD_SQL_MACHINE_TYPE.AddToParser(parser)
flags.WEB_SERVER_MACHINE_TYPE.AddToParser(parser)
flags.AddMaintenanceWindowFlagsGroup(parser)
permission_info = '{} must hold permission {}'.format(
"The 'Cloud Composer Service Agent' service account",
"'Cloud KMS CryptoKey Encrypter/Decrypter'")
kms_resource_args.AddKmsKeyResourceArg(parser,
'environment',
permission_info=permission_info)
if release_track == base.ReleaseTrack.GA:
flags.ENVIRONMENT_SIZE_GA.choice_arg.AddToParser(parser)
elif release_track == base.ReleaseTrack.BETA:
flags.ENVIRONMENT_SIZE_BETA.choice_arg.AddToParser(parser)
elif release_track == base.ReleaseTrack.ALPHA:
flags.ENVIRONMENT_SIZE_ALPHA.choice_arg.AddToParser(parser)
autoscaling_group_parser = parser.add_argument_group(
flags.AUTOSCALING_FLAG_GROUP_DESCRIPTION)
flags.SCHEDULER_CPU.AddToParser(autoscaling_group_parser)
flags.WORKER_CPU.AddToParser(autoscaling_group_parser)
flags.WEB_SERVER_CPU.AddToParser(autoscaling_group_parser)
flags.SCHEDULER_MEMORY.AddToParser(autoscaling_group_parser)
flags.WORKER_MEMORY.AddToParser(autoscaling_group_parser)
flags.WEB_SERVER_MEMORY.AddToParser(autoscaling_group_parser)
flags.SCHEDULER_STORAGE.AddToParser(autoscaling_group_parser)
flags.WORKER_STORAGE.AddToParser(autoscaling_group_parser)
flags.WEB_SERVER_STORAGE.AddToParser(autoscaling_group_parser)
flags.MIN_WORKERS.AddToParser(autoscaling_group_parser)
flags.MAX_WORKERS.AddToParser(autoscaling_group_parser)
flags.NUM_SCHEDULERS.AddToParser(autoscaling_group_parser)
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser, 'to describe')
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser,
'for which to list data.',
positional=False)
parser.display_info.AddFormat('table(name)')
def Args(parser):
resource_args.AddEnvironmentResourceArg(parser,
'to restart web server for')
base.ASYNC_FLAG.AddToParser(parser)
