def ParseMonitoredProject(monitored_project_name):
"""Returns the metrics scope and monitored project.
Parse the specified monitored project name and return the metrics scope and
monitored project.
Args:
monitored_project_name: The name of the monitored project to create/delete.
Raises:
MonitoredProjectNameError: If an invalid monitored project name is
specified.
Returns:
(metrics_scope_def, monitored_project_def): Project parsed metrics scope
project id, Project parsed metrics scope project id
"""
matched = re.match(
'locations/global/metricsScopes/([a-z0-9:\\-]+)/projects/([a-z0-9:\\-]+)',
monitored_project_name)
if bool(matched):
if matched.group(0) != monitored_project_name:
raise MonitoredProjectNameError(
'Invalid monitored project name has been specified.')
# full name
metrics_scope_def = projects_util.ParseProject(matched.group(1))
monitored_project_def = projects_util.ParseProject(matched.group(2))
else:
metrics_scope_def = projects_util.ParseProject(
properties.VALUES.core.project.Get(required=True))
monitored_project_def = projects_util.ParseProject(
monitored_project_name)
return metrics_scope_def, monitored_project_def
def Run(self, args):
project_ref = command_lib_util.ParseProject(args.id)
try:
create_op = projects_api.Create(
project_ref,
display_name=args.name,
update_labels=labels_util.GetUpdateLabelsDictFromArgs(args))
except apitools_exceptions.HttpError as error:
if error.status_code == httplib.CONFLICT:
msg = ('Project creation failed. The project ID you specified is '
'already in use by another project. Please try an alternative '
'ID.')
unused_type, unused_value, traceback = sys.exc_info()
raise exceptions.HttpException, msg, traceback
raise
log.CreatedResource(project_ref, async=True)
create_op = operations.WaitForOperation(create_op)
# Enable cloudapis.googleapis.com
if args.enable_cloud_apis:
log.debug('Enabling cloudapis.googleapis.com')
services_client = apis.GetClientInstance('servicemanagement', 'v1')
enable_operation = services_enable_api.EnableServiceApiCall(
project_ref.Name(), 'cloudapis.googleapis.com')
services_util.WaitForOperation(enable_operation.name, services_client)
# TODO(user): Retry in case it failed?
return operations.ExtractOperationResponse(
create_op, apis.GetMessagesModule('cloudresourcemanager', 'v1').Project)
def testCreateCertificatePermissionsIgnoresMissingKmsKey(self):
project_ref = projects_command_util.ParseProject(self._PROJECT_NAME)
self._ExpectProjectIamCall(project_ref,
['privateca.certificateAuthorities.create'],
['privateca.certificateAuthorities.create'])
iam.CheckCreateCertificateAuthorityPermissions(project_ref, None)
def RunImageCloudBuild(args, builder, builder_args, tags, output_filter,
compute_release_track):
"""Run a build related to image on Google Cloud Builder.
Args:
args: An argparse namespace. All the arguments that were provided to this
command invocation.
builder: Path to builder image.
builder_args: A list of key-value pairs to pass to builder.
tags: A list of strings for adding tags to the Argo build.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
compute_release_track: release track to be used for Compute API calls. One
of - "alpha", "beta" or "ga"
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
if compute_release_track in ['alpha', 'beta']:
_CheckIamPermissionsBeta(project_id)
else:
_CheckIamPermissions(project_id)
return _RunCloudBuild(args, builder, builder_args, ['gce-daisy'] + tags,
output_filter, args.log_location)
def Run(self, args):
project_ref = command_lib_util.ParseProject(args.id)
if not console_io.PromptContinue('Your project will be deleted.'):
return None
result = projects_api.Delete(project_ref)
log.DeletedResource(project_ref)
return result
def RunImageCloudBuild(args, builder, builder_args, tags, output_filter,
cloudbuild_service_account_roles,
compute_service_account_roles):
"""Run a build related to image on Google Cloud Builder.
Args:
args: An argparse namespace. All the arguments that were provided to this
command invocation.
builder: Path to builder image.
builder_args: A list of key-value pairs to pass to builder.
tags: A list of strings for adding tags to the Argo build.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
cloudbuild_service_account_roles: roles required for cloudbuild service
account.
compute_service_account_roles: roles required for compute service account.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
_CheckIamPermissions(project_id, cloudbuild_service_account_roles,
compute_service_account_roles)
return _RunCloudBuild(args, builder, builder_args, ['gce-daisy'] + tags,
output_filter, args.log_location)
def RunImageImport(args, import_args, tags, output_filter):
"""Run a build over gce_vm_image_import on Google Cloud Builder.
Args:
args: An argparse namespace. All the arguments that were provided to this
command invocation.
import_args: A list of key-value pairs to pass to importer.
tags: A list of strings for adding tags to the Argo build.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
_CheckIamPermissions(project_id)
return _RunCloudBuild(args, _IMAGE_IMPORT_BUILDER, import_args,
['gce-daisy'] + tags, output_filter, args.log_location)
def Run(self, args):
project_ref = command_lib_util.ParseProject(args.id)
condition = iam_util.ValidateAndExtractCondition(args)
return projects_api.RemoveIamPolicyBindingWithCondition(
project_ref, args.member, args.role, condition, args.all)
def Run(self, args):
lake_ref = args.CONCEPTS.project.Parse()
service_account = 'service-' + str(
project_util.GetProjectNumber(lake_ref.projectsId)
) + '@gcp-sa-dataplex.iam.gserviceaccount.com'
if args.IsSpecified('storage_bucket_resource'):
return lake.RemoveServiceAccountFromBucketPolicy(
storage_util.BucketReference(args.storage_bucket_resource),
'serviceAccount:' + service_account,
'roles/dataplex.serviceAgent')
if args.IsSpecified('bigquery_dataset_resource'):
get_dataset_request = apis.GetMessagesModule(
'bigquery', 'v2').BigqueryDatasetsGetRequest(
datasetId=args.bigquery_dataset_resource,
projectId=args.secondary_project)
dataset = apis.GetClientInstance(
'bigquery', 'v2').datasets.Get(request=get_dataset_request)
lake.RemoveServiceAccountFromDatasetPolicy(
dataset, service_account, 'roles/dataplex.serviceAgent')
return apis.GetClientInstance('bigquery', 'v2').datasets.Patch(
apis.GetMessagesModule(
'bigquery', 'v2').BigqueryDatasetsPatchRequest(
datasetId=args.bigquery_dataset_resource,
projectId=args.secondary_project,
dataset=dataset))
if args.IsSpecified('project_resource'):
return projects_api.RemoveIamPolicyBinding(
project_util.ParseProject(args.project_resource),
'serviceAccount:' + service_account,
'roles/dataplex.serviceAgent')
def Run(self, args):
labels_diff = labels_util.Diff.FromUpdateArgs(args)
project_ref = command_lib_util.ParseProject(args.id)
result = projects_api.Update(project_ref, name=args.name,
labels_diff=labels_diff)
log.UpdatedResource(project_ref)
return result
def _GetDiagnosticsServiceAccount(self, project):
"""Locates or creates a service account with the correct permissions.
Attempts to locate the service account meant for creating the signed url.
If not found, it will subsequently create the service account. It will then
give the service account the correct IAM permissions to create a signed url
to a GCS Bucket.
Args:
project: The project to search for the service account in.
Returns:
A string email of the service account to use.
"""
# Search for service account by name.
service_account = None
for account in self._diagnose_client.ListServiceAccounts(project):
if account.email.startswith('{}@'.format(_SERVICE_ACCOUNT_NAME)):
service_account = account.email
if service_account is None:
service_account = self._diagnose_client.CreateServiceAccount(
project, _SERVICE_ACCOUNT_NAME)
# We can apply the correct IAM permissions for accessing the GCS Bucket
# regardless of whether or not the account already has them.
project_ref = projects_util.ParseProject(project)
service_account_ref = 'serviceAccount:{}'.format(service_account)
projects_api.AddIamPolicyBinding(project_ref, service_account_ref,
'roles/storage.objectCreator')
projects_api.AddIamPolicyBinding(project_ref, service_account_ref,
'roles/storage.objectViewer')
return service_account
def _BindRolesToServiceAccount(self, sa_email, roles):
"""Binds roles to the provided service account.
Args:
sa_email: str, the service account to bind roles to.
roles: iterable, the roles to be bound to the service account.
"""
formatted_roles = '\n'.join(
['- {}'.format(role) for role in sorted(roles)])
log.status.Print(
'To use Eventarc with Cloud Run for Anthos/GKE destinations, Eventarc Service Agent [{}] '
'needs to be bound to the following required roles:\n{}'.format(
sa_email, formatted_roles))
console_io.PromptContinue(
default=False,
throw_if_unattended=True,
prompt_string='\nWould you like to bind these roles?',
cancel_on_no=True)
project_ref = projects_util.ParseProject(
properties.VALUES.core.project.Get(required=True))
member_str = 'serviceAccount:{}'.format(sa_email)
member_roles = [(member_str, role) for role in roles]
self._AddIamPolicyBindingsWithRetry(project_ref, member_roles)
log.status.Print('Roles successfully bound.')
def Run(self, args):
if args.name is None:
raise ArgumentError('--name must be specified.')
project_ref = command_lib_util.ParseProject(args.id)
result = projects_api.Update(project_ref, name=args.name)
log.UpdatedResource(project_ref)
return result
def RunIamCheck(self, project_id):
project_ref = project_util.ParseProject(project_id)
result = projects_api.TestIamPermissions(project_ref, REQUIRED_PERMISSIONS)
granted_permissions = result.permissions
if set(REQUIRED_PERMISSIONS) != set(granted_permissions):
raise Exception("caller doesn't have sufficient permission.")
def SetUp(self):
self.name_generator = e2e_utils.GetResourceNameGenerator('sdk-e2e')
project_name = next(self.name_generator)
self.project_id = command_lib_util.IdFromName(project_name)
self.project_ref = command_lib_util.ParseProject(self.project_id)
create_op = projects_api.Create(
self.project_ref,
parent=projects_api.ParentNameToResourceId(
CLOUD_SDK_TESTING_FOLDER_ID))
log.CreatedResource(self.project_ref, is_async=True)
operations.WaitForOperation(create_op)
log.debug('Enabling cloudapis.googleapis.com')
services_enable_api.EnableService(self.project_ref.Name(),
'cloudapis.googleapis.com')
self.Run('services enable cloudbilling')
self.Run(('alpha billing accounts projects link {project} '
'--account-id={billing}').format(project=self.project_id,
billing=self.BillingId()))
self.Run('projects add-iam-policy-binding {project} '
'--member="group:[email protected]" '
'--role="roles/owner"'.format(project=self.project_id))
properties.VALUES.core.disable_prompts.Set(True)
# This is set to false by sdk_test_base.SdkBase.
properties.VALUES.core.should_prompt_to_enable_api.Set(True)
# The api enablement check will prompt, and this will inject a yes into that
self.StartPatch(
'googlecloudsdk.core.console.console_io.PromptContinue',
return_value=True)
def RunIamCheck(self, project_id):
project_ref = project_util.ParseProject(project_id)
result = projects_api.TestIamPermissions(project_ref,
REQUIRED_PERMISSIONS)
granted_permissions = result.permissions
if set(REQUIRED_PERMISSIONS) != set(granted_permissions):
raise memberships_errors.InsufficientPermissionsError()
def testGetHttpError(self):
test_project = util.GetTestActiveProject()
test_project_ref = command_lib_util.ParseProject(test_project.projectId)
self.mock_client.projects.Get.Expect(
self.messages.CloudresourcemanagerProjectsGetRequest(
projectId=test_project.projectId),
exception=self.HttpError())
with self.assertRaises(exceptions.HttpError):
projects_api.Get(test_project_ref)
def testGet(self):
test_project = util.GetTestActiveProject()
test_project_ref = command_lib_util.ParseProject(test_project.projectId)
self.mock_client.projects.Get.Expect(
self.messages.CloudresourcemanagerProjectsGetRequest(
projectId=test_project.projectId),
test_project)
response = projects_api.Get(test_project_ref)
self.assertEqual(response, test_project)
def testUndelete(self):
test_project = util.GetTestActiveProject()
test_project_ref = command_lib_util.ParseProject(test_project.projectId)
self.mock_client.projects.Undelete.Expect(
self.messages.CloudresourcemanagerProjectsUndeleteRequest(
projectId=test_project.projectId),
self.messages.Empty())
response = projects_api.Undelete(test_project_ref)
self.assertEqual(response.projectId, test_project.projectId)
def Run(self, args):
project_ref = command_lib_util.ParseProject(args.id)
result = projects_api.Create(
project_ref,
args.name,
args.enable_cloud_apis,
update_labels=labels_util.GetUpdateLabelsDictFromArgs(args))
log.CreatedResource(project_ref)
return result
def Run(self, args):
folder_flags.CheckParentFlags(args)
project_ref = command_lib_util.ParseProject(args.id)
result = projects_api.Update(
project_ref,
parent=projects_api.ParentNameToResourceId(
folder_flags.GetParentFromFlags(args)))
log.UpdatedResource(project_ref)
return result
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
project_ref = util.ParseProject(properties.VALUES.core.project.GetOrFail())
return client.MakeRequests([(client.apitools_client.projects, 'Get',
client.messages.ComputeProjectsGetRequest(
project=project_ref.projectId))])[0]
def testCreateCertificatePermissionsSucceedsWithRequiredPermissions(self):
project_ref = projects_command_util.ParseProject(self._PROJECT_NAME)
key_ref = GetCryptoKeyRef(self._KMS_KEY_NAME)
self._ExpectProjectIamCall(project_ref,
['privateca.certificateAuthorities.create'],
['privateca.certificateAuthorities.create'])
self._ExpectKmsIamCall(key_ref, ['cloudkms.cryptoKeys.setIamPolicy'],
['cloudkms.cryptoKeys.setIamPolicy'])
iam.CheckCreateCertificateAuthorityPermissions(project_ref, key_ref)
def _IsExistingProject(project_id):
project_ref = projects_command_util.ParseProject(project_id)
try:
project = projects_api.Get(project_ref)
return projects_util.IsActive(project)
except Exception: # pylint: disable=broad-except
# Yeah, this isn't great, but there isn't a perfect exception super class
# that covers both API related errors and network errors.
return False
def Run(self, args):
"""Default Run method implementation."""
flags.CheckParentFlags(args, parent_required=False)
project_id = args.id
if not project_id and args.name:
candidate = command_lib_util.IdFromName(args.name)
if candidate and console_io.PromptContinue(
'No project id provided.',
'Use [{}] as project id'.format(candidate),
throw_if_unattended=True):
project_id = candidate
if not project_id:
raise exceptions.RequiredArgumentException(
'PROJECT_ID', 'an id must be provided for the new project')
project_ref = command_lib_util.ParseProject(project_id)
labels = labels_util.ParseCreateArgs(
args,
projects_util.GetMessages().Project.LabelsValue)
try:
create_op = projects_api.Create(
project_ref,
display_name=args.name,
parent=projects_api.ParentNameToResourceId(
flags.GetParentFromFlags(args)),
labels=labels)
except apitools_exceptions.HttpConflictError:
msg = (
'Project creation failed. The project ID you specified is '
'already in use by another project. Please try an alternative '
'ID.')
core_exceptions.reraise(exceptions.HttpException(msg))
log.CreatedResource(project_ref, is_async=True)
create_op = operations.WaitForOperation(create_op)
# Enable cloudapis.googleapis.com
if args.enable_cloud_apis:
log.debug('Enabling cloudapis.googleapis.com')
services_client = apis.GetClientInstance('servicemanagement', 'v1')
enable_operation = services_enable_api.EnableServiceApiCall(
project_ref.Name(), 'cloudapis.googleapis.com')
enable_operation_ref = resources.REGISTRY.Parse(
enable_operation.name,
collection='servicemanagement.operations')
services_util.WaitForOperation(enable_operation_ref,
services_client)
if args.set_as_default:
project_property = properties.FromString('core/project')
properties.PersistProperty(project_property, project_id)
log.status.Print(
'Updated property [core/project] to [{0}].'.format(project_id))
return operations.ExtractOperationResponse(
create_op,
apis.GetMessagesModule('cloudresourcemanager', 'v1').Project)
def Run(self, args):
labels_diff = labels_util.Diff.FromUpdateArgs(args)
if args.name is None and not labels_diff.MayHaveUpdates():
raise ArgumentError('At least one of --name, --update-labels or '
'--remove-labels must be specified.')
project_ref = command_lib_util.ParseProject(args.id)
result = projects_api.Update(project_ref, name=args.name,
labels_diff=labels_diff)
log.UpdatedResource(project_ref)
return result
def _GetComputeProject(release_track):
holder = compute_base_classes.ComputeApiHolder(release_track)
client = holder.client
project_ref = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
return client.MakeRequests([(client.apitools_client.projects, 'Get',
client.messages.ComputeProjectsGetRequest(
project=project_ref.projectId))])[0]
def RunDaisyBuild(args,
workflow,
variables,
daisy_bucket,
tags=None,
user_zone=None,
output_filter=None):
"""Run a build with Daisy on Google Cloud Builder.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
workflow: The path to the Daisy workflow to run.
variables: A string of key-value pairs to pass to Daisy.
daisy_bucket: A string containing the name of the GCS bucket that daisy
should use.
tags: A list of strings for adding tags to the Argo build.
user_zone: The GCP zone to tell Daisy to do work in. If unspecified,
defaults to wherever the Argo runner happens to be.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
_CheckIamPermissions(project_id)
# Make Daisy time out before gcloud by shaving off 2% from the timeout time,
# up to a max of 5m (300s).
two_percent = int(args.timeout * 0.02)
daisy_timeout = args.timeout - min(two_percent, 300)
daisy_args = [
'-gcs_path=gs://{0}/'.format(daisy_bucket),
'-default_timeout={0}s'.format(daisy_timeout),
'-variables={0}'.format(variables),
workflow,
]
if user_zone is not None:
daisy_args = ['-zone={0}'.format(user_zone)] + daisy_args
build_tags = ['gce-daisy']
if tags:
build_tags.extend(tags)
return _RunCloudBuild(args, _DAISY_BUILDER, daisy_args, build_tags,
output_filter, args.log_location)
def RunOsUpgradeBuild(args, output_filter, instance_uri, release_track):
"""Run a OS Upgrade on Google Cloud Builder.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
instance_uri: instance to be upgraded.
release_track: release track of the command used. One of - "alpha", "beta"
or "ga"
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
_CheckIamPermissions(
project_id,
frozenset(OS_UPGRADE_ROLES_FOR_CLOUDBUILD_SERVICE_ACCOUNT),
frozenset(OS_UPGRADE_ROLES_FOR_COMPUTE_SERVICE_ACCOUNT))
# Make OS Upgrade time-out before gcloud by shaving off 2% from the timeout
# time, up to a max of 5m (300s).
two_percent = int(args.timeout * 0.02)
os_upgrade_timeout = args.timeout - min(two_percent, 300)
os_upgrade_args = []
AppendArg(os_upgrade_args, 'instance', instance_uri)
AppendArg(os_upgrade_args, 'source-os', args.source_os)
AppendArg(os_upgrade_args, 'target-os', args.target_os)
AppendArg(os_upgrade_args, 'timeout', os_upgrade_timeout, '-{0}={1}s')
AppendArg(os_upgrade_args, 'client-id', 'gcloud')
if not args.create_machine_backup:
AppendArg(os_upgrade_args, 'create-machine-backup', 'false')
AppendBoolArg(os_upgrade_args, 'auto-rollback', args.auto_rollback)
AppendBoolArg(os_upgrade_args, 'use-staging-install-media',
args.use_staging_install_media)
AppendArg(os_upgrade_args, 'client-version', config.CLOUD_SDK_VERSION)
build_tags = ['gce-os-upgrade']
builder = _GetBuilder(args, _OS_UPGRADE_BUILDER_EXECUTABLE,
args.docker_image_tag, release_track,
_GetOSUpgradeRegion)
return _RunCloudBuild(args, builder, os_upgrade_args, build_tags,
output_filter, args.log_location)
def testCreateCertificatePermissionsFailsWithoutProjectPermissions(self):
project_ref = projects_command_util.ParseProject(self._PROJECT_NAME)
key_ref = GetCryptoKeyRef(self._KMS_KEY_NAME)
self._ExpectProjectIamCall(project_ref, [
'privateca.certificateauthorities.create', 'storage.buckets.create'
], ['privateca.certificateauthorities.create'])
with self.AssertRaisesExceptionMatches(
exceptions.InsufficientPermissionException, 'project'):
iam.CheckCreateCertificateAuthorityPermissions(
project_ref, key_ref)
