def InitializeApiClients(cls, http): cls._debug_client = apis.GetClientInstance('debug', 'v2', http) cls._debug_messages = apis.GetMessagesModule('debug', 'v2') cls._resource_client = apis.GetClientInstance('projects', 'v1beta1', http) cls._resource_messages = apis.GetMessagesModule('projects', 'v1beta1')
def GetAdminClient(): """Shortcut to get the latest Bigtable Admin client.""" return apis.GetClientInstance('bigtableadmin', 'v2')
def __init__(self, api_client=None): self.api_client = api_client if self.api_client is None: self.api_client = core_apis.GetClientInstance(API_NAME, API_VERSION) self.api_messages = self.api_client.MESSAGES_MODULE
def OrganizationsClient(self): client = apis.GetClientInstance('cloudresourcemanager', 'v1') return client.organizations
def GetClientInstance(): return apis.GetClientInstance('servicemanagement', 'v1')
def GetApiClientInstance(): return core_apis.GetClientInstance('replicapoolupdater', 'v1beta1')
def GetClient(): """Import and return the appropriate storage client.""" return core_apis.GetClientInstance('storage', 'v1')
def Get(job): client = apis.GetClientInstance('ml', 'v1beta1') ref = resources.REGISTRY.Parse(job, collection='ml.projects.jobs') req = client.MESSAGES_MODULE.MlProjectsJobsGetRequest( projectsId=ref.projectsId, jobsId=ref.jobsId) return client.projects_jobs.Get(req)
def SetApiEndpoint(cls): cls._client = apis.GetClientInstance('sourcerepo', 'v1')
def SetApiEndpoint(cls, http): cls._client = apis.GetClientInstance('source', 'v1', http)
def GetClientInstance(): return apis.GetClientInstance(DATAFLOW_API_NAME, DATAFLOW_API_VERSION)
def FetchLogs(log_filter=None, log_ids=None, order_by='DESC', limit=None, parent=None): """Fetches log entries. This method uses Cloud Logging V2 api. https://cloud.google.com/logging/docs/api/introduction_v2 Entries are sorted on the timestamp field, and afterwards filter is applied. If limit is passed, returns only up to that many matching entries. It is recommended to provide a filter with resource.type, and log_ids. If neither log_filter nor log_ids are passed, no filtering is done. Args: log_filter: filter expression used in the request. log_ids: if present, contructs full log names based on parent and filters only those logs in addition to filtering with log_filter. order_by: the sort order, either DESC or ASC. limit: how many entries to return. parent: the name of the log's parent resource, e.g. "projects/foo" or "organizations/123". Defaults to the current project. Returns: A generator that returns matching log entries. Callers are responsible for handling any http exceptions. """ if parent: if not ('projects/' in parent or 'organizations/' in parent): raise exceptions.InvalidArgumentException( 'parent', 'Unknown parent type in parent %s' % parent) else: parent = 'projects/%s' % properties.VALUES.core.project.Get( required=True) # The backend has an upper limit of 1000 for page_size. # However, there is no need to retrieve more entries if limit is specified. page_size = min(limit or 1000, 1000) id_filter = _LogFilterForIds(log_ids, parent) if id_filter and log_filter: combined_filter = '%s AND (%s)' % (id_filter, log_filter) else: combined_filter = id_filter or log_filter if order_by.upper() == 'DESC': order_by = 'timestamp desc' else: order_by = 'timestamp asc' client = apis.GetClientInstance('logging', 'v2beta1') messages = apis.GetMessagesModule('logging', 'v2beta1') request = messages.ListLogEntriesRequest(resourceNames=[parent], filter=combined_filter, orderBy=order_by) if 'projects/' in parent: request.projectIds = [parent[len('projects/'):]] return list_pager.YieldFromList(client.entries, request, field='entries', limit=limit, batch_size=page_size, batch_size_attribute='pageSize')
def SetApiEndpoint(cls): cls._client = core_apis.GetClientInstance('bigquery', 'v2') cls._messages = core_apis.GetMessagesModule('bigquery', 'v2')
def OperationsClient(): return apis.GetClientInstance('cloudresourcemanager', OPERATIONS_API_VERSION)
def client(self): return apis.GetClientInstance('deploymentmanager', 'v2')
def Delete(session): client = apis.GetClientInstance('spanner', 'v1') msgs = apis.GetMessagesModule('spanner', 'v1') req = msgs.SpannerProjectsInstancesDatabasesSessionsDeleteRequest( name=session.name) return client.projects_instances_databases_sessions.Delete(req)
def Filter(self, context, args): context['iam-client'] = apis.GetClientInstance('iam', 'v1') context['iam-messages'] = apis.GetMessagesModule('iam', 'v1') context['iam-resources'] = resources
def GetClientInstance(no_http=False): return apis.GetClientInstance('ml', 'v1beta1', no_http=no_http)
def GetClientInstance(use_http=True): return apis.GetClientInstance(_API_NAME, _API_VERSION, no_http=(not use_http))
def GetClient(): """Returns the client for the logging API.""" return core_apis.GetClientInstance('logging', 'v2')
def __init__(self, project): self.project = project self.client = core_apis.GetClientInstance('storage', 'v1') self.messages = core_apis.GetMessagesModule('storage', 'v1')
def GetClientV1(): """Returns the client for the v1 logging API.""" return core_apis.GetClientInstance('logging', 'v1beta3')
if not result: return messages = GetMessagesModule() RaiseIfResultNotTypeOf(result, messages.Operation) result_dict = encoding.MessageToDict(result) if not async: op_name = result_dict['name'] log.status.Print( 'Waiting for async operation {0} to complete...'.format(op_name)) result_dict = encoding.MessageToDict( WaitForOperation(op_name, apis.GetClientInstance('servicemanagement', 'v1'))) # Convert metadata startTime to local time if 'metadata' in result_dict and 'startTime' in result_dict['metadata']: result_dict['metadata']['startTime'] = ( ConvertUTCDateTimeStringToLocalTimeString( result_dict['metadata']['startTime'])) return result_dict def RaiseIfResultNotTypeOf(test_object, expected_type, nonetype_ok=False): if nonetype_ok and test_object is None: return if not isinstance(test_object, expected_type): raise TypeError('result must be of type %s' % expected_type)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ if args.gcs_source_staging_dir is None: args.gcs_source_staging_dir = 'gs://{project}_cloudbuild/source'.format( project=properties.VALUES.core.project.Get(), ) if args.gcs_log_dir is None: args.gcs_log_dir = 'gs://{project}_cloudbuild/logs'.format( project=properties.VALUES.core.project.Get(), ) client = core_apis.GetClientInstance('cloudbuild', 'v1') messages = core_apis.GetMessagesModule('cloudbuild', 'v1') registry = self.context['registry'] gcs_client = storage_api.StorageClient() # First, create the build request. build_timeout = properties.VALUES.container.build_timeout.Get() if build_timeout is not None: timeout_str = build_timeout + 's' else: timeout_str = None if args.tag: build_config = messages.Build( images=[args.tag], steps=[ messages.BuildStep( name='gcr.io/cloud-builders/docker', args=['build', '-t', args.tag, '.'], ), ], timeout=timeout_str, ) elif args.config: build_config = config.LoadCloudbuildConfig(args.config, messages) if build_config.timeout is None: build_config.timeout = timeout_str suffix = '.tgz' if args.source.startswith('gs://') or os.path.isfile(args.source): _, suffix = os.path.splitext(args.source) # Next, stage the source to Cloud Storage. staged_object = '{stamp}_{tag_ish}{suffix}'.format( stamp=times.GetTimeStampFromDateTime(times.Now()), tag_ish='_'.join(build_config.images or 'null').replace('/', '_'), suffix=suffix, ) gcs_source_staging_dir = registry.Parse(args.gcs_source_staging_dir, collection='storage.objects') gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket) if gcs_source_staging_dir.object: staged_object = gcs_source_staging_dir.object + '/' + staged_object gcs_source_staging = registry.Create( collection='storage.objects', bucket=gcs_source_staging_dir.bucket, object=staged_object) if args.source.startswith('gs://'): gcs_source = registry.Parse(args.source, collection='storage.objects') staged_source_obj = gcs_client.Copy(gcs_source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: if not os.path.exists(args.source): raise c_exceptions.BadFileException( 'could not find source [{src}]'.format(src=args.source)) if os.path.isdir(args.source): source_snapshot = snapshot.Snapshot(args.source) size_str = resource_transform.TransformSize( source_snapshot.uncompressed_size) log.status.write( 'Creating temporary tarball archive of {num_files} file(s)' ' totalling {size} before compression.\n'.format( num_files=len(source_snapshot.files), size=size_str)) staged_source_obj = source_snapshot.CopyTarballToGCS( gcs_client, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) elif os.path.isfile(args.source): unused_root, ext = os.path.splitext(args.source) if ext not in _ALLOWED_SOURCE_EXT: raise c_exceptions.BadFileException( 'Local file [{src}] is none of ' + _ALLOWED_SOURCE_EXT.join(', ')) log.status.write('Uploading local file [{src}] to ' '[gs://{bucket}/{object}]\n'.format( src=args.source, bucket=gcs_source_staging.bucket, object=gcs_source_staging.object, )) staged_source_obj = gcs_client.CopyFileToGCS( storage_util.BucketReference.FromBucketUrl( gcs_source_staging.bucket), args.source, gcs_source_staging.object) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) gcs_log_dir = registry.Parse(args.gcs_log_dir, collection='storage.objects') if gcs_log_dir.bucket != gcs_source_staging.bucket: # Create the logs bucket if it does not yet exist. gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket) build_config.logsBucket = 'gs://' + gcs_log_dir.bucket + '/' + gcs_log_dir.object log.debug('creating build: ' + repr(build_config)) # Start the build. op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = registry.Create(collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) if build.logUrl: log.status.write( 'Logs are permanently available at [{log_url}]\n'.format( log_url=build.logUrl)) else: log.status.write('Logs are available in the Cloud Console.\n') # If the command is run --async, we just print out a reference to the build. if args. async: return build # Otherwise, logs are streamed from GCS. return cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
def GetApiKeysClientInstance(): return apis.GetClientInstance('apikeys', 'v1')
def __init__(self, client=None, messages=None): self.client = client or apis.GetClientInstance('ml', 'v1beta1') self.messages = messages or apis.GetMessagesModule('ml', 'v1beta1')
def GetGenomicsClient(version='v1'): return core_apis.GetClientInstance('genomics', version)
def ExecuteCloudBuild(project, bucket_ref, object_name, output_image): """Execute a call to CloudBuild service and wait for it to finish. Args: project: the cloud project ID. bucket_ref: Reference to GCS bucket containing source to build. object_name: GCS object name containing source to build. output_image: GCR location for the output docker image; eg, gcr.io/test-gae/hardcoded-output-tag. Raises: BuildFailedError: when the build fails. """ builder = properties.VALUES.app.container_builder_image.Get() log.debug('Using builder image: [{0}]'.format(builder)) logs_bucket = bucket_ref.bucket cloud_build_timeout = properties.VALUES.app.cloud_build_timeout.Get() if cloud_build_timeout is not None: timeout_str = cloud_build_timeout + 's' else: timeout_str = None cloudbuild_client = core_apis.GetClientInstance('cloudbuild', 'v1') cloudbuild_messages = core_apis.GetMessagesModule('cloudbuild', 'v1') build_op = cloudbuild_client.projects_builds.Create( cloudbuild_messages.CloudbuildProjectsBuildsCreateRequest( projectId=project, build=cloudbuild_messages.Build( timeout=timeout_str, source=cloudbuild_messages.Source( storageSource=cloudbuild_messages.StorageSource( bucket=bucket_ref.bucket, object=object_name, ), ), steps=[ cloudbuild_messages.BuildStep( name=builder, args=['build', '-t', output_image, '.']) ], images=[output_image], logsBucket=logs_bucket, ), )) # Find build ID from operation metadata and print the logs URL. build_id = None logs_uri = None if build_op.metadata is not None: for prop in build_op.metadata.additionalProperties: if prop.key == 'build': for build_prop in prop.value.object_value.properties: if build_prop.key == 'id': build_id = build_prop.value.string_value if logs_uri is not None: break if build_prop.key == 'logUrl': logs_uri = build_prop.value.string_value if build_id is not None: break break if build_id is None: raise BuildFailedError('Could not determine build ID') log.status.Print( 'Started cloud build [{build_id}].'.format(build_id=build_id)) log_object = CLOUDBUILD_LOGFILE_FMT_STRING.format(build_id=build_id) log_tailer = cloudbuild_logs.LogTailer(bucket=logs_bucket, obj=log_object) log_loc = None if logs_uri: log.status.Print('To see logs in the Cloud Console: ' + logs_uri) log_loc = 'at ' + logs_uri else: log.status.Print('Logs can be found in the Cloud Console.') log_loc = 'in the Cloud Console.' op = operations.WaitForOperation( operation_service=cloudbuild_client.operations, operation=build_op, retry_interval=1, max_retries=60 * 60, retry_callback=log_tailer.Poll) # Poll the logs one final time to ensure we have everything. We know this # final poll will get the full log contents because GCS is strongly consistent # and Container Builder waits for logs to finish pushing before marking the # build complete. log_tailer.Poll(is_last=True) final_status = _GetStatusFromOp(op) if final_status != CLOUDBUILD_SUCCESS: raise BuildFailedError('Cloud build failed with status ' + final_status + '. Check logs ' + log_loc)
def Run(self, args): project = properties.VALUES.core.project.Get(required=True) version = args.version or util.GenerateVersionId() use_cloud_build = properties.VALUES.app.use_cloud_build.GetBool() config_cleanup = None if args.deployables: app_config = yaml_parsing.AppConfigSet(args.deployables) else: if not os.path.exists(DEFAULT_DEPLOYABLE): console_io.PromptContinue( 'Deployment to Google App Engine requires an app.yaml file. ' 'This command will run `gcloud preview app gen-config` to generate ' 'an app.yaml file for you in the current directory (if the current ' 'directory does not contain an App Engine module, please answer ' '"no").', cancel_on_no=True) # This generates the app.yaml AND the Dockerfile (and related files). params = ext_runtime.Params(deploy=True) configurator = fingerprinter.IdentifyDirectory(os.getcwd(), params=params) if configurator is None: raise NoAppIdentifiedError( 'Could not identify an app in the current directory.\n\n' 'Please prepare an app.yaml file for your application manually ' 'and deploy again.') config_cleanup = configurator.GenerateConfigs() log.status.Print( '\nCreated [{0}] in the current directory.\n'.format( DEFAULT_DEPLOYABLE)) app_config = yaml_parsing.AppConfigSet([DEFAULT_DEPLOYABLE]) # If the app has enabled Endpoints API Management features, pass # control to the cloud_endpoints handler. for _, module in app_config.Modules().items(): if module and module.parsed and module.parsed.beta_settings: bs = module.parsed.beta_settings use_endpoints = bs.get('use_endpoints_api_management', '').lower() if (use_endpoints in ('true', '1', 'yes') and bs.get('endpoints_swagger_spec_file')): cloud_endpoints.PushServiceConfig( bs.get('endpoints_swagger_spec_file'), project, apis.GetClientInstance('servicemanagement', 'v1', self.Http()), apis.GetMessagesModule('servicemanagement', 'v1')) remote_build = True docker_build_property = properties.VALUES.app.docker_build.Get() if args.docker_build: remote_build = args.docker_build == 'remote' elif docker_build_property: remote_build = docker_build_property == 'remote' clients = _AppEngineClients( appengine_client.AppengineClient(args.server, args.ignore_bad_certs), appengine_api_client.GetApiClient(self.Http(timeout=None))) log.debug( 'API endpoint: [{endpoint}], API version: [{version}]'.format( endpoint=clients.api.client.url, version=clients.api.api_version)) cloudbuild_client = apis.GetClientInstance('cloudbuild', 'v1', self.Http()) storage_client = apis.GetClientInstance('storage', 'v1', self.Http()) promote = properties.VALUES.app.promote_by_default.GetBool() deployed_urls = _DisplayProposedDeployment(project, app_config, version, promote) if args.version or promote: # Prompt if there's a chance that you're overwriting something important: # If the version is set manually, you could be deploying over something. # If you're setting the new deployment to be the default version, you're # changing the target of the default URL. # Otherwise, all existing URLs will continue to work, so need to prompt. console_io.PromptContinue(default=True, throw_if_unattended=False, cancel_on_no=True) log.status.Print('Beginning deployment...') source_contexts = [] if args.repo_info_file: if args.image_url: raise NoRepoInfoWithImageUrlError() try: with open(args.repo_info_file, 'r') as f: source_contexts = json.load(f) except (ValueError, IOError) as ex: raise RepoInfoLoadError(args.repo_info_file, ex) if isinstance(source_contexts, dict): # This is an old-style source-context.json file. Convert to a new- # style array of extended contexts. source_contexts = [ context_util.ExtendContextDict(source_contexts) ] code_bucket_ref = None if use_cloud_build or app_config.NonHermeticModules(): # If using Argo CloudBuild, we'll need to upload source to a GCS bucket. code_bucket_ref = self._GetCodeBucket(clients.api, args) metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET) log.debug('Using bucket [{b}].'.format(b=code_bucket_ref)) modules = app_config.Modules() if any([m.RequiresImage() for m in modules.values()]): deploy_command_util.DoPrepareManagedVms(clients.gae) if args.image_url: if len(modules) != 1: raise MultiDeployError() for registry in constants.ALL_SUPPORTED_REGISTRIES: if args.image_url.startswith(registry): break else: raise UnsupportedRegistryError(args.image_url) module = modules.keys()[0] images = {module: args.image_url} else: images = deploy_command_util.BuildAndPushDockerImages( modules, version, cloudbuild_client, storage_client, self.Http(), code_bucket_ref, self.cli, remote_build, source_contexts, config_cleanup) deployment_manifests = {} if app_config.NonHermeticModules(): if properties.VALUES.app.use_gsutil.GetBool(): copy_func = deploy_app_command_util.CopyFilesToCodeBucket metric_name = metric_names.COPY_APP_FILES else: copy_func = deploy_app_command_util.CopyFilesToCodeBucketNoGsUtil metric_name = metric_names.COPY_APP_FILES_NO_GSUTIL deployment_manifests = copy_func( app_config.NonHermeticModules().items(), code_bucket_ref, source_contexts, storage_client) metrics.CustomTimedEvent(metric_name) all_services = clients.api.ListServices() # Now do deployment. for (module, info) in app_config.Modules().iteritems(): message = 'Updating module [{module}]'.format(module=module) with console_io.ProgressTracker(message): if args.force: log.warning( 'The --force argument is deprecated and no longer ' 'required. It will be removed in a future release.') clients.api.DeployModule(module, version, info, deployment_manifests.get(module), images.get(module)) metrics.CustomTimedEvent(metric_names.DEPLOY_API) stop_previous_version = ( deploy_command_util.GetStopPreviousVersionFromArgs(args)) if promote: new_version = version_util.Version(project, module, version) _Promote(all_services, new_version, clients, stop_previous_version) elif stop_previous_version: log.info( 'Not stopping previous version because new version was not ' 'promoted.') # Config files. for (c, info) in app_config.Configs().iteritems(): message = 'Updating config [{config}]'.format(config=c) with console_io.ProgressTracker(message): clients.gae.UpdateConfig(c, info.parsed) return deployed_urls
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Yields: Some value that we want to have printed later. """ client = core_apis.GetClientInstance('cloudbuild', 'v1') messages = core_apis.GetMessagesModule('cloudbuild', 'v1') if args.ongoing: tz = times.GetTimeZone('UTC') now = times.Now(tz) now_seconds = times.GetTimeStampFromDateTime(now) # We are wrapping list_pager.YieldFromList in another yield loop so that # we can use custom exit-early and filtering functionality. This code will # be simplified once the cloudbuild service supports server-side filtering. # # The exit-early is to ensure that, when listing ongoing builds, the command # doesn't page through the entire history of terminated builds to find out # that there weren't any. The build list will always be delivered in sorted # order with createTime descending. # # The custom filtering checks build.status to see if a build is ongoing or # not, and skips those that are terminated. # # We copy and decrement the limit, because otherwise YieldFromList would # not understand when to stop, due to skipping terminated builds. # # We cannot give YieldFromList a predicate, because it would not know that # it needs to stop paging after a certain time threshold - with no ongoing # builds it would page through the entire build history. limit = args.limit for build in list_pager.YieldFromList( client.projects_builds, messages.CloudbuildProjectsBuildsListRequest( pageSize=args.page_size, projectId=properties.VALUES.core.project.Get()), field='builds', batch_size_attribute='pageSize'): if args.ongoing: tz_create_time = build.createTime create_time = times.ParseDateTime(tz_create_time, tz) create_seconds = times.GetTimeStampFromDateTime(create_time) delta_seconds = now_seconds - create_seconds if delta_seconds > _ONGOING_THRESHOLD_SECONDS: break if build.status not in [ messages.Build.StatusValueValuesEnum.QUEUED, messages.Build.StatusValueValuesEnum.WORKING]: continue yield build limit -= 1 if limit == 0: break