def Run(self, args): lite_util.RequirePython36('gcloud pubsub lite-subscriptions subscribe') try: # pylint: disable=g-import-not-at-top from googlecloudsdk.api_lib.pubsub import lite_subscriptions # pylint: enable=g-import-not-at-top except ImportError: raise lite_util.NoGrpcInstalled() log.out.Print( 'Initializing the Subscriber stream... This may take up to 30 seconds.' ) printer = resource_printer.Printer(args.format or MESSAGE_FORMAT) with lite_subscriptions.SubscriberClient( args.CONCEPTS.subscription.Parse(), args.partitions or [], args.num_messages, args.auto_ack) as subscriber_client: received = 0 while received < args.num_messages: message = subscriber_client.Pull() if message: splits = message.message_id.split(',') message.message_id = 'Partition: {}, Offset: {}'.format( splits[0], splits[1]) printer.Print([message]) received += 1
def testGetReferencedKeyNamesFilterFormatPrinter(self): printer = resource_printer.Printer('table(abc.xyz:label=ABC, def, pdq)') with self.assertRaisesRegex(ValueError, ''): resource_reference.GetReferencedKeyNames( filter_string='ABC:xyz OR pdq<123 OR rst:no]', format_string='table(abc.xyz:label=ABC, def, pdq)', printer=printer, defaults=printer.column_attributes, )
def testGetReferencedKeyNamesFilterPrinter(self): expected = {'abc.xyz', 'def', 'pdq', 'rst'} printer = resource_printer.Printer('table(abc.xyz:label=ABC, def, pdq)') actual = resource_reference.GetReferencedKeyNames( filter_string='ABC:xyz OR pdq<123 OR rst:no]', printer=printer, defaults=printer.column_attributes, ) self.assertEqual(expected, actual)
def testWithPager(self): mock_more = self.StartObjectPatch(console_io, 'More') [resource] = self.CreateResourceList(1) resource_printer.Printer('json[pager]').PrintSingleRecord(resource) mock_more.assert_called_once_with(textwrap.dedent("""\ { "SelfLink": "http://g/selfie/a-0", "kind": "compute#instance", "labels": { "empty": "", "full": "value", "Ṳᾔḯ¢◎ⅾℯ": "®ǖɬɘς" }, "metadata": { "items": [ { "key": "a", "value": "b" }, { "key": "c", "value": "d" }, { "key": "e", "value": "f" }, { "key": "g", "value": "h" } ], "kind": "compute#metadata.2" }, "name": "my-instance-a-0", "networkInterfaces": [ { "accessConfigs": [ { "kind": "compute#accessConfig", "name": "External NAT", "natIP": "74.125.239.110", "type": "ONE_TO_ONE_NAT" } ], "name": "nic0", "network": "default", "networkIP": "10.240.150.0" } ], "size": 0, "unicode": "python 2 Ṳᾔḯ¢◎ⅾℯ ṧʊ¢кṧ" } """), out=log.out)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ # TODO(b/33234717): remove this after deprecation period flags.ProcessPackages(args) region = properties.VALUES.compute.region.Get(required=True) uris = jobs_prep.RunSetupAndUpload(args.packages, args.staging_bucket, args.package_path, args.job) log.debug('Using {0} as trainer uris'.format(uris)) job = jobs.BuildTrainingJob(path=args.config, module_name=args.module_name, job_name=args.job, trainer_uri=uris, region=region, user_args=args.user_args) job = jobs.Create(job) if args. async: log.status.Print('Job [{}] submitted successfully.'.format( job.jobId)) log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) return job log_fetcher = jobs.LogFetcher(job_id=job.jobId, polling_interval=_POLLING_INTERVAL, allow_multiline_logs=False) printer = resource_printer.Printer(jobs.LogFetcher.LOG_FORMAT, out=log.err) def _CtrlCHandler(signal, frame): del signal, frame # Unused raise KeyboardInterrupt with execution_utils.CtrlCSection(_CtrlCHandler): try: printer.Print(log_fetcher.YieldLogs()) except KeyboardInterrupt: log.status.Print('Received keyboard interrupt.') log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) job = jobs.Get(job.jobId) # If the job itself failed, we will return a failure status. if job.state is not job.StateValueValuesEnum.SUCCEEDED: self.exit_code = 1 return job
def Print(self, print_format, out=None, defaults=None): """Prints the unified diff of formatter output for original and changed. Prints a unified diff, eg, --- +++ @@ -27,6 +27,6 @@ settings.pricingPlan: PER_USE settings.replicationType: SYNCHRONOUS settings.settingsVersion: 1 -settings.tier: D1 +settings.tier: D0 state: RUNNABLE Args: print_format: The print format name. out: The output stream, stdout if None. defaults: Optional resource_projection_spec.ProjectionSpec defaults. """ # Fill a buffer with the object as rendered originally. buff_original = StringIO() printer = resource_printer.Printer(print_format, out=buff_original, defaults=defaults) printer.PrintSingleRecord(self.original) # Fill a buffer with the object as rendered after the change. buff_changed = StringIO() printer = resource_printer.Printer(print_format, out=buff_changed, defaults=defaults) printer.PrintSingleRecord(self.changed) # Send these two buffers to the unified_diff() function for printing. lines_original = buff_original.getvalue().split('\n') lines_changed = buff_changed.getvalue().split('\n') lines_diff = difflib.unified_diff(lines_original, lines_changed) out = out or log.out for line in lines_diff: out.write(line + '\n')
def GetRedirectionEnablementReport(project): """Prints a redirection enablement report and returns mis-configured repos. This checks all the GCR repositories in the supplied project and checks if they each have a repository in Artifact Registry create to be the redirection target. It prints a report as it validates. Args: project: The project to validate Returns: A list of the GCR repos that do not have a redirection repo configured in Artifact Registry. """ locations = ar_requests.ListLocations(project, 100) # Sorting is performed so that Python2 & 3 agree on the order of API calls # in scenario tests. gcr_repos = GetExistingGCRBuckets( sorted(_GCR_BUCKETS.values(), key=lambda x: x["repository"]), project) failed_repos = [] repo_report = [] report_line = [] con = console_attr.GetConsoleAttr() # For each gcr repo in a location that our environment supports, # is there an associated repo in AR? for gcr_repo in gcr_repos: if gcr_repo["location"] in locations: report_line = [gcr_repo["repository"], gcr_repo["location"]] ar_repo_name = "projects/{}/locations/{}/repositories/{}".format( project, gcr_repo["location"], gcr_repo["repository"]) try: ar_repo = ar_requests.GetRepository(ar_repo_name) report_line.append(con.Colorize(ar_repo.name, "green")) except apitools_exceptions.HttpNotFoundError: report_line.append(con.Colorize("NOT FOUND", "red")) failed_repos.append(gcr_repo) repo_report.append(report_line) log.status.Print("Redirection enablement report:\n") printer = resource_printer.Printer("table", out=log.status) printer.AddHeading([ con.Emphasize("Container Registry Host", bold=True), con.Emphasize("Location", bold=True), con.Emphasize("Artifact Registry Repository", bold=True) ]) for line in repo_report: printer.AddRecord(line) printer.Finish() log.status.Print() return failed_repos
def print_resources_and_outputs(self): """Prints the Resources and Outputs of this deployment.""" rsp = dm_api_util.FetchResourcesAndOutputs( self.client, self.messages, self.config['project'], self.config['name'], # self.ReleaseTrack() is base.ReleaseTrack.ALPHA ) printer = resource_printer.Printer(flags.RESOURCES_AND_OUTPUTS_FORMAT) printer.AddRecord(rsp) printer.Finish() return rsp
def Print(self, style='table', projection=None, attr=None, resource=None): if not projection: projection = ('(' 'path.basename(), ' 'time.iso(), ' 'list.list(), ' 'res.resolution(unknown):label=RESOLUTION, ' 'size.size(-), ' 'uri(), ' 'value.yesno(set, empty))') fmt = style + projection if attr: fmt += attr printer = resource_printer.Printer(fmt) for record in resource or self._resource: printer.AddRecord(record) printer.Finish()
def _PrintTable(self, title, show_versions, to_print, func, ignore_if_empty): """Prints a table of updatable components. Args: title: str, The title for the table. show_versions: bool, True to print versions in the table. to_print: list(list(snapshots.ComponentDiff)), The available components divided by state. func: func(snapshots.ComponentDiff) -> bool, Decides whether the component should be printed. ignore_if_empty: bool, True to not show the table at all if there are no matching components. """ attributes = [ 'box', 'legend=" "', ] if ignore_if_empty: attributes.append('no-empty-legend') else: attributes.append('empty-legend="No updates."') if title: attributes.append('title="{title}"'.format(title=title)) columns = [ 'state.name:label=Status', 'name:label=Name', ] if show_versions: columns.extend([ 'current.version.version_string:label=Installed:align=right', 'latest.version.version_string:label=Latest:align=right', ]) columns.extend([ 'id:label=ID', 'size.size(zero="",min=1048576):label=Size:align=right', ]) fmt = 'table[{attributes}]({columns})'.format( attributes=','.join(attributes), columns=','.join(columns)) printer = resource_printer.Printer(fmt) for components in to_print: for c in components: if not c.is_hidden and func(c): printer.AddRecord(c) printer.Finish()
def PrintTable(header, resource_list): """Print a table of results with the specified columns. Prints a table whose columns are the proto fields specified in the header list. Any fields which cannot be found are printed as empty. Args: header: A list of strings which are the field names to include in the table. Must match field names in the resource_list items. resource_list: A list of resource objects, each corresponding to a row in the table to print. """ printer = resource_printer.Printer('table', out=log.out) printer.AddHeading(header) for resource in resource_list: printer.AddRecord([resource[column] if column in resource else '' for column in header]) printer.Finish()
def GetReferencedKeyNames(filter_string=None, format_string=None, printer=None, defaults=None): """Returns the set of key names referenced by filter / format expressions. NOTICE: OnePlatform is forgiving on filter and format key reference name spelling. Use resource_property.GetMatchingIndex() when verifying against resource dictionaries to handle camel and snake case spellings. Args: filter_string: The resource filter expression string. format_string: The resource format expression string. printer: The parsed format_string. defaults: The resource format and filter default projection. Raises: ValueError: If both format_string and printer are specified. Returns: The set of key names referenced by filter / format expressions. """ keys = set() # Add the format key references. if printer: if format_string: raise ValueError('Cannot specify both format_string and printer.') elif format_string: printer = resource_printer.Printer(format_string, defaults=defaults) defaults = printer.column_attributes if printer: for col in printer.column_attributes.Columns(): keys.add(resource_lex.GetKeyName(col.key, omit_indices=True)) # Add the filter key references. if filter_string: expr = resource_filter.Compile(filter_string, defaults=defaults, backend=resource_keys_expr.Backend()) for key in expr.Evaluate(None): keys.add(resource_lex.GetKeyName(key, omit_indices=True)) return keys
def _InitPrinter(self): """Initializes the printer and associated attributes.""" if self._printer_is_initialized: return self._printer_is_initialized = True # Determine the format. self._format = self.GetFormat() # Initialize the default symbols. self._defaults = self._GetResourceInfoDefaults() # Instantiate a printer if output is enabled. if self._format: self._printer = resource_printer.Printer( self._format, defaults=self._defaults, out=log.out) if self._printer: self._defaults = self._printer.column_attributes
def Epilog(self, unused_results_were_displayed): accounts = auth_util.AllAccounts() printer = resource_printer.Printer( auth_util.ACCOUNT_TABLE_FORMAT, out=log.status) printer.Print(accounts)
def testUnknownFormat(self): with self.assertRaisesRegex( resource_printer.UnknownFormatError, r'Format must be one of .* received \[UnKnOwN\]'): resource_printer.Printer('UnKnOwN')
def Print(self, style='table', attributes='', fields=None, count=None, heading=None, resource=None, encoding=None): """Prints an example resource. Args: style: The format style name. For 'list' the resource is a list of names, otherwise the resource is a list of dicts. attributes: The ,-separated list of [no-]name[=value] attributes. fields: The field projection expression. count: The number of resource records to print. heading: The list of heading strings. resource: resource override. encoding: If not None then top level unicode keys and values in the resource are encoded using this encoding. """ if resource is None: if style == 'list': resource = [ 'Ṁöë', 'Larry', 'Shemp', 'Curly', 'Joe', 'Curly Joe', ] elif 'utf-8' in attributes.split(',') or 'win' in attributes.split(','): resource = [ {'name': 'Moe', 'kind': 'aaa', 'id': 1267}, {'name': 'Larry', 'kind': 'xxx', 'id': 1245}, {'name': 'Shemp', 'kind': 'xxx', 'id': 1233}, {'name': 'Curly', 'kind': 'qqq', 'id': 1234}, {'name': 'Joe', 'kind': ['new', 1], 'id': ['x', 6789]}, {'name': 'Curly Joe', 'kind': {'new': 2, 'a': 'b', 'q': 'a,z'}, 'id': {'x': 890}}, ] if fields is None: fields = '(name, kind, id)' else: resource = [ { 'name': 'Ṁöë', 'quote': ".TꙅAꟻ ɘↄAlq oᴎ 'ᴎiTTɘg ɘᴙ'ɘW", 'id': 1267, }, { 'name': 'Larry', 'quote': "ι ∂ι∂η'т ωαηηα ѕαу уєѕ, вυт ι ¢συℓ∂η'т ѕαу ησ.", 'id': 1245, }, { 'name': 'Shemp', 'quote': 'Hey, Ṁöë! Hey, Larry!', 'id': 'lrlrlrl', }, { 'name': 'Curly', 'quote': 'Søɨŧɇnłɏ!', 'id': 1234, }, { 'name': 'Joe', 'quote': 'Oh, cut it ouuuuuut!', 'id': ['new', 6789], }, { 'name': 'Curly Joe', 'quote': "One of these days, you're gonna poke my eyes out.", 'id': {'new': 890}, }, ] # This branch is flagged by mutant analysis. If it is omitted no tests # fail. This is because the code under test handles encoded data by # design. If a test were to fail here, either with or without encoding, # that would mean a bug in the code under test. if encoding: encoded_resource = [] for r in resource: item = {} for k, v in six.iteritems(r): if isinstance(k, six.text_type): k = k.encode(encoding) if isinstance(v, six.text_type): v = v.encode(encoding) item[k] = v encoded_resource.append(item) resource = encoded_resource if count is None: count = 4 if fields is None: fields = '(name, quote, id)' fmt = style + attributes + fields printer = resource_printer.Printer(fmt) if heading: printer.AddHeading(heading) if count is not None: resource = resource[:count] for record in resource: printer.AddRecord(record) printer.Finish()
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ region = properties.VALUES.compute.region.Get(required=True) staging_location = jobs_prep.GetStagingLocation( staging_bucket=args.staging_bucket, job_id=args.job, job_dir=args.job_dir) try: uris = jobs_prep.UploadPythonPackages( packages=args.packages, package_path=args.package_path, staging_location=staging_location) except jobs_prep.NoStagingLocationError: raise flags.ArgumentError( 'If local packages are provided, the `--staging-bucket` or ' '`--job-dir` flag must be given.') log.debug('Using {0} as trainer uris'.format(uris)) scale_tier_enum = (jobs.GetMessagesModule( ).GoogleCloudMlV1beta1TrainingInput.ScaleTierValueValuesEnum) scale_tier = scale_tier_enum( args.scale_tier) if args.scale_tier else None job = jobs.BuildTrainingJob( path=args.config, module_name=args.module_name, job_name=args.job, trainer_uri=uris, region=region, job_dir=args.job_dir.ToUrl() if args.job_dir else None, scale_tier=scale_tier, user_args=args.user_args, runtime_version=args.runtime_version) jobs_client = jobs.JobsClient() project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') job = jobs_client.Create(project_ref, job) log.status.Print('Job [{}] submitted successfully.'.format(job.jobId)) if args. async: log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) return job log_fetcher = stream.LogFetcher( filters=log_utils.LogFilters(job.jobId), polling_interval=_POLLING_INTERVAL, continue_func=log_utils.MakeContinueFunction(job.jobId)) printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err) def _CtrlCHandler(signal, frame): del signal, frame # Unused raise KeyboardInterrupt with execution_utils.CtrlCSection(_CtrlCHandler): try: printer.Print(log_utils.SplitMultiline( log_fetcher.YieldLogs())) except KeyboardInterrupt: log.status.Print('Received keyboard interrupt.') log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) job_ref = resources.REGISTRY.Parse(job.jobId, collection='ml.projects.jobs') job = jobs_client.Get(job_ref) # If the job itself failed, we will return a failure status. if job.state is not job.StateValueValuesEnum.SUCCEEDED: self.exit_code = 1 return job
def SubmitTraining(jobs_client, job, job_dir=None, staging_bucket=None, packages=None, package_path=None, scale_tier=None, config=None, module_name=None, runtime_version=None, async_=None, user_args=None): """Submit a training job.""" region = properties.VALUES.compute.region.Get(required=True) staging_location = jobs_prep.GetStagingLocation( staging_bucket=staging_bucket, job_id=job, job_dir=job_dir) try: uris = jobs_prep.UploadPythonPackages( packages=packages, package_path=package_path, staging_location=staging_location) except jobs_prep.NoStagingLocationError: raise flags.ArgumentError( 'If local packages are provided, the `--staging-bucket` or ' '`--job-dir` flag must be given.') log.debug('Using {0} as trainer uris'.format(uris)) scale_tier_enum = jobs_client.training_input_class.ScaleTierValueValuesEnum scale_tier = scale_tier_enum(scale_tier) if scale_tier else None job = jobs_client.BuildTrainingJob( path=config, module_name=module_name, job_name=job, trainer_uri=uris, region=region, job_dir=job_dir.ToUrl() if job_dir else None, scale_tier=scale_tier, user_args=user_args, runtime_version=runtime_version) project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') job = jobs_client.Create(project_ref, job) log.status.Print('Job [{}] submitted successfully.'.format(job.jobId)) if async_: log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) return job log_fetcher = stream.LogFetcher( filters=log_utils.LogFilters(job.jobId), polling_interval=_POLLING_INTERVAL, continue_func=log_utils.MakeContinueFunction(job.jobId)) printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err) with execution_utils.RaisesKeyboardInterrupt(): try: printer.Print(log_utils.SplitMultiline(log_fetcher.YieldLogs())) except KeyboardInterrupt: log.status.Print('Received keyboard interrupt.\n') log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) except exceptions.HttpError as err: log.status.Print('Polling logs failed:\n{}\n'.format(str(err))) log.info('Failure details:', exc_info=True) log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) job_ref = resources.REGISTRY.Parse(job.jobId, collection='ml.projects.jobs') job = jobs_client.Get(job_ref) return job
def Printer(self, fmt='config'): self._printer = resource_printer.Printer(fmt) return self._printer
def SetUp(self): self._printer = resource_printer.Printer('yaml') self.SetEncoding('utf-8')
def SetUp(self): self.SetEncoding('utf-8') self._printer = resource_printer.Printer('flattened')
def SubmitTraining(jobs_client, job, job_dir=None, staging_bucket=None, packages=None, package_path=None, scale_tier=None, config=None, module_name=None, runtime_version=None, python_version=None, stream_logs=None, user_args=None, labels=None, custom_train_server_config=None): """Submit a training job.""" region = properties.VALUES.compute.region.Get(required=True) staging_location = jobs_prep.GetStagingLocation( staging_bucket=staging_bucket, job_id=job, job_dir=job_dir) try: uris = jobs_prep.UploadPythonPackages( packages=packages, package_path=package_path, staging_location=staging_location) except jobs_prep.NoStagingLocationError: raise flags.ArgumentError( 'If local packages are provided, the `--staging-bucket` or ' '`--job-dir` flag must be given.') log.debug('Using {0} as trainer uris'.format(uris)) scale_tier_enum = jobs_client.training_input_class.ScaleTierValueValuesEnum scale_tier = scale_tier_enum(scale_tier) if scale_tier else None try: job = jobs_client.BuildTrainingJob( path=config, module_name=module_name, job_name=job, trainer_uri=uris, region=region, job_dir=job_dir.ToUrl() if job_dir else None, scale_tier=scale_tier, user_args=user_args, runtime_version=runtime_version, python_version=python_version, labels=labels, custom_train_server_config=custom_train_server_config) except jobs_prep.NoStagingLocationError: raise flags.ArgumentError( 'If `--package-path` is not specified, at least one Python package ' 'must be specified via `--packages`.') project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') job = jobs_client.Create(project_ref, job) if not stream_logs: PrintSubmitFollowUp(job.jobId, print_follow_up_message=True) return job else: PrintSubmitFollowUp(job.jobId, print_follow_up_message=False) log_fetcher = stream.LogFetcher( filters=log_utils.LogFilters(job.jobId), polling_interval=properties.VALUES.ml_engine.polling_interval.GetInt(), continue_interval=_CONTINUE_INTERVAL, continue_func=log_utils.MakeContinueFunction(job.jobId)) printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err) with execution_utils.RaisesKeyboardInterrupt(): try: printer.Print(log_utils.SplitMultiline(log_fetcher.YieldLogs())) except KeyboardInterrupt: log.status.Print('Received keyboard interrupt.\n') log.status.Print( _FOLLOW_UP_MESSAGE.format(job_id=job.jobId, project=project_ref.Name())) except exceptions.HttpError as err: log.status.Print('Polling logs failed:\n{}\n'.format( six.text_type(err))) log.info('Failure details:', exc_info=True) log.status.Print( _FOLLOW_UP_MESSAGE.format(job_id=job.jobId, project=project_ref.Name())) job_ref = resources.REGISTRY.Parse( job.jobId, params={'projectsId': properties.VALUES.core.project.GetOrFail}, collection='ml.projects.jobs') job = jobs_client.Get(job_ref) return job
def SetUp(self): self._printer = resource_printer.Printer( 'multi(metadata:format=json, networkInterfaces:format="table(name, ' 'network, networkIP)")')