def Run(self, args):
lite_util.RequirePython36('gcloud pubsub lite-subscriptions subscribe')
try:
# pylint: disable=g-import-not-at-top
from googlecloudsdk.api_lib.pubsub import lite_subscriptions
# pylint: enable=g-import-not-at-top
except ImportError:
raise lite_util.NoGrpcInstalled()
log.out.Print(
'Initializing the Subscriber stream... This may take up to 30 seconds.'
)
printer = resource_printer.Printer(args.format or MESSAGE_FORMAT)
with lite_subscriptions.SubscriberClient(
args.CONCEPTS.subscription.Parse(), args.partitions or [],
args.num_messages, args.auto_ack) as subscriber_client:
received = 0
while received < args.num_messages:
message = subscriber_client.Pull()
if message:
splits = message.message_id.split(',')
message.message_id = 'Partition: {}, Offset: {}'.format(
splits[0], splits[1])
printer.Print([message])
received += 1
def testGetReferencedKeyNamesFilterFormatPrinter(self):
printer = resource_printer.Printer('table(abc.xyz:label=ABC, def, pdq)')
with self.assertRaisesRegex(ValueError, ''):
resource_reference.GetReferencedKeyNames(
filter_string='ABC:xyz OR pdq<123 OR rst:no]',
format_string='table(abc.xyz:label=ABC, def, pdq)',
printer=printer,
defaults=printer.column_attributes,
)
def testGetReferencedKeyNamesFilterPrinter(self):
expected = {'abc.xyz', 'def', 'pdq', 'rst'}
printer = resource_printer.Printer('table(abc.xyz:label=ABC, def, pdq)')
actual = resource_reference.GetReferencedKeyNames(
filter_string='ABC:xyz OR pdq<123 OR rst:no]',
printer=printer,
defaults=printer.column_attributes,
)
self.assertEqual(expected, actual)
def testWithPager(self):
mock_more = self.StartObjectPatch(console_io, 'More')
[resource] = self.CreateResourceList(1)
resource_printer.Printer('json[pager]').PrintSingleRecord(resource)
mock_more.assert_called_once_with(textwrap.dedent("""\
{
"SelfLink": "http://g/selfie/a-0",
"kind": "compute#instance",
"labels": {
"empty": "",
"full": "value",
"Ṳᾔḯ¢◎ⅾℯ": "®ǖɬɘς"
},
"metadata": {
"items": [
{
"key": "a",
"value": "b"
},
{
"key": "c",
"value": "d"
},
{
"key": "e",
"value": "f"
},
{
"key": "g",
"value": "h"
}
],
"kind": "compute#metadata.2"
},
"name": "my-instance-a-0",
"networkInterfaces": [
{
"accessConfigs": [
{
"kind": "compute#accessConfig",
"name": "External NAT",
"natIP": "74.125.239.110",
"type": "ONE_TO_ONE_NAT"
}
],
"name": "nic0",
"network": "default",
"networkIP": "10.240.150.0"
}
],
"size": 0,
"unicode": "python 2 Ṳᾔḯ¢◎ⅾℯ ṧʊ¢кṧ"
}
"""),
out=log.out)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
# TODO(b/33234717): remove this after deprecation period
flags.ProcessPackages(args)
region = properties.VALUES.compute.region.Get(required=True)
uris = jobs_prep.RunSetupAndUpload(args.packages, args.staging_bucket,
args.package_path, args.job)
log.debug('Using {0} as trainer uris'.format(uris))
job = jobs.BuildTrainingJob(path=args.config,
module_name=args.module_name,
job_name=args.job,
trainer_uri=uris,
region=region,
user_args=args.user_args)
job = jobs.Create(job)
if args. async:
log.status.Print('Job [{}] submitted successfully.'.format(
job.jobId))
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = jobs.LogFetcher(job_id=job.jobId,
polling_interval=_POLLING_INTERVAL,
allow_multiline_logs=False)
printer = resource_printer.Printer(jobs.LogFetcher.LOG_FORMAT,
out=log.err)
def _CtrlCHandler(signal, frame):
del signal, frame # Unused
raise KeyboardInterrupt
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
printer.Print(log_fetcher.YieldLogs())
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job = jobs.Get(job.jobId)
# If the job itself failed, we will return a failure status.
if job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
return job
def Print(self, print_format, out=None, defaults=None):
"""Prints the unified diff of formatter output for original and changed.
Prints a unified diff, eg,
---
+++
@@ -27,6 +27,6 @@
settings.pricingPlan: PER_USE
settings.replicationType: SYNCHRONOUS
settings.settingsVersion: 1
-settings.tier: D1
+settings.tier: D0
state: RUNNABLE
Args:
print_format: The print format name.
out: The output stream, stdout if None.
defaults: Optional resource_projection_spec.ProjectionSpec defaults.
"""
# Fill a buffer with the object as rendered originally.
buff_original = StringIO()
printer = resource_printer.Printer(print_format,
out=buff_original,
defaults=defaults)
printer.PrintSingleRecord(self.original)
# Fill a buffer with the object as rendered after the change.
buff_changed = StringIO()
printer = resource_printer.Printer(print_format,
out=buff_changed,
defaults=defaults)
printer.PrintSingleRecord(self.changed)
# Send these two buffers to the unified_diff() function for printing.
lines_original = buff_original.getvalue().split('\n')
lines_changed = buff_changed.getvalue().split('\n')
lines_diff = difflib.unified_diff(lines_original, lines_changed)
out = out or log.out
for line in lines_diff:
out.write(line + '\n')
def GetRedirectionEnablementReport(project):
"""Prints a redirection enablement report and returns mis-configured repos.
This checks all the GCR repositories in the supplied project and checks if
they each have a repository in Artifact Registry create to be the redirection
target. It prints a report as it validates.
Args:
project: The project to validate
Returns:
A list of the GCR repos that do not have a redirection repo configured in
Artifact Registry.
"""
locations = ar_requests.ListLocations(project, 100)
# Sorting is performed so that Python2 & 3 agree on the order of API calls
# in scenario tests.
gcr_repos = GetExistingGCRBuckets(
sorted(_GCR_BUCKETS.values(), key=lambda x: x["repository"]), project)
failed_repos = []
repo_report = []
report_line = []
con = console_attr.GetConsoleAttr()
# For each gcr repo in a location that our environment supports,
# is there an associated repo in AR?
for gcr_repo in gcr_repos:
if gcr_repo["location"] in locations:
report_line = [gcr_repo["repository"], gcr_repo["location"]]
ar_repo_name = "projects/{}/locations/{}/repositories/{}".format(
project, gcr_repo["location"], gcr_repo["repository"])
try:
ar_repo = ar_requests.GetRepository(ar_repo_name)
report_line.append(con.Colorize(ar_repo.name, "green"))
except apitools_exceptions.HttpNotFoundError:
report_line.append(con.Colorize("NOT FOUND", "red"))
failed_repos.append(gcr_repo)
repo_report.append(report_line)
log.status.Print("Redirection enablement report:\n")
printer = resource_printer.Printer("table", out=log.status)
printer.AddHeading([
con.Emphasize("Container Registry Host", bold=True),
con.Emphasize("Location", bold=True),
con.Emphasize("Artifact Registry Repository", bold=True)
])
for line in repo_report:
printer.AddRecord(line)
printer.Finish()
log.status.Print()
return failed_repos
def print_resources_and_outputs(self):
"""Prints the Resources and Outputs of this deployment."""
rsp = dm_api_util.FetchResourcesAndOutputs(
self.client,
self.messages,
self.config['project'],
self.config['name'],
# self.ReleaseTrack() is base.ReleaseTrack.ALPHA
)
printer = resource_printer.Printer(flags.RESOURCES_AND_OUTPUTS_FORMAT)
printer.AddRecord(rsp)
printer.Finish()
return rsp
def Print(self, style='table', projection=None, attr=None, resource=None):
if not projection:
projection = ('('
'path.basename(), '
'time.iso(), '
'list.list(), '
'res.resolution(unknown):label=RESOLUTION, '
'size.size(-), '
'uri(), '
'value.yesno(set, empty))')
fmt = style + projection
if attr:
fmt += attr
printer = resource_printer.Printer(fmt)
for record in resource or self._resource:
printer.AddRecord(record)
printer.Finish()
def _PrintTable(self, title, show_versions, to_print, func,
ignore_if_empty):
"""Prints a table of updatable components.
Args:
title: str, The title for the table.
show_versions: bool, True to print versions in the table.
to_print: list(list(snapshots.ComponentDiff)), The available components
divided by state.
func: func(snapshots.ComponentDiff) -> bool, Decides whether the component
should be printed.
ignore_if_empty: bool, True to not show the table at all if there are no
matching components.
"""
attributes = [
'box',
'legend=" "',
]
if ignore_if_empty:
attributes.append('no-empty-legend')
else:
attributes.append('empty-legend="No updates."')
if title:
attributes.append('title="{title}"'.format(title=title))
columns = [
'state.name:label=Status',
'name:label=Name',
]
if show_versions:
columns.extend([
'current.version.version_string:label=Installed:align=right',
'latest.version.version_string:label=Latest:align=right',
])
columns.extend([
'id:label=ID',
'size.size(zero="",min=1048576):label=Size:align=right',
])
fmt = 'table[{attributes}]({columns})'.format(
attributes=','.join(attributes), columns=','.join(columns))
printer = resource_printer.Printer(fmt)
for components in to_print:
for c in components:
if not c.is_hidden and func(c):
printer.AddRecord(c)
printer.Finish()
def PrintTable(header, resource_list):
"""Print a table of results with the specified columns.
Prints a table whose columns are the proto fields specified in the
header list. Any fields which cannot be found are printed as empty.
Args:
header: A list of strings which are the field names to include in the
table. Must match field names in the resource_list items.
resource_list: A list of resource objects, each corresponding to a row
in the table to print.
"""
printer = resource_printer.Printer('table', out=log.out)
printer.AddHeading(header)
for resource in resource_list:
printer.AddRecord([resource[column] if column in resource else ''
for column in header])
printer.Finish()
def GetReferencedKeyNames(filter_string=None,
format_string=None,
printer=None,
defaults=None):
"""Returns the set of key names referenced by filter / format expressions.
NOTICE: OnePlatform is forgiving on filter and format key reference name
spelling. Use resource_property.GetMatchingIndex() when verifying against
resource dictionaries to handle camel and snake case spellings.
Args:
filter_string: The resource filter expression string.
format_string: The resource format expression string.
printer: The parsed format_string.
defaults: The resource format and filter default projection.
Raises:
ValueError: If both format_string and printer are specified.
Returns:
The set of key names referenced by filter / format expressions.
"""
keys = set()
# Add the format key references.
if printer:
if format_string:
raise ValueError('Cannot specify both format_string and printer.')
elif format_string:
printer = resource_printer.Printer(format_string, defaults=defaults)
defaults = printer.column_attributes
if printer:
for col in printer.column_attributes.Columns():
keys.add(resource_lex.GetKeyName(col.key, omit_indices=True))
# Add the filter key references.
if filter_string:
expr = resource_filter.Compile(filter_string,
defaults=defaults,
backend=resource_keys_expr.Backend())
for key in expr.Evaluate(None):
keys.add(resource_lex.GetKeyName(key, omit_indices=True))
return keys
def _InitPrinter(self):
"""Initializes the printer and associated attributes."""
if self._printer_is_initialized:
return
self._printer_is_initialized = True
# Determine the format.
self._format = self.GetFormat()
# Initialize the default symbols.
self._defaults = self._GetResourceInfoDefaults()
# Instantiate a printer if output is enabled.
if self._format:
self._printer = resource_printer.Printer(
self._format, defaults=self._defaults, out=log.out)
if self._printer:
self._defaults = self._printer.column_attributes
def Epilog(self, unused_results_were_displayed):
accounts = auth_util.AllAccounts()
printer = resource_printer.Printer(
auth_util.ACCOUNT_TABLE_FORMAT,
out=log.status)
printer.Print(accounts)
def testUnknownFormat(self):
with self.assertRaisesRegex(
resource_printer.UnknownFormatError,
r'Format must be one of .* received \[UnKnOwN\]'):
resource_printer.Printer('UnKnOwN')
def Print(self, style='table', attributes='', fields=None, count=None,
heading=None, resource=None, encoding=None):
"""Prints an example resource.
Args:
style: The format style name. For 'list' the resource is a list of names,
otherwise the resource is a list of dicts.
attributes: The ,-separated list of [no-]name[=value] attributes.
fields: The field projection expression.
count: The number of resource records to print.
heading: The list of heading strings.
resource: resource override.
encoding: If not None then top level unicode keys and values in the
resource are encoded using this encoding.
"""
if resource is None:
if style == 'list':
resource = [
'Ṁöë',
'Larry',
'Shemp',
'Curly',
'Joe',
'Curly Joe',
]
elif 'utf-8' in attributes.split(',') or 'win' in attributes.split(','):
resource = [
{'name': 'Moe', 'kind': 'aaa', 'id': 1267},
{'name': 'Larry', 'kind': 'xxx', 'id': 1245},
{'name': 'Shemp', 'kind': 'xxx', 'id': 1233},
{'name': 'Curly', 'kind': 'qqq', 'id': 1234},
{'name': 'Joe', 'kind': ['new', 1], 'id': ['x', 6789]},
{'name': 'Curly Joe',
'kind': {'new': 2, 'a': 'b', 'q': 'a,z'}, 'id': {'x': 890}},
]
if fields is None:
fields = '(name, kind, id)'
else:
resource = [
{
'name': 'Ṁöë',
'quote': ".TꙅAꟻ ɘↄAlq oᴎ 'ᴎiTTɘg ɘᴙ'ɘW",
'id': 1267,
},
{
'name': 'Larry',
'quote': "ι ∂ι∂η'т ωαηηα ѕαу уєѕ, вυт ι ¢συℓ∂η'т ѕαу ησ.",
'id': 1245,
},
{
'name': 'Shemp',
'quote': 'Hey, Ṁöë! Hey, Larry!',
'id': 'lrlrlrl',
},
{
'name': 'Curly',
'quote': 'Søɨŧɇnłɏ!',
'id': 1234,
},
{
'name': 'Joe',
'quote': 'Oh, cut it ouuuuuut!',
'id': ['new', 6789],
},
{
'name': 'Curly Joe',
'quote': "One of these days, you're gonna poke my eyes out.",
'id': {'new': 890},
},
]
# This branch is flagged by mutant analysis. If it is omitted no tests
# fail. This is because the code under test handles encoded data by
# design. If a test were to fail here, either with or without encoding,
# that would mean a bug in the code under test.
if encoding:
encoded_resource = []
for r in resource:
item = {}
for k, v in six.iteritems(r):
if isinstance(k, six.text_type):
k = k.encode(encoding)
if isinstance(v, six.text_type):
v = v.encode(encoding)
item[k] = v
encoded_resource.append(item)
resource = encoded_resource
if count is None:
count = 4
if fields is None:
fields = '(name, quote, id)'
fmt = style + attributes + fields
printer = resource_printer.Printer(fmt)
if heading:
printer.AddHeading(heading)
if count is not None:
resource = resource[:count]
for record in resource:
printer.AddRecord(record)
printer.Finish()
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
region = properties.VALUES.compute.region.Get(required=True)
staging_location = jobs_prep.GetStagingLocation(
staging_bucket=args.staging_bucket,
job_id=args.job,
job_dir=args.job_dir)
try:
uris = jobs_prep.UploadPythonPackages(
packages=args.packages,
package_path=args.package_path,
staging_location=staging_location)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If local packages are provided, the `--staging-bucket` or '
'`--job-dir` flag must be given.')
log.debug('Using {0} as trainer uris'.format(uris))
scale_tier_enum = (jobs.GetMessagesModule(
).GoogleCloudMlV1beta1TrainingInput.ScaleTierValueValuesEnum)
scale_tier = scale_tier_enum(
args.scale_tier) if args.scale_tier else None
job = jobs.BuildTrainingJob(
path=args.config,
module_name=args.module_name,
job_name=args.job,
trainer_uri=uris,
region=region,
job_dir=args.job_dir.ToUrl() if args.job_dir else None,
scale_tier=scale_tier,
user_args=args.user_args,
runtime_version=args.runtime_version)
jobs_client = jobs.JobsClient()
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs_client.Create(project_ref, job)
log.status.Print('Job [{}] submitted successfully.'.format(job.jobId))
if args. async:
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = stream.LogFetcher(
filters=log_utils.LogFilters(job.jobId),
polling_interval=_POLLING_INTERVAL,
continue_func=log_utils.MakeContinueFunction(job.jobId))
printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err)
def _CtrlCHandler(signal, frame):
del signal, frame # Unused
raise KeyboardInterrupt
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
printer.Print(log_utils.SplitMultiline(
log_fetcher.YieldLogs()))
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job_ref = resources.REGISTRY.Parse(job.jobId,
collection='ml.projects.jobs')
job = jobs_client.Get(job_ref)
# If the job itself failed, we will return a failure status.
if job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
return job
def SubmitTraining(jobs_client,
job,
job_dir=None,
staging_bucket=None,
packages=None,
package_path=None,
scale_tier=None,
config=None,
module_name=None,
runtime_version=None,
async_=None,
user_args=None):
"""Submit a training job."""
region = properties.VALUES.compute.region.Get(required=True)
staging_location = jobs_prep.GetStagingLocation(
staging_bucket=staging_bucket, job_id=job, job_dir=job_dir)
try:
uris = jobs_prep.UploadPythonPackages(
packages=packages,
package_path=package_path,
staging_location=staging_location)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If local packages are provided, the `--staging-bucket` or '
'`--job-dir` flag must be given.')
log.debug('Using {0} as trainer uris'.format(uris))
scale_tier_enum = jobs_client.training_input_class.ScaleTierValueValuesEnum
scale_tier = scale_tier_enum(scale_tier) if scale_tier else None
job = jobs_client.BuildTrainingJob(
path=config,
module_name=module_name,
job_name=job,
trainer_uri=uris,
region=region,
job_dir=job_dir.ToUrl() if job_dir else None,
scale_tier=scale_tier,
user_args=user_args,
runtime_version=runtime_version)
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs_client.Create(project_ref, job)
log.status.Print('Job [{}] submitted successfully.'.format(job.jobId))
if async_:
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = stream.LogFetcher(
filters=log_utils.LogFilters(job.jobId),
polling_interval=_POLLING_INTERVAL,
continue_func=log_utils.MakeContinueFunction(job.jobId))
printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err)
with execution_utils.RaisesKeyboardInterrupt():
try:
printer.Print(log_utils.SplitMultiline(log_fetcher.YieldLogs()))
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.\n')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
except exceptions.HttpError as err:
log.status.Print('Polling logs failed:\n{}\n'.format(str(err)))
log.info('Failure details:', exc_info=True)
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job_ref = resources.REGISTRY.Parse(job.jobId,
collection='ml.projects.jobs')
job = jobs_client.Get(job_ref)
return job
def Printer(self, fmt='config'):
self._printer = resource_printer.Printer(fmt)
return self._printer
def SetUp(self):
self._printer = resource_printer.Printer('yaml')
self.SetEncoding('utf-8')
def SetUp(self):
self.SetEncoding('utf-8')
self._printer = resource_printer.Printer('flattened')
def SubmitTraining(jobs_client,
job,
job_dir=None,
staging_bucket=None,
packages=None,
package_path=None,
scale_tier=None,
config=None,
module_name=None,
runtime_version=None,
python_version=None,
stream_logs=None,
user_args=None,
labels=None,
custom_train_server_config=None):
"""Submit a training job."""
region = properties.VALUES.compute.region.Get(required=True)
staging_location = jobs_prep.GetStagingLocation(
staging_bucket=staging_bucket, job_id=job, job_dir=job_dir)
try:
uris = jobs_prep.UploadPythonPackages(
packages=packages,
package_path=package_path,
staging_location=staging_location)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If local packages are provided, the `--staging-bucket` or '
'`--job-dir` flag must be given.')
log.debug('Using {0} as trainer uris'.format(uris))
scale_tier_enum = jobs_client.training_input_class.ScaleTierValueValuesEnum
scale_tier = scale_tier_enum(scale_tier) if scale_tier else None
try:
job = jobs_client.BuildTrainingJob(
path=config,
module_name=module_name,
job_name=job,
trainer_uri=uris,
region=region,
job_dir=job_dir.ToUrl() if job_dir else None,
scale_tier=scale_tier,
user_args=user_args,
runtime_version=runtime_version,
python_version=python_version,
labels=labels,
custom_train_server_config=custom_train_server_config)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If `--package-path` is not specified, at least one Python package '
'must be specified via `--packages`.')
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs_client.Create(project_ref, job)
if not stream_logs:
PrintSubmitFollowUp(job.jobId, print_follow_up_message=True)
return job
else:
PrintSubmitFollowUp(job.jobId, print_follow_up_message=False)
log_fetcher = stream.LogFetcher(
filters=log_utils.LogFilters(job.jobId),
polling_interval=properties.VALUES.ml_engine.polling_interval.GetInt(),
continue_interval=_CONTINUE_INTERVAL,
continue_func=log_utils.MakeContinueFunction(job.jobId))
printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err)
with execution_utils.RaisesKeyboardInterrupt():
try:
printer.Print(log_utils.SplitMultiline(log_fetcher.YieldLogs()))
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.\n')
log.status.Print(
_FOLLOW_UP_MESSAGE.format(job_id=job.jobId,
project=project_ref.Name()))
except exceptions.HttpError as err:
log.status.Print('Polling logs failed:\n{}\n'.format(
six.text_type(err)))
log.info('Failure details:', exc_info=True)
log.status.Print(
_FOLLOW_UP_MESSAGE.format(job_id=job.jobId,
project=project_ref.Name()))
job_ref = resources.REGISTRY.Parse(
job.jobId,
params={'projectsId': properties.VALUES.core.project.GetOrFail},
collection='ml.projects.jobs')
job = jobs_client.Get(job_ref)
return job
def SetUp(self):
self._printer = resource_printer.Printer(
'multi(metadata:format=json, networkInterfaces:format="table(name, '
'network, networkIP)")')