def Rewrite(expr):
"""Returns the server side rewrite of a --filter expression.
Args:
expr: A client side --filter expression.
Raises:
resource_exceptions.ExpressionSyntaxError: Expression syntax error.
ValueError: Invalid expression operands.
Returns:
The server side rewrite of a --filter expression, None if the expression is
completely client side.
"""
# Rewrite the expression.
rewrite = resource_filter.Compile(expr, backend=_Backend()).Rewrite()
# Add a project id restriction.
if rewrite:
rewrite += ' AND '
else:
rewrite = ''
rewrite += 'projectId={0}'.format(genomics_util.GetProjectId())
return rewrite
def TransformFilter(r, expression):
"""Selects elements of r that match the filter expression.
Args:
r: A JSON-serializable object.
expression: The filter expression to apply to r.
Returns:
The elements of r that match the filter expression.
Example:
x.filter("key:val") selects elements of r that have 'key' fields containing
'val'.
"""
# import loop
from googlecloudsdk.core.resource import resource_filter # pylint: disable=g-import-not-at-top
if not r:
return r
select = resource_filter.Compile(expression).Evaluate
if not resource_property.IsListLike(r):
return r if select(r) else ''
transformed = []
for item in r:
if select(item):
transformed.append(item)
return transformed
def Rewrite(self, expression, defaults=None):
"""Returns (frontend_expression, backend_expression) for expression.
There are 3 outcomes:
(None, backend) -- only need to apply the backend expression
(frontend, None) -- only need to apply the frontend expression
(frontend, backend) -- must apply both frontend and backend expressions
Args:
expression: The expression string to rewrite.
defaults: resource_projection_spec.ProjectionSpec defaults.
Returns:
Returns (frontend_expression, backend_expression) for expression.
"""
if defaults and defaults.symbols:
conditionals = defaults.symbols.get(
resource_transform.GetTypeDataName('conditionals'))
if hasattr(conditionals, 'flatten') and conditionals.flatten:
# If --flatten flag is presented we cannot do serverside filtering.
return expression, None
self.partial_rewrite = False
defaults = resource_projection_spec.ProjectionSpec(defaults=defaults)
defaults.symbols = _BelieveMe()
backend_expression = resource_filter.Compile(
expression, backend=self, defaults=defaults).Rewrite()
frontend_expression = expression if self.partial_rewrite else None
if frontend_expression and frontend_expression.isspace():
frontend_expression = None
return frontend_expression, backend_expression
def Run(self, args):
if args.json_file:
with open(args.json_file, 'r') as f:
resources = json.load(f)
else:
resources = json.load(sys.stdin)
# TODO(gsfowler): Drop this if when the --aggregate global flag lands.
if args.aggregate:
key = resource_lex.Lexer(args.aggregate).Key()
resources = Aggregator(resources, key)
# TODO(gsfowler): Return resources here when the --filter global flag lands.
if not args.format:
args.format = 'json'
if not args.filter:
return resources
select = resource_filter.Compile(args.filter).Evaluate
filtered_resources = []
if resource_property.IsListLike(resources):
for resource in resources:
if select(resource):
filtered_resources.append(resource)
elif select(resources):
# treat non-iterable resources as a list of length 1
filtered_resources.append(resources)
return filtered_resources
def __init__(self, expression, defaults):
"""Constructor.
Args:
expression: The resource filter expression string.
defaults: The resource format and filter default projection.
"""
self._match = resource_filter.Compile(
expression, defaults=defaults).Evaluate
def _ParseKey(self):
"""Parses a key and optional attributes from the expression.
The parsed key is appended to the ordered list of keys via _AddKey().
Transform functions and key attributes are also handled here.
Raises:
ExpressionSyntaxError: The expression has a syntax error.
"""
key, attribute = self._lex.KeyWithAttribute()
if self._lex.IsCharacter('(', eoi_ok=True):
add_transform = self._lex.Transform(key.pop(),
self._projection.active)
else:
add_transform = None
if (not self.__key_attributes_only
and attribute) or (self.__key_attributes_only and attribute
and not key):
# When a key is repeated in the format expression, we want to duplicate
# the attribute and add transfrom to the key, as the previous behaviour
# was. However _AddKey is also processed for attribute only keys; in this
# case, we want to reference the same attribute if the attribute is
# referenced by its label.
attribute = copy.copy(attribute)
else:
attribute = self._Attribute(self._projection.PROJECT)
if not attribute.transform:
attribute.transform = add_transform
elif add_transform:
# Compose the alias attribute.transform with add_transforms.
attribute.transform._transforms.extend(add_transform._transforms) # pylint: disable=protected-access
self._lex.SkipSpace()
if self._lex.IsCharacter(':'):
self._ParseKeyAttributes(key, attribute)
if attribute.transform and attribute.transform.conditional:
# Parse and evaluate if() transform conditional expression,
conditionals = self._projection.symbols.get(
resource_transform.GetTypeDataName('conditionals'))
def EvalGlobalRestriction(unused_obj, restriction, unused_pattern):
return getattr(conditionals, restriction, None)
defaults = resource_projection_spec.ProjectionSpec(
symbols={
resource_projection_spec.GLOBAL_RESTRICTION_NAME:
EvalGlobalRestriction
})
if not resource_filter.Compile(
attribute.transform.conditional,
defaults=defaults).Evaluate(conditionals):
return
if attribute.label is None and not key and attribute.transform:
attribute.label = self._AngrySnakeCase(
[attribute.transform.name] +
attribute.transform._transforms[0].args) # pylint: disable=protected-access
self._AddKey(key, attribute)
def __init__(self, expression, defaults):
"""Constructor.
Args:
expression: The resource filter expression string.
defaults: The resource format and filter default projection.
"""
self._compiled_expression = resource_filter.Compile(expression,
defaults=defaults)
self._missing_keys = resource_filter.GetAllKeys(
self._compiled_expression)
def Rewrite(self, expression, defaults=None):
"""Returns the rewrite of expression.
Args:
expression: The expression string to rewrite.
defaults: resource_projection_spec.ProjectionSpec defaults.
Returns:
The rewritten expression string, None if the rewrite collapses to nothing.
"""
return resource_filter.Compile(expression,
backend=self,
defaults=defaults).Rewrite()
def Run(self, args):
query = resource_filter.Compile(self._GetInventoryFilterQuery(args))
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
request_data = lister.ParseMultiScopeFlags(args, holder.resources)
list_implementation = lister.MultiScopeLister(
client,
zonal_service=client.apitools_client.instances,
aggregation_service=client.apitools_client.instances)
instances_iterator = lister.Invoke(request_data, list_implementation)
instances = list(instances_iterator)
responses = self._GetAllGuestInventoryGuestAttributes(holder, instances)
return self._GetInventoryFilteredInstances(instances, responses, query)
def RunSubTest(self, expression, deprecated=False):
def _Error(resource=None):
"""Always raises ValueError for testing.
Args:
resource: The resource object.
Raises:
ValueError: Always for testing.
"""
_ = resource
raise ValueError('Transform function value error.')
default_symbols = {
'date': resource_transform.TransformDate,
'len': lambda r, x=None: resource_transform.TransformLen(x or r),
}
defaults = resource_projection_parser.Parse(
'(compound.string:alias=s, floating:alias=f)',
symbols=default_symbols,
aliases=self._ALIASES)
symbols = {
'error': _Error, # 'error' not a magic name.
}
defaults = resource_projection_spec.ProjectionSpec(
defaults=defaults, symbols=symbols)
evaluate = resource_filter.Compile(expression, defaults=defaults).Evaluate
if isinstance(self.resource, list):
results = []
for r in self.resource:
results.append(evaluate(r))
return results
actual = evaluate(self.resource)
err = self.GetErr()
self.ClearErr()
warning = ('WARNING: --filter : operator evaluation is changing for '
'consistency across Google APIs.')
if err and not deprecated:
self.fail('Error [%s] not expected.' % err)
elif not err and deprecated:
self.fail('Warning [%s] expected.' % warning)
elif err and deprecated and warning not in err:
self.fail('Warning [%s] expected but got [%s].' % (warning, err))
return actual
def GetReferencedKeyNames(filter_string=None,
format_string=None,
printer=None,
defaults=None):
"""Returns the set of key names referenced by filter / format expressions.
NOTICE: OnePlatform is forgiving on filter and format key reference name
spelling. Use resource_property.GetMatchingIndex() when verifying against
resource dictionaries to handle camel and snake case spellings.
Args:
filter_string: The resource filter expression string.
format_string: The resource format expression string.
printer: The parsed format_string.
defaults: The resource format and filter default projection.
Raises:
ValueError: If both format_string and printer are specified.
Returns:
The set of key names referenced by filter / format expressions.
"""
keys = set()
# Add the format key references.
if printer:
if format_string:
raise ValueError('Cannot specify both format_string and printer.')
elif format_string:
printer = resource_printer.Printer(format_string, defaults=defaults)
defaults = printer.column_attributes
if printer:
for col in printer.column_attributes.Columns():
keys.add(resource_lex.GetKeyName(col.key, omit_indices=True))
# Add the filter key references.
if filter_string:
expr = resource_filter.Compile(filter_string,
defaults=defaults,
backend=resource_keys_expr.Backend())
for key in expr.Evaluate(None):
keys.add(resource_lex.GetKeyName(key, omit_indices=True))
return keys
def _ParseKey(self):
"""Parses a key and optional attributes from the expression.
Transform functions and key attributes are also handled here.
Raises:
ExpressionSyntaxError: The expression has a syntax error.
Returns:
The parsed key.
"""
key = self._lex.Key()
attribute = self._Attribute(self._projection.PROJECT)
if self._lex.IsCharacter('(', eoi_ok=True):
func_name = key.pop()
attribute.transform = self._lex.Transform(func_name,
self._projection.active)
func_name = attribute.transform.name
else:
func_name = None
self._lex.SkipSpace()
if self._lex.IsCharacter(':'):
self._ParseKeyAttributes(key, attribute)
if attribute.transform and attribute.transform.conditional:
# Parse and evaluate if() transform conditional expression,
conditionals = self._projection.symbols.get(
resource_transform.GetTypeDataName('conditionals'))
def EvalGlobalRestriction(unused_obj, restriction, unused_pattern):
return getattr(conditionals, restriction, None)
defaults = resource_projection_spec.ProjectionSpec(
symbols={
resource_projection_spec.GLOBAL_RESTRICTION_NAME:
EvalGlobalRestriction
})
if not resource_filter.Compile(
attribute.transform.conditional,
defaults=defaults).Evaluate(conditionals):
return
if func_name and attribute.label is None and not key:
attribute.label = self._AngrySnakeCase([func_name])
self._AddKey(key, attribute)
def Run(self, args):
"""Runs the command.
Args:
args: All the arguments that were provided to this command invocation.
Returns:
An iterator over Job messages.
"""
if args.filter:
filter_expr = resource_filter.Compile(args.filter)
filter_pred = lambda x: filter_expr.Evaluate(x) and _JobFilter(args
)(x)
else:
filter_pred = _JobFilter(args)
project_id = properties.VALUES.core.project.Get(required=True)
jobs = self._JobSummariesForProject(project_id, args, filter_pred)
return [job_display.DisplayInfo(job) for job in jobs]
def Rewrite(self, expression, defaults=None):
"""Returns (frontend_expression, backend_expression) for expression.
There are 3 outcomes:
(None, backend) -- only need to apply the backend expression
(frontend, None) -- only need to apply the frontend expression
(frontend, backend) -- must apply both frontend and backend expressions
Args:
expression: The expression string to rewrite.
defaults: resource_projection_spec.ProjectionSpec defaults.
Returns:
Returns (frontend_expression, backend_expression) for expression.
"""
self.partial_rewrite = False
backend_expression = resource_filter.Compile(
expression, backend=self, defaults=defaults).Rewrite()
frontend_expression = expression if self.partial_rewrite else None
return frontend_expression, backend_expression
def _ResourceSearchList(limit=None,
page_size=None,
query=None,
sort_by=None,
uri=False):
"""resource_search.list mock."""
del limit, page_size, sort_by, uri
results = None
select_resource = resource_filter.Compile(query).Evaluate
for name, value in six.iteritems(search_resources):
if select_resource({'@type': name}):
if results is None:
results = []
results += value
if results is None:
raise resource_search.CollectionNotIndexed(
'Collection [{}] not indexed for search.'.format(
query))
return results
def GetReferencedKeyNames(self):
"""Returns the list of key names referenced by the command."""
keys = set()
# Add the format key references.
self._InitPrinter()
if self._printer:
for col in self._printer.column_attributes.Columns():
keys.add(resource_lex.GetKeyName(col.key))
# Add the filter key references.
filter_expression = self._GetFlag('filter')
if filter_expression:
expr = resource_filter.Compile(filter_expression,
defaults=self._defaults,
backend=resource_keys_expr.Backend())
for key in expr.Evaluate(None):
keys.add(resource_lex.GetKeyName(key))
return keys
def GetAssetInventoryListInput(folder,
project,
org,
file_path=None,
asset_types_filter=None,
filter_expression=None,
krm_kind_filter=None):
"""Generate a AssetInventory export data set from api list call.
Calls AssetInventory List API via shared api client (AssetListClient) and
generates a list of exportable assets. If `asset_types_filter`,
`gvk_kind_filter` or `filter_expression` is passed, it will filter out
non-matching resources. If `file_path` is None list will be returned as a
string otherwise it is written to disk at specified path.
Args:
folder: string, folder parent for resource export.
project: string, project parent for resource export.
org: string, organization parent for resource export.
file_path: string, path to write AssetInventory export file to. If None,
results are returned as string.
asset_types_filter: [string], list of asset types to include in the output
file.
filter_expression: string, a valid gcloud filter expression. See `gcloud
topic filter` for more details.
krm_kind_filter: [string], list of KrmKinds corresponding to asset types to
include in the output.
Returns:
string: file path where AssetInventory data has been written or raw data if
`temp_file_path` is None. Returns None if no results returned from API.
Raises:
RequiredArgumentException: If none of folder, project or org is provided.
ResourceNotFoundException: If no resources are found or returned from
filtering.
ClientException: Writing file to disk.
"""
root_asset = asset_utils.GetParentNameForExport(
organization=org, project=project, folder=folder)
asset_client = client_util.AssetListClient(root_asset)
filter_func = (
resource_filter.Compile(filter_expression.strip()).Evaluate
if filter_expression else None)
asset_filter = asset_types_filter or []
if krm_kind_filter:
kind_filters = _BuildAssetTypeFilterFromKind(krm_kind_filter)
if not kind_filters:
raise ResourceNotFoundException(
'No matching resource types found for {}'.format(krm_kind_filter))
asset_filter.extend(kind_filters)
args = ApiClientArgs(
snapshot_time=None,
limit=None,
page_size=None,
content_type=None,
asset_types=sorted(asset_filter),
parent=root_asset,
filter_func=filter_func,
relationship_types=[])
asset_results = asset_client.List(args, do_filter=True)
asset_string_array = []
for item in asset_results: # list of apitools Asset messages.
item_str = encoding.MessageToJson(item)
item_str = item_str.replace('"assetType"', '"asset_type"')
asset_string_array.append(item_str)
if not asset_string_array:
if asset_types_filter:
asset_msg = '\n With resource types in [{}].'.format(asset_types_filter)
else:
asset_msg = ''
if filter_expression:
filter_msg = '\n Matching provided filter [{}].'.format(filter_expression)
else:
filter_msg = ''
raise ResourceNotFoundException(
'No matching resources found for [{parent}] {assets} {filter}'.format(
parent=root_asset, assets=asset_msg, filter=filter_msg))
if not file_path:
return '\n'.join(asset_string_array)
else:
try:
files.WriteFileAtomically(file_path, '\n'.join(asset_string_array))
except (ValueError, TypeError) as e:
raise ClientException(e) # pylint: disable=raise-missing-from
return file_path
