def __generate_index(ctx):
price_index = {}
for service, data in PRICE_INDEX_CONFIG.items():
print('Getting prices for %s...' % service)
price_index.setdefault(service, {})
price_data = requests.get(data['url']).json()
product_query = "products.* "
for k, v in data['attributes'].items():
if v.startswith('^'):
# do a starts with exp
product_query += " | [?starts_with(attributes.%s, '%s')]" % (k, v[1:])
else:
product_query += " | [?attributes.%s=='%s']" % (k, v)
products = jmespath.search(product_query, price_data)
for product in products:
instance_type = product['attributes']['instanceType']
if instance_type in price_index[service]:
raise RuntimeError("Duplicate instance type: " + instance_type)
price_query = ("terms.OnDemand.*.*[]"
"| [?sku=='%s'].priceDimensions.*[].pricePerUnit"
"| [0].USD"
) % product['sku']
price_index[service][instance_type] = float(jmespath.search(price_query, price_data))
with open('price_index.json', 'w') as f:
json.dump(price_index, f, indent=True)
def collect(query_args, putter_args, collector_name, db_conn=None):
'''does the collection, either puts the data in a table
or returns it in a dictarray to the caller (in case of real time collection)
'''
data = d_q(query_args)
if putter_args['base'] != '':
base = search(putter_args['base'], data)
else:
base = data
if putter_args['table_name'] == '@return':
#return a dictarray to be turned into a table
if type(base) is list:
vals = [{key: search(val, i) for key,val in putter_args['mapping'].items()} for i in base]
elif type(base) is dict:
vals = {key: search(val, base) for key,val in putter_args['mapping'].items()}
#print vals
return vals
else:
cursor = db_conn.cursor()
mk_table(putter_args, cursor)
db_conn.commit()
queries = []
if type(base) is list:
for i in base:
qry = grab(i,putter_args, collector_name)
queries.append(qry)
elif type(base) is dict:
queries.append(grab(base, putter_args, collector_name))
else:
raise Exception('Data Structure \'%s\' not recognised!' % type(base))
for qry in queries:
cursor.execute(qry)
db_conn.commit()
def github_repos(organization, github_url, github_token):
"""Return all github repositories in an organization."""
# Get github repos
headers = {"Authorization": "token {}".format(github_token)}
next_cursor = None
while next_cursor is not False:
params = {'query': query, 'variables': {
'organization': organization, 'cursor': next_cursor}}
response = requests.post(github_url, headers=headers, json=params)
result = response.json()
if response.status_code != 200 or 'errors' in result:
raise ValueError("Github api error %s" % (
response.content.decode('utf8'),))
repos = jmespath.search(
'data.organization.repositories.edges[].node', result)
for r in repos:
yield r
page_info = jmespath.search(
'data.organization.repositories.pageInfo', result)
if page_info:
next_cursor = (page_info['hasNextPage'] and
page_info['endCursor'] or False)
else:
next_cursor = False
def _get_values(record, field_list, tag_map):
tag_prefix = 'tag:'
list_prefix = 'list:'
count_prefix = 'count:'
vals = []
for field in field_list:
if field.startswith(tag_prefix):
tag_field = field.replace(tag_prefix, '', 1)
value = tag_map.get(tag_field, '')
elif field.startswith(list_prefix):
list_field = field.replace(list_prefix, '', 1)
value = jmespath.search(list_field, record)
if value is None:
value = ''
else:
value = ', '.join([str(v) for v in value])
elif field.startswith(count_prefix):
count_field = field.replace(count_prefix, '', 1)
value = jmespath.search(count_field, record)
if value is None:
value = ''
else:
value = str(len(value))
else:
value = jmespath.search(field, record)
if value is None:
value = ''
if not isinstance(value, six.text_type):
value = six.text_type(value)
vals.append(value)
return vals
def test_error_message_is_pluralized(self):
with self.assertRaises(exceptions.ArityError) as e:
jmespath.search('sort_by(@)', [0, 1])
exception = e.exception
self.assertEqual(
str(exception),
'Expected 2 arguments for function sort_by(), received 1')
def test_variadic_is_pluralized(self):
with self.assertRaises(exceptions.VariadictArityError) as e:
jmespath.search('not_null()', 'foo')
exception = e.exception
self.assertEqual(
str(exception),
'Expected at least 1 argument for function not_null(), received 0')
def test_singular_in_error_message(self):
with self.assertRaises(exceptions.ArityError) as e:
jmespath.search('length(@, @)', [0, 1])
exception = e.exception
self.assertEqual(
str(exception),
'Expected 1 argument for function length(), received 2')
def get_values(self):
contents, format = self.get_contents()
if format == 'json':
data = json.loads(contents)
if 'expr' in self.data:
res = jmespath.search(self.data['expr'], data)
if res is None:
log.warning('ValueFrom filter: %s key returned None' % self.data['expr'])
return res
elif format == 'csv' or format == 'csv2dict':
data = csv.reader(io.StringIO(contents))
if format == 'csv2dict':
data = {x[0]: list(x[1:]) for x in zip(*data)}
else:
if isinstance(self.data.get('expr'), int):
return [d[self.data['expr']] for d in data]
data = list(data)
if 'expr' in self.data:
res = jmespath.search(self.data['expr'], data)
if res is None:
log.warning('ValueFrom filter: %s key returned None' % self.data['expr'])
return res
return data
elif format == 'txt':
return [s.strip() for s in io.StringIO(contents).readlines()]
def test_can_provide_custom_functions(self):
class CustomFunctions(jmespath.functions.Functions):
@jmespath.functions.signature(
{'types': ['number']},
{'types': ['number']})
def _func_custom_add(self, x, y):
return x + y
@jmespath.functions.signature(
{'types': ['number']},
{'types': ['number']})
def _func_my_subtract(self, x, y):
return x - y
options = jmespath.Options(custom_functions=CustomFunctions())
self.assertEqual(
jmespath.search('custom_add(`1`, `2`)', {}, options=options),
3
)
self.assertEqual(
jmespath.search('my_subtract(`10`, `3`)', {}, options=options),
7
)
# Should still be able to use the original functions without
# any interference from the CustomFunctions class.
self.assertEqual(
jmespath.search('length(`[1, 2]`)', {}), 2
)
def parse_case(self, response):
'''
@url https://api.oyez.org/cases/2014/14-556
@returns requests 8 8
@returns items 0 0
'''
results = []
json_response = json.loads(response.body)
# load case
loader = CaseLoader(json_object=json_response)
case = loader.load_case_data()
case.send(self.session)
case_id = case['oyez_id']
# if there is explicit decision data, go and get it; otherwise add the case to the results
decision_url = jmespath.search('decisions[0].href', json_response)
if decision_url != None:
results.append(scrapy.Request(url=decision_url, callback=self.parse_decision, meta={'case': case}))
# if there is advocacy data, go and get it
advocate_links = jmespath.search('advocates[*].href', json_response)
if advocate_links == None:
advocate_links = []
for advocate_link in advocate_links:
results.append(scrapy.Request(url=advocate_link, callback=self.parse_advocate, meta={'case_id':case_id}))
# if there is transcript data, go and get it
transcript_links = jmespath.search('oral_argument_audio[*].href', json_response)
if transcript_links == None:
transcript_links = []
for transcript_link in transcript_links:
results.append(scrapy.Request(url=transcript_link, callback=self.parse_transcript, meta={'case_id':case_id}))
return results
def test_null_to_nonetype(self):
data = {
'a': {
'b': [1, 2, 3]
}
}
self.assertEqual(jmespath.search('length0(a.b)', data, self.options), 3)
self.assertEqual(jmespath.search('length0(a.c)', data, self.options), 0)
def _invoke_client_enum(self, client, enum_op, params, path):
if client.supports_pagination(enum_op):
results = []
for page in client.execute_paged_query(enum_op, params):
results.extend(jmespath.search(path, page))
return results
else:
return jmespath.search(path,
client.execute_query(enum_op, verb_arguments=params))
def test_get_one_node_info(self, fixture):
response = fixture.app.get('/api/nodes/%s/_info' % fixture.cluster_v6_name)
assert 200 == response.status_code
res = fixture.get_response_data(response)
node_id = list(jmespath.search('nodes', res['data'][0]).keys())[0]
response = fixture.app.get('/api/nodes/%s/%s/_info' % (fixture.cluster_v6_name, node_id))
res = fixture.get_response_data(response)
assert list(jmespath.search('nodes', res['data'][0]).keys())[0] == node_id
assert fixture.has_all_keys(fixture.config.KEYS_NODE_INFO, res['data'][0]['nodes'][node_id], excludes=['build', 'http_address']) is True
def test_type_error_messages(self):
with self.assertRaises(exceptions.JMESPathTypeError) as e:
jmespath.search('length(@)', 2)
exception = e.exception
# 1. Function name should be in error message
self.assertIn('length()', str(exception))
# 2. Mention it's an invalid type
self.assertIn('invalid type for value: 2', str(exception))
# 3. Mention the valid types:
self.assertIn("expected one of: ['string', 'array', 'object']",
str(exception))
# 4. Mention the actual type.
self.assertIn('received: "number"', str(exception))
def test_get_one_node_stats(self, fixture):
response = fixture.app.get('/api/nodes/%s/_stats' % fixture.cluster_v6_name)
assert 200 == response.status_code
res = fixture.get_response_data(response)
# get node ID:
node_id = list(jmespath.search('nodes', res['data'][0]).keys())[0]
response = fixture.app.get('/api/nodes/%s/%s/_stats' % (fixture.cluster_v6_name, node_id))
res = fixture.get_response_data(response)
assert list(jmespath.search('nodes', res['data'][0]).keys())[0] == node_id
assert fixture.has_all_keys(fixture.config.KEYS_NODE_STATS, res['data'][0]['nodes'][node_id]) is True
def apply_configuration(config):
"""
This function applies the simulations configurations
:param config the configuration file
"""
log_level = str(jp.search(struct.LOG_LEVEL, config)).upper()
log_level = 10 if log_level == 'DEBUG' else 20
configuration_parameter = {
'logging_level': log_level,
'clear_log_flag': jp.search(struct.CLEAR_OLD_LOGS, config)
}
configuration_parameter = convert_dict_values(configuration_parameter)
services.apply_configuration(services.ConfigurationParameters(**configuration_parameter))
def test_multiple_hosts(self, host, add_host):
# Add the rules
result = host.run(
'stack add host firewall frontend-0-0 backend-0-0 service=1234 '
'chain=INPUT action=ACCEPT protocol=TCP rulename=test'
)
assert result.rc == 0
# Make sure they are in the DB now
for hostname in ('frontend-0-0', 'backend-0-0'):
result = host.run(f'stack list host firewall {hostname} output-format=json')
assert result.rc == 0
rule = jmespath.search("[?name=='test']", json.loads(result.stdout))
assert rule == [{
'host': hostname,
'name': 'test',
'table': 'filter',
'service': '1234',
'protocol': 'TCP',
'chain': 'INPUT',
'action': 'ACCEPT',
'network': None,
'output-network': None,
'flags': None,
'comment': None,
'source': 'H',
'type': 'var'
}]
def format_resource(self, r):
details = {}
for k in r:
if isinstance(k, (list, dict)):
continue
details[k] = r[k]
for f in self.fields:
value = jmespath.search(f['expr'], r)
if not value:
continue
details[f['key']] = value
for k, v in details.items():
if isinstance(v, datetime):
v = v.isoformat()
elif isinstance(v, (list, dict)):
v = dumps(v)
elif isinstance(v, (int, float, bool)):
v = str(v)
else:
continue
details[k] = v
details['c7n:resource-type'] = self.manager.type
other = {
'Type': 'Other',
'Id': self.manager.get_arns([r])[0],
'Region': self.manager.config.region,
'Details': {'Other': filter_empty(details)}
}
tags = {t['Key']: t['Value'] for t in r.get('Tags', [])}
if tags:
other['Tags'] = tags
return other
def __init__(self, endpoint, data):
super(Table, self).__init__(endpoint, data)
self._id = data
detail_op, param_name, detail_path = self.Meta.detail_spec
params = {param_name: self.id}
data = endpoint.call(detail_op, **params)
self.data = jmespath.search(detail_path, data)
def validate_healthcheck(self, data):
for procType, healthcheck in data.items():
if healthcheck is None:
continue
for key, value in healthcheck.items():
if value is None:
continue
if key not in ['livenessProbe', 'readinessProbe']:
raise serializers.ValidationError(
"Healthcheck keys must be either livenessProbe or readinessProbe")
try:
jsonschema.validate(value, PROBE_SCHEMA)
except jsonschema.ValidationError as e:
raise serializers.ValidationError(
"could not validate {}: {}".format(value, e.message))
# http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_probe
# liveness only supports successThreshold=1, no other value
# This is not in the schema since readiness supports other values
threshold = jmespath.search('livenessProbe.successThreshold', healthcheck)
if threshold is not None and threshold != 1:
raise serializers.ValidationError(
'livenessProbe successThreshold can only be 1'
)
return data
def _extract_fields(source, mappings):
result = {}
for mapping in mappings:
result[mapping[1]] = jmespath.search(mapping[0], source)
return result
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
if not re.match(self._expected_result, str(actual_result), re.IGNORECASE):
raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result,
execution_result.output)
def get_trail_ids(cls, event, mode):
"""extract resources ids from a cloud trail event."""
resource_ids = ()
event_name = event['detail']['eventName']
event_source = event['detail']['eventSource']
for e in mode.get('events', []):
if not isinstance(e, dict):
# Check if we have a short cut / alias
info = CloudWatchEvents.match(event)
if info:
return info['ids'].search(event)
continue
if event_name != e.get('event'):
continue
if event_source != e.get('source'):
continue
id_query = e.get('ids')
if not id_query:
raise ValueError("No id query configured")
evt = event
# be forgiving for users specifying with details or without
if not id_query.startswith('detail.'):
evt = event.get('detail', {})
resource_ids = jmespath.search(id_query, evt)
if resource_ids:
break
return resource_ids
def test_all_parameters(self, host):
# Add the rule
result = host.run(
'stack add host firewall frontend-0-0 table=nat service=1234 '
'chain=INPUT action=ACCEPT protocol=TCP network=private '
'output-network=private flags=test_flag comment=test_comment '
'rulename=test'
)
assert result.rc == 0
# Make sure it is in the DB now
result = host.run('stack list host firewall frontend-0-0 output-format=json')
assert result.rc == 0
rule = jmespath.search("[?name=='test']", json.loads(result.stdout))
assert rule == [{
'host': 'frontend-0-0',
'name': 'test',
'table': 'nat',
'service': '1234',
'protocol': 'TCP',
'chain': 'INPUT',
'action': 'ACCEPT',
'network': 'private',
'output-network': 'private',
'flags': 'test_flag',
'comment': 'test_comment',
'source': 'H',
'type': 'var'
}]
def test_no_rulename(self, host):
# Add the rule
result = host.run(
'stack add host firewall frontend-0-0 service=1234 chain=INPUT '
'action=ACCEPT protocol=TCP network=private'
)
assert result.rc == 0
# Make sure it is in the DB now
result = host.run('stack list host firewall frontend-0-0 output-format=json')
assert result.rc == 0
rule = jmespath.search("[?service=='1234']", json.loads(result.stdout))
# Make sure our rule name was a UUID and then remove it for the match
assert re.match(
r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
rule[0]['name']
)
del rule[0]['name']
assert rule == [{
'host': 'frontend-0-0',
'table': 'filter',
'service': '1234',
'protocol': 'TCP',
'chain': 'INPUT',
'action': 'ACCEPT',
'network': 'private',
'output-network': None,
'flags': None,
'comment': None,
'source': 'H',
'type': 'var'
}]
def retrieve_candidate_values(self, service, operation, param):
if service not in self._services_with_completions:
return
# Example call:
# service='ec2', operation='terminate-instances',
# param='--instance-ids'.
# We need to convert this to botocore syntax.
# First try to load the resource model.
LOG.debug("Called with: %s, %s, %s", service, operation, param)
# Now convert operation to the name used by botocore.
client = self._get_client(service)
api_operation_name = client.meta.method_to_api_mapping.get(
operation.replace('-', '_'))
if api_operation_name is None:
return
# Now we need to convert the param name to the
# casing used by the API.
completer = self._get_completer_for_service(service)
result = completer.describe_autocomplete(
service, api_operation_name, param)
if result is None:
return
# DEBUG:awsshell.resource.index:RESULTS:
# ServerCompletion(service=u'ec2', operation=u'DescribeInstances',
# params={}, path=u'Reservations[].Instances[].InstanceId')
try:
response = getattr(client, xform_name(result.operation, '_'))()
except Exception:
return
results = jmespath.search(result.path, response)
return results
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
if not actual_result:
raise JMESPathCheckAssertionError(self._query, 'some value', actual_result,
execution_result.output)
def process(self, resources):
related_resources = dict(
zip(jmespath.search('[].%s' % self.data['key'], resources), resources))
related_ids = set(related_resources)
related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids)
missing_related_tags = related_ids.difference(related_tag_map.keys())
if not self.data.get('skip_missing', True) and missing_related_tags:
raise PolicyExecutionError(
"Unable to find all %d %s related resources tags %d missing" % (
len(related_ids), self.data['resource'], len(missing_related_tags)))
# rely on resource manager tag action implementation as it can differ between resources
tag_action = self.manager.action_registry.get('tag')({}, self.manager)
tag_action.id_key = tag_action.manager.get_model().id
client = tag_action.get_client()
stats = Counter()
for related, r in related_resources.items():
if related in missing_related_tags or not related_tag_map[related]:
stats['missing'] += 1
elif self.process_resource(
client, r, related_tag_map[related], self.data['tags'], tag_action):
stats['tagged'] += 1
else:
stats['unchanged'] += 1
self.log.info(
'Tagged %d resources from related, missing-skipped %d unchanged %d',
stats['tagged'], stats['missing'], stats['unchanged'])
def process(self, resources, event=None):
ec2 = local_session(self.manager.session_factory).client('ec2')
cidrs = jmespath.search(
"PrefixLists[].Cidrs[]", ec2.describe_prefix_lists())
cidrs = [parse_cidr(cidr) for cidr in cidrs]
results = []
check_egress = self.data.get('egress', True)
check_ingress = self.data.get('ingress', True)
present = self.data.get('present', False)
for r in resources:
matched = {cidr: None for cidr in cidrs}
for entry in r['Entries']:
if entry['Egress'] and not check_egress:
continue
if not entry['Egress'] and not check_ingress:
continue
entry_cidr = parse_cidr(entry['CidrBlock'])
for c in matched:
if c in entry_cidr and matched[c] is None:
matched[c] = (
entry['RuleAction'] == 'allow' and True or False)
if present and all(matched.values()):
results.append(r)
elif not present and not all(matched.values()):
results.append(r)
return results
def __init__(self, client, data, query=None):
super(Table, self).__init__(client, data, query)
self._id = data
detail_op, param_name, detail_path = self.Meta.detail_spec
params = {param_name: self.id}
data = client.call(detail_op, **params)
self.data = jmespath.search(detail_path, data)
def _operation_type(cls, message_data):
''' We only care about _events_ that alter assets. Maintaining this
list is going to be annoying. Long term, this entire class should be
replaced when google provides a real-time event delivery solution '''
method_name = jmespath.search('protoPayload.methodName', message_data) or ''
last = method_name.split('.')[-1].lower()
# For batch methods, look for the verb after the word 'batch'
if last.startswith('batch'):
last = last[5:]
read_prefixes = ('get', 'list')
if last.startswith(read_prefixes):
return 'read'
write_prefixes = (
'create',
'update',
'insert',
'patch',
'set',
'debug',
'enable',
'disable',
'expand',
'deactivate',
'activate'
)
if last.startswith(write_prefixes):
return 'write'
delete_prefixes = ('delete')
if last.startswith(delete_prefixes):
return 'delete'
else:
return 'unknown'
def list_docker_images():
extra_globals = {
"global.baseDomain": "foo.com",
"blackboxExporterEnabled": True,
"postgresqlEnabled": True,
"prometheusPostgresExporterEnabled": True,
"pspEnabled": True,
"veleroEnabled": True,
}
public_repo_docs = render_chart(values={"global": extra_globals})
# Listing docker images
search_string = "spec.template.spec.containers[*].image"
docker_images = []
for public_repo_doc in public_repo_docs:
docker_image_sublist = jmespath.search(search_string, public_repo_doc)
if docker_image_sublist is not None:
docker_images = docker_image_sublist + docker_images
return docker_images
def list_spot_requests(ec2_conn):
"""Returns a list of object-like spot instance requests
# sir.status:
# open The request is waiting to be fulfilled.
# active The request is fulfilled and has an associated Instance
# failed The request has one or more bad parameters.
# closed The Spot Instance was interrupted or terminated.
# cancelled You cancelled the request, or the request expired.
"""
spot_instances = list_instances(ec2_conn,
spot=True,
state=["running", "terminated"])
sirs = ec2_conn.describe_spot_instance_requests()['SpotInstanceRequests']
requests = []
for req in sirs:
try:
launch, state, hostname = jmespath.search(
"[?SpotInstanceRequestId=='%s'][][LaunchTime,State.Name,"
"Tags[?Key=='Name'].Value|[0]]" % req['SpotInstanceRequestId'],
spot_instances)[0]
except IndexError:
launch, state, hostname = list('-' * 3)
launch_specification = req['LaunchSpecification']
sir = types.SimpleNamespace(
ami_id=launch_specification['ImageId'],
az=req['LaunchedAvailabilityZone'],
status_code=req['Status']["Code"],
status_timestamp=req['Status']["UpdateTime"],
hostname=hostname,
id=req['SpotInstanceRequestId'],
instance_id=req.get('InstanceId'),
instance_status=state,
instance_type=launch_specification['InstanceType'],
instance_launch=launch,
state=req['State'],
subnet=launch_specification['NetworkInterfaces'][0]['SubnetId'],
duration=req.get('BlockDurationMinutes'))
requests.append(sir)
return requests
def test_deregister_delete_snaps(self):
factory = self.replay_flight_data('test_ami_deregister_delete_snap')
p = self.load_policy(
{
'name': 'deregister-snap',
'resource': 'ami',
'actions': [{
'type': 'deregister',
'delete-snapshots': True
}]
},
session_factory=factory,
config={'account_id': '644160558196'})
resources = p.run()
self.assertEqual(len(resources), 1)
client = factory().client('ec2')
snap_ids = jmespath.search('BlockDeviceMappings[].Ebs.SnapshotId',
resources[0])
self.assertRaises(ClientError,
client.describe_snapshots,
SnapshotIds=snap_ids,
OwnerIds=['self'])
def _build_url(self, resource):
"""
Compose URL with query string parameters.
Will not lose existing static parameters in the URL string
but does not support 'duplicate' parameter entries
"""
if not self.query_params:
return self.url
evaluated_params = {
k: jmespath.search(v, resource)
for k, v in self.query_params.items()
}
url_parts = list(parse.urlparse(self.url))
query = dict(parse.parse_qsl(url_parts[4]))
query.update(evaluated_params)
url_parts[4] = parse.urlencode(query)
return parse.urlunparse(url_parts)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('expression')
parser.add_argument('-f', '--filename',
help=('The filename containing the input data. '
'If a filename is not given then data is '
'read from stdin.'))
parser.add_argument('--ast', action='store_true',
help=('Pretty print the AST, do not search the data.'))
args = parser.parse_args()
expression = args.expression
if args.ast:
# Only print the AST
expression = jmespath.compile(args.expression)
sys.stdout.write(pformat(expression.parsed))
sys.stdout.write('\n')
return 0
if args.filename:
with open(args.filename, 'r') as f:
data = json.load(f)
else:
data = sys.stdin.read()
data = json.loads(data)
try:
sys.stdout.write(json.dumps(
jmespath.search(expression, data), indent=4))
sys.stdout.write('\n')
except exceptions.ArityError as e:
sys.stderr.write("invalid-arity: %s\n" % e)
return 1
except exceptions.JMESPathTypeError as e:
sys.stderr.write("invalid-type: %s\n" % e)
return 1
except exceptions.UnknownFunctionError as e:
sys.stderr.write("unknown-function: %s\n" % e)
return 1
except exceptions.ParseError as e:
sys.stderr.write("syntax-error: %s\n" % e)
return 1
def test_kafka_cluster_kms_filter(self):
session_factory = self.replay_flight_data('test_kafka_cluster_kms_filter')
kms = session_factory().client('kms')
expression = 'EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId'
p = self.load_policy(
{
'name': 'kafka-kms-filter',
'resource': 'kafka',
'filters': [
{
'type': 'kms-key',
'key': 'c7n:AliasName',
'value': 'alias/aws/kafka'
}
]
},
session_factory=session_factory
)
resources = p.run()
self.assertTrue(len(resources), 1)
aliases = kms.list_aliases(KeyId=(jmespath.search(expression, resources[0])))
self.assertEqual(aliases['Aliases'][0]['AliasName'], 'alias/aws/kafka')
def extract_tag_pageing_api(self, response_json, playlist_url):
results = []
itemList = jmespath.search("itemList[].data", response_json)
has_more = True if jmespath.search("nextPageUrl",
response_json) else False
for ele in itemList:
data = jmespath.search("content.data", ele)
if data is None:
data = ele
try:
data['title']
except:
continue
tag = jmespath.search("tags[].name", data)
try:
upload_ts = data['releaseTime'] // 1000
except:
upload_ts = None
try:
result = {
"cover": jmespath.search("cover.feed", data),
"play_addr": jmespath.search("playUrl", data),
"tag": tag,
"category": data['category'],
"description": data['description'],
"title": data['title'],
"author": data['author']['name'],
"author_id": data['author']['id'],
"upload_ts": upload_ts,
"vid": data['id'],
"duration": data['duration'],
"from": self.from_,
"playlist_url": playlist_url,
"webpage_url": jmespath.search("webUrl.forWeibo", data),
}
except:
traceback.print_exc()
continue
results.append(result)
else:
return results, has_more, {}
def test_externalelasticsearch_with_awsServiceAccountAnnotation(self, kube_version):
"""Test External ElasticSearch with eks iam roles passed as Service Account Annotation."""
docs = render_chart(
kube_version=kube_version,
values={
"global": {
"customLogging": {
"enabled": True,
"scheme": "https",
"host": "esdemo.example.com",
"awsServiceAccountAnnotation": "arn:aws:iam::xxxxxxxx:role/customrole",
}
}
},
show_only=[
"charts/external-es-proxy/templates/external-es-proxy-deployment.yaml",
"charts/external-es-proxy/templates/external-es-proxy-serviceaccount.yaml",
"charts/external-es-proxy/templates/external-es-proxy-service.yaml",
],
)
assert len(docs) == 3
doc = docs[0]
expected_env = [{"name": "ENDPOINT", "value": "https://esdemo.example.com"}]
assert expected_env == doc["spec"]["template"]["spec"]["containers"][1]["env"]
doc = docs[1]
assert "arn:aws:iam::xxxxxxxx:role/customrole" == jmespath.search(
'metadata.annotations."eks.amazonaws.com/role-arn"', doc
)
assert "Service" == jmespath.search("kind", docs[2])
assert "release-name-external-es-proxy" == jmespath.search(
"metadata.name", docs[2]
)
assert "ClusterIP" == jmespath.search("spec.type", docs[2])
assert {
"name": "secure-http",
"protocol": "TCP",
"port": 9200,
} in jmespath.search("spec.ports", docs[2])
assert {"name": "http", "protocol": "TCP", "port": 9201} in jmespath.search(
"spec.ports", docs[2]
)
def format_resource(self, r):
details = {}
for k in r:
if isinstance(k, (list, dict)):
continue
details[k] = r[k]
for f in self.fields:
value = jmespath.search(f['expr'], r)
if not value:
continue
details[f['key']] = value
for k, v in details.items():
if isinstance(v, datetime):
v = v.isoformat()
elif isinstance(v, (list, dict)):
v = dumps(v)
elif isinstance(v, (int, float, bool)):
v = str(v)
else:
continue
details[k] = v[:SECHUB_VALUE_SIZE_LIMIT]
details['c7n:resource-type'] = self.manager.type
other = {
'Type': self.resource_type,
'Id': self.manager.get_arns([r])[0],
'Region': self.manager.config.region,
'Partition': get_partition(self.manager.config.region),
'Details': {
self.resource_type: filter_empty(details)
}
}
tags = {t['Key']: t['Value'] for t in r.get('Tags', [])}
if tags:
other['Tags'] = tags
return other
def test_asg_propagate_tag_action(test, aws_asg):
factory = test.replay_flight_data('test_asg_propagate_tag_action')
p = test.load_policy(
{
'name':
'asg-tagger',
'resource':
'aws.asg',
'filters': [{
'AutoScalingGroupName':
aws_asg['aws_autoscaling_group.bar.id']
}, {
'tag:Owner': 'absent'
}],
'actions': [{
'type': 'tag',
'key': 'Owner',
'value': 'Kapil',
'propagate': True
}, {
'type': 'propagate-tags',
'tags': ['Owner']
}]
},
session_factory=factory)
resources = p.run()
test.assertEqual(len(resources), 1)
client = factory().client('ec2')
itags = {
t['Key']: t['Value']
for t in jmespath.search(
'Reservations[0].Instances[0].Tags',
client.describe_instances(
InstanceIds=[resources[0]['Instances'][0]['InstanceId']]))
}
assert 'Owner' in itags
assert itags['Owner'] == 'Kapil'
def load_expected_data(source_file_name, dir_path):
expected_path = os.path.join(dir_path, source_file_name)
if not os.path.exists(expected_path):
return None
with open(expected_path, "r") as f:
expected_data = json.load(f)
# Convert to absolute path: "buckets/bucket.tf[main.tf#0]"
# ^^^^^^^^^^^^^^^^^ ^^^^^^^
# HERE & HERE
#
resolved_pattern = re.compile(r"(.+)\[(.+)#(\d+)]") # groups: location (1), referrer (2), index (3)
# Expected files should have the filenames relative to their base directory, but the parser will
# use the absolute path. This loop with replace relative filenames with absolute.
keys = list(expected_data.keys())
for key in keys:
# NOTE: Sometimes keys have module referrers, sometimes they don't
match = resolved_pattern.match(key)
if match:
new_key = _make_module_ref_absolute(match, dir_path)
else:
if os.path.isabs(key):
continue
new_key = os.path.join(dir_path, key)
expected_data[new_key] = expected_data[key]
del expected_data[key]
for resolved_list in jmespath.search("*.module[].*[].__resolved__", expected_data):
for list_index in range(0, len(resolved_list)):
match = resolved_pattern.match(resolved_list[list_index])
assert match is not None, f"Unexpected module resolved data: {resolved_list[list_index]}"
resolved_list[list_index] = _make_module_ref_absolute(match, dir_path)
# print(f"{match[0]} -> {resolved_list[list_index]}")
return expected_data
def build_iterator(self, **kwargs) -> List:
results = []
for feed_name, feed in self.feed_name_to_config.items():
r = requests.get(url=feed.get('url', self.url),
verify=self.verify,
auth=self.auth,
cert=self.cert,
headers=self.headers,
**kwargs)
try:
r.raise_for_status()
data = r.json()
result = jmespath.search(expression=feed.get('extractor'),
data=data)
results.append({feed_name: result})
except ValueError as VE:
raise ValueError(
f'Could not parse returned data to Json. \n\nError massage: {VE}'
)
return results
def should_notify(replica: Replica, subscription: dict, metadata_document: dict, key: str) -> bool:
"""
Check if a notification should be attempted for subscription and key
"""
jmespath_query = subscription.get(SubscriptionData.JMESPATH_QUERY)
if not jmespath_query:
return True
else:
try:
if jmespath.search(jmespath_query, metadata_document):
return True
else:
return False
except JMESPathError:
logger.error("jmespath query failed for owner={} replica={} uuid={} jmespath_query='{}' key={}".format(
subscription[SubscriptionData.OWNER],
subscription[SubscriptionData.REPLICA],
subscription[SubscriptionData.UUID],
subscription[SubscriptionData.JMESPATH_QUERY],
key
))
return False
def restore_security_group(instance_id, instance_to_mimic_id):
try:
client = boto3.client('rds')
response = client.describe_db_instances(
DBInstanceIdentifier=instance_to_mimic_id)
security_group_groups = jmespath.search(
"DBInstances[*].VpcSecurityGroups[*].VpcSecurityGroupId", response)
# the outer group should correspond to the unique instance specified, so should have
# length==1
if len(security_group_groups) == 1:
security_group_ids = security_group_groups[0]
client.modify_db_instance(DBInstanceIdentifier=instance_id,
VpcSecurityGroupIds=security_group_ids)
except botocore.exceptions.ClientError as e:
print("Could not add security group(s) to %s. Error %s" %
(instance_id, str(e)))
def test_keda_enabled(self, executor, is_created):
"""
ScaledObject should only be created when set to enabled and executor is Celery or CeleryKubernetes
"""
docs = render_chart(
values={
"workers": {
"keda": {
"enabled": True
},
"persistence": {
"enabled": False
}
},
'executor': executor,
},
show_only=["templates/workers/worker-kedaautoscaler.yaml"],
)
if is_created:
assert jmespath.search("metadata.name",
docs[0]) == "RELEASE-NAME-worker"
else:
assert docs == []
def retrieve_os_users(client, instances):
ec2 = client
ami_query = jmespath.search(
"Images[].{id: ImageId, name: Name, loc: ImageLocation}",
ec2.describe_images(
ImageIds=[instance.get("ami") for instance in instances]),
)
ami_query = {ami_dic["id"]: ami_dic for ami_dic in ami_query}
instance_dics = []
for instance in instances:
ami = instance["ami"]
ami_info = ami_query.get(ami)
os_user = _osuser_from_ami(ami_info)
instance["os_user"] = os_user
instance_dics.append(instance)
return instance_dics
def valid_available_instances(client):
ec2 = client
inst_dict = jmespath.search(
"Reservations[].Instances[].{name: Tags[?Key=='Name'].Value | [0], id: InstanceId, avz: Placement.AvailabilityZone, ami: ImageId, ip: PublicIpAddress}",
ec2.describe_instances(),
)
if inst_dict is None:
return
has_name, no_name = partition(
lambda dic: dic.get("name") is None or dic.get("name") == "",
inst_dict)
no_whitespace, whitespace = partition(lambda dic: " " in dic.get("name"),
has_name)
valid_instances = list(no_whitespace)
invalid_instances = list(whitespace) + list(no_name)
return valid_instances, invalid_instances
def get_tmpl_context(ml, fname):
if fname not in datadict:
objs = cldf.objects(table_map[fname]) \
if table_map[fname] else cldf.iter_rows(fname, 'id')
datadict[fname] = {
r.id if isinstance(r, orm.Object) else r['id']: r
for r in objs
}
tmpl_context = {
k: True if v[0] == '' else v[0]
for k, v in ml.parsed_url_query.items()
}
for k in tmpl_context:
if k.startswith('with_') and (tmpl_context[k]
in ['0', 'false', 'False']):
tmpl_context[k] = False
if table_map[fname] == 'Metadata':
tmpl_context['ctx'] = jmespath.search(ml.objid, datadict[fname])
else:
tmpl_context['ctx'] = list(datadict[fname].values()) \
if ml.all else datadict[fname][ml.objid]
tmpl_context['cldf'] = cldf
return tmpl_context
def verify_transform(subject, mapping):
"""
Determines if a key from mapping exists in subject and if so
verifies that subject[k] is of type mapping[k]
"""
import jmespath
for k in mapping.keys():
result = jmespath.search(k, subject)
if result is None:
raise AttributeError('The property "{}" is required'.format(k))
if not isinstance(result, mapping[k]):
supplemental_info = ""
if mapping[k] == dict:
wiki_link = "https://github.com/Azure/azure-iot-cli-extension/wiki/Tips"
supplemental_info = "Review inline JSON examples here --> {}".format(
wiki_link)
raise TypeError(
'The property "{}" must be of {} but is {}. Input: {}. {}'.
format(k, str(mapping[k]), str(type(result)), result,
supplemental_info))
def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(
self):
docs = render_chart(
values={
"dags": {
"persistence": {
"enabled": True,
"existingClaim": "test-claim"
}
}
},
show_only=["templates/pod-template-file.yaml"],
)
self.assertIn(
{
"name": "dags",
"persistentVolumeClaim": {
"claimName": "test-claim"
}
},
jmespath.search("spec.volumes", docs[0]),
)
def build_identifiers(identifiers_def, parent, params, raw_response):
"""
Builds a mapping of identifier names to values based on the
identifier source location, type, and target. Identifier
values may be scalars or lists depending on the source type
and location.
:type identifiers_def: list
:param identifiers_def: List of identifier definitions
:type parent: ServiceResource
:param parent: The resource instance to which this action is attached.
:type params: dict
:param params: Request parameters sent to the service.
:type response: dict
:param response: Low-level operation response.
"""
results = {}
for identifier in identifiers_def:
source = identifier.get('source', '')
source_type = identifier.get('sourceType')
target = identifier.get('target')
if source_type == 'responsePath':
value = jmespath.search(source, raw_response)
results[xform_name(target)] = value
elif source_type in ['identifier', 'dataMember']:
value = getattr(parent, xform_name(source))
results[xform_name(target)] = value
elif source_type == 'requestParameter':
value = params[source]
results[xform_name(target)] = value
else:
raise NotImplementedError(
'Unsupported source type: {0}'.format(source_type))
return results
def recurse_access_key(data, query):
"""
Search for something in the given data using the given query.
Example:
>>> recurse_access_key({'a': 'b'}, 'a')
'b'
>>> recurse_access_key({'a': {'b': ['c', 'd']}}, 'a.b[0]')
'c'
Args:
data (dict, list): Data to search in
query (str): Query to run
Returns:
object: Whatever was found by the search
"""
try:
from_jmespath = jmespath.search(query, data)
except jmespath.exceptions.ParseError as e:
logger.error("Error parsing JMES query")
try:
_deprecated_recurse_access_key(data, query.split("."))
except (IndexError, KeyError):
logger.debug("Nothing found searching using old method")
else:
# If we found a key using 'old' style searching
logger.warning(
"Something was found using 'old style' searching in the response - please change the query to use jmespath instead - see http://jmespath.org/ for more information"
)
raise exceptions.JMESError("Invalid JMES query") from e
return from_jmespath
def get_regions(profile=None, region=None):
while True:
try:
session = boto3.Session(profile_name=profile)
response = session.client('ec2', region_name=region,
config=config).describe_regions()
return jmespath.search('Regions[].RegionName', response)
except awsClientError as error:
if error.response['Error']['Code'] in [
"RequestTimeout", "RequestTimeoutException",
"PriorRequestNotComplete", "ConnectionError",
"HTTPClientError", "Throttling", "ThrottlingException",
"ThrottledException", "RequestThrottledException",
"TooManyRequestsException",
"ProvisionedThroughputExceededException",
"TransactionInProgressException", "RequestLimitExceeded",
"BandwidthLimitExceeded", "LimitExceededException",
"RequestThrottled", "SlowDown", "EC2ThrottledException"
]:
logger.error(
"Encountered Error: {} !! Sleeping for 20 seconds".format(
error.response['Error']['Code']))
logger.info(traceback.format_exc())
time.sleep(20)
elif error.response['Error']['Code'] in ["UnauthorizedOperation"]:
logger.error(
"Encountered Error: {} !! Sleeping for 20 seconds".format(
error.response['Error']['Code']))
logger.info(traceback.format_exc())
region = "ap-south-1"
time.sleep(20)
else:
logger.error(
"Encountered Error: {} !! Ommitting region {} for profile {}"
.format(error.response['Error']['Code'], region, profile))
logger.info(traceback.format_exc())
return []
def process(self, resources):
related_resources = list(
zip(
jmespath.search(
'[].[%s || "c7n:NotFound"]|[]' % self.data['key'],
resources), resources))
related_ids = set([r[0] for r in related_resources])
related_ids.discard('c7n:NotFound')
related_tag_map = self.get_resource_tag_map(self.data['resource'],
related_ids)
missing_related_tags = related_ids.difference(related_tag_map.keys())
if not self.data.get('skip_missing', True) and missing_related_tags:
raise PolicyExecutionError(
"Unable to find all %d %s related resources tags %d missing" %
(len(related_ids), self.data['resource'],
len(missing_related_tags)))
# rely on resource manager tag action implementation as it can differ between resources
tag_action = self.manager.action_registry.get('tag')({}, self.manager)
tag_action.id_key = tag_action.manager.get_model().id
client = tag_action.get_client()
stats = Counter()
for related, r in related_resources:
if related in missing_related_tags or not related_tag_map[related]:
stats['missing'] += 1
elif self.process_resource(client, r, related_tag_map[related],
self.data['tags'], tag_action):
stats['tagged'] += 1
else:
stats['unchanged'] += 1
self.log.info(
'Tagged %d resources from related, missing-skipped %d unchanged %d',
stats['tagged'], stats['missing'], stats['unchanged'])
def save_response(test_executor: TestExecutor, r, response_dict):
"""
processes the http request to that mentioned in YAML
:param test_executor: The TestExecutor object to give context for execution
:type test_executor: TestExecutor
:type r: request
:param response_dict: response dict fetched from YAML
:type response_dict: dict
:returns; Status of Execution and error stack
"""
try:
logger.info(r.json())
if 'save' in response_dict:
for entry in response_dict['save']:
b = entry['save in']
# if key is not present in dict, whole json will be saved to the 'save in' parameter
if 'key' in entry:
test_executor.variable_dictionary[b] = jmespath.search(
entry['key'], r.json())
else:
test_executor.variable_dictionary[b] = r.text
logger.info(
str(test_executor.variable_dictionary) +
" new key, value pairs added")
except JSONDecodeError:
logger.info(r.text)
# save the text response in the variable_dict
if 'save' in response_dict:
for entry in response_dict['save']:
b = entry['save in']
test_executor.variable_dictionary[b] = r.text
logger.info(
str(test_executor.variable_dictionary) +
" new (key,value) pairs added;"
" value contains the whole text received")
return True, None
def _process_for_equipment_reports(self, data, stop_area_equipments_list):
"""
For each stop_area_equipments within equipment_reports response, the structure 'equipment_details' is updated if the corresponding code is present
:param data: equipments data received from the webservice
:param stop_area_equipments_list: list of stop_area_equipments from the protobuf response
"""
for sae in stop_area_equipments_list:
"""
Codes might be duplicated across different stop points.
Because we report equipments on a stop area basis, we don't want them duplicated
"""
unique_codes = {
six.text_type(code): code
for st in sae.stop_area.stop_points for code in st.codes
}
for code in unique_codes.values():
if code.type in self.code_types:
equipments_list = jmespath.search(
"equipments_details[?id=='{}']".format(code.value),
data)
if equipments_list:
equipment = equipments_list[0]
# Fill PB
equipment_details = sae.equipment_details.add()
self._fill_equipment_details(
equipment_form_web_service=equipment,
equipment_details=equipment_details)
else:
equipment_details = sae.equipment_details.add()
self._fill_default_equipment_details(
id=code.value,
embedded_type=self._embedded_type(code.type),
equipment_details=equipment_details,
)
def extract(self, response, url, page):
results = []
data = jmespath.search("result", response)
count = int(response["count"])
max_page = count // len(data) + 1
for ele in data:
try:
results.append({
"vid":
ele['id'],
"cover":
ele['thumb'],
"title":
ele['title'],
"author":
ele['editor'],
"tag":
ele['keyword'],
"from":
self.from_,
"playlist_url":
url,
"webpage_url":
'https://www.topys.cn/article/' + ele['id'] + '.html',
"view_count":
ele["view_count"],
"comment_count":
ele["comment"],
"upload_ts":
ele["add_time"],
"category":
ele["cname"],
})
except:
continue
else:
return results, page < max_page, {}
def custom_build_iterator(client: Client, feed: Dict, limit: int = 0, **kwargs) -> List:
url = feed.get('url', client.url)
fetch_time = feed.get('fetch_time')
start_date, end_date = parse_date_range(fetch_time, utc=True, to_timestamp=True)
integration_context = get_integration_context()
last_fetch = integration_context.get(f"{feed.get('indicator_type')}_fetch_time")
params = {'lastUpdatedFrom': last_fetch if last_fetch else start_date}
result: List[Dict] = []
should_continue = True
while should_continue:
r = requests.get(
url=url,
verify=client.verify,
auth=client.auth,
cert=client.cert,
headers=client.headers,
params=params,
**kwargs
)
try:
r.raise_for_status()
data = r.json()
current_result = jmespath.search(expression=feed.get('extractor'), data=data)
if current_result:
result = result + current_result
# gets next page reference and handles paging.
should_continue = len(result) < limit if result else True
should_continue = should_continue or data.get('cursorNext') != params.get('cursor')
params['cursor'] = data.get('cursorNext') if should_continue else ''
except ValueError as VE:
raise ValueError(f'Could not parse returned data to Json. \n\nError massage: {VE}')
set_integration_context({f"{feed.get('indicator_type')}_fetch_time": str(end_date)})
return result
def extract_user_video_page(self, response, container_id, webpage_url):
results = []
cards = jmespath.search("cards[].mblog", response)
has_next = jmespath.search("cardlistInfo.since_id", response)
since_id = jmespath.search("cardlistInfo.since_id", response)
for ele in cards:
try:
results.append({
"title": None,
"vid": ele['idstr'],
"description": ele['text'],
"author": jmespath.search("user.name", ele),
"author_id": jmespath.search("user.idstr", ele),
"webpage_url": jmespath.search("url_struct[0].short_url", ele),
"from": self.from_,
"playlist_url": webpage_url,
"cover": jmespath.search("page_info.pic_info.pic_big.url", ele),
})
except:
continue
else:
return results, True if has_next else False, {"since_id": since_id, "container_id": container_id}
