def __generate_index(ctx): price_index = {} for service, data in PRICE_INDEX_CONFIG.items(): print('Getting prices for %s...' % service) price_index.setdefault(service, {}) price_data = requests.get(data['url']).json() product_query = "products.* " for k, v in data['attributes'].items(): if v.startswith('^'): # do a starts with exp product_query += " | [?starts_with(attributes.%s, '%s')]" % (k, v[1:]) else: product_query += " | [?attributes.%s=='%s']" % (k, v) products = jmespath.search(product_query, price_data) for product in products: instance_type = product['attributes']['instanceType'] if instance_type in price_index[service]: raise RuntimeError("Duplicate instance type: " + instance_type) price_query = ("terms.OnDemand.*.*[]" "| [?sku=='%s'].priceDimensions.*[].pricePerUnit" "| [0].USD" ) % product['sku'] price_index[service][instance_type] = float(jmespath.search(price_query, price_data)) with open('price_index.json', 'w') as f: json.dump(price_index, f, indent=True)
def collect(query_args, putter_args, collector_name, db_conn=None): '''does the collection, either puts the data in a table or returns it in a dictarray to the caller (in case of real time collection) ''' data = d_q(query_args) if putter_args['base'] != '': base = search(putter_args['base'], data) else: base = data if putter_args['table_name'] == '@return': #return a dictarray to be turned into a table if type(base) is list: vals = [{key: search(val, i) for key,val in putter_args['mapping'].items()} for i in base] elif type(base) is dict: vals = {key: search(val, base) for key,val in putter_args['mapping'].items()} #print vals return vals else: cursor = db_conn.cursor() mk_table(putter_args, cursor) db_conn.commit() queries = [] if type(base) is list: for i in base: qry = grab(i,putter_args, collector_name) queries.append(qry) elif type(base) is dict: queries.append(grab(base, putter_args, collector_name)) else: raise Exception('Data Structure \'%s\' not recognised!' % type(base)) for qry in queries: cursor.execute(qry) db_conn.commit()
def github_repos(organization, github_url, github_token): """Return all github repositories in an organization.""" # Get github repos headers = {"Authorization": "token {}".format(github_token)} next_cursor = None while next_cursor is not False: params = {'query': query, 'variables': { 'organization': organization, 'cursor': next_cursor}} response = requests.post(github_url, headers=headers, json=params) result = response.json() if response.status_code != 200 or 'errors' in result: raise ValueError("Github api error %s" % ( response.content.decode('utf8'),)) repos = jmespath.search( 'data.organization.repositories.edges[].node', result) for r in repos: yield r page_info = jmespath.search( 'data.organization.repositories.pageInfo', result) if page_info: next_cursor = (page_info['hasNextPage'] and page_info['endCursor'] or False) else: next_cursor = False
def _get_values(record, field_list, tag_map): tag_prefix = 'tag:' list_prefix = 'list:' count_prefix = 'count:' vals = [] for field in field_list: if field.startswith(tag_prefix): tag_field = field.replace(tag_prefix, '', 1) value = tag_map.get(tag_field, '') elif field.startswith(list_prefix): list_field = field.replace(list_prefix, '', 1) value = jmespath.search(list_field, record) if value is None: value = '' else: value = ', '.join([str(v) for v in value]) elif field.startswith(count_prefix): count_field = field.replace(count_prefix, '', 1) value = jmespath.search(count_field, record) if value is None: value = '' else: value = str(len(value)) else: value = jmespath.search(field, record) if value is None: value = '' if not isinstance(value, six.text_type): value = six.text_type(value) vals.append(value) return vals
def test_error_message_is_pluralized(self): with self.assertRaises(exceptions.ArityError) as e: jmespath.search('sort_by(@)', [0, 1]) exception = e.exception self.assertEqual( str(exception), 'Expected 2 arguments for function sort_by(), received 1')
def test_variadic_is_pluralized(self): with self.assertRaises(exceptions.VariadictArityError) as e: jmespath.search('not_null()', 'foo') exception = e.exception self.assertEqual( str(exception), 'Expected at least 1 argument for function not_null(), received 0')
def test_singular_in_error_message(self): with self.assertRaises(exceptions.ArityError) as e: jmespath.search('length(@, @)', [0, 1]) exception = e.exception self.assertEqual( str(exception), 'Expected 1 argument for function length(), received 2')
def get_values(self): contents, format = self.get_contents() if format == 'json': data = json.loads(contents) if 'expr' in self.data: res = jmespath.search(self.data['expr'], data) if res is None: log.warning('ValueFrom filter: %s key returned None' % self.data['expr']) return res elif format == 'csv' or format == 'csv2dict': data = csv.reader(io.StringIO(contents)) if format == 'csv2dict': data = {x[0]: list(x[1:]) for x in zip(*data)} else: if isinstance(self.data.get('expr'), int): return [d[self.data['expr']] for d in data] data = list(data) if 'expr' in self.data: res = jmespath.search(self.data['expr'], data) if res is None: log.warning('ValueFrom filter: %s key returned None' % self.data['expr']) return res return data elif format == 'txt': return [s.strip() for s in io.StringIO(contents).readlines()]
def test_can_provide_custom_functions(self): class CustomFunctions(jmespath.functions.Functions): @jmespath.functions.signature( {'types': ['number']}, {'types': ['number']}) def _func_custom_add(self, x, y): return x + y @jmespath.functions.signature( {'types': ['number']}, {'types': ['number']}) def _func_my_subtract(self, x, y): return x - y options = jmespath.Options(custom_functions=CustomFunctions()) self.assertEqual( jmespath.search('custom_add(`1`, `2`)', {}, options=options), 3 ) self.assertEqual( jmespath.search('my_subtract(`10`, `3`)', {}, options=options), 7 ) # Should still be able to use the original functions without # any interference from the CustomFunctions class. self.assertEqual( jmespath.search('length(`[1, 2]`)', {}), 2 )
def parse_case(self, response): ''' @url https://api.oyez.org/cases/2014/14-556 @returns requests 8 8 @returns items 0 0 ''' results = [] json_response = json.loads(response.body) # load case loader = CaseLoader(json_object=json_response) case = loader.load_case_data() case.send(self.session) case_id = case['oyez_id'] # if there is explicit decision data, go and get it; otherwise add the case to the results decision_url = jmespath.search('decisions[0].href', json_response) if decision_url != None: results.append(scrapy.Request(url=decision_url, callback=self.parse_decision, meta={'case': case})) # if there is advocacy data, go and get it advocate_links = jmespath.search('advocates[*].href', json_response) if advocate_links == None: advocate_links = [] for advocate_link in advocate_links: results.append(scrapy.Request(url=advocate_link, callback=self.parse_advocate, meta={'case_id':case_id})) # if there is transcript data, go and get it transcript_links = jmespath.search('oral_argument_audio[*].href', json_response) if transcript_links == None: transcript_links = [] for transcript_link in transcript_links: results.append(scrapy.Request(url=transcript_link, callback=self.parse_transcript, meta={'case_id':case_id})) return results
def test_null_to_nonetype(self): data = { 'a': { 'b': [1, 2, 3] } } self.assertEqual(jmespath.search('length0(a.b)', data, self.options), 3) self.assertEqual(jmespath.search('length0(a.c)', data, self.options), 0)
def _invoke_client_enum(self, client, enum_op, params, path): if client.supports_pagination(enum_op): results = [] for page in client.execute_paged_query(enum_op, params): results.extend(jmespath.search(path, page)) return results else: return jmespath.search(path, client.execute_query(enum_op, verb_arguments=params))
def test_get_one_node_info(self, fixture): response = fixture.app.get('/api/nodes/%s/_info' % fixture.cluster_v6_name) assert 200 == response.status_code res = fixture.get_response_data(response) node_id = list(jmespath.search('nodes', res['data'][0]).keys())[0] response = fixture.app.get('/api/nodes/%s/%s/_info' % (fixture.cluster_v6_name, node_id)) res = fixture.get_response_data(response) assert list(jmespath.search('nodes', res['data'][0]).keys())[0] == node_id assert fixture.has_all_keys(fixture.config.KEYS_NODE_INFO, res['data'][0]['nodes'][node_id], excludes=['build', 'http_address']) is True
def test_type_error_messages(self): with self.assertRaises(exceptions.JMESPathTypeError) as e: jmespath.search('length(@)', 2) exception = e.exception # 1. Function name should be in error message self.assertIn('length()', str(exception)) # 2. Mention it's an invalid type self.assertIn('invalid type for value: 2', str(exception)) # 3. Mention the valid types: self.assertIn("expected one of: ['string', 'array', 'object']", str(exception)) # 4. Mention the actual type. self.assertIn('received: "number"', str(exception))
def test_get_one_node_stats(self, fixture): response = fixture.app.get('/api/nodes/%s/_stats' % fixture.cluster_v6_name) assert 200 == response.status_code res = fixture.get_response_data(response) # get node ID: node_id = list(jmespath.search('nodes', res['data'][0]).keys())[0] response = fixture.app.get('/api/nodes/%s/%s/_stats' % (fixture.cluster_v6_name, node_id)) res = fixture.get_response_data(response) assert list(jmespath.search('nodes', res['data'][0]).keys())[0] == node_id assert fixture.has_all_keys(fixture.config.KEYS_NODE_STATS, res['data'][0]['nodes'][node_id]) is True
def apply_configuration(config): """ This function applies the simulations configurations :param config the configuration file """ log_level = str(jp.search(struct.LOG_LEVEL, config)).upper() log_level = 10 if log_level == 'DEBUG' else 20 configuration_parameter = { 'logging_level': log_level, 'clear_log_flag': jp.search(struct.CLEAR_OLD_LOGS, config) } configuration_parameter = convert_dict_values(configuration_parameter) services.apply_configuration(services.ConfigurationParameters(**configuration_parameter))
def test_multiple_hosts(self, host, add_host): # Add the rules result = host.run( 'stack add host firewall frontend-0-0 backend-0-0 service=1234 ' 'chain=INPUT action=ACCEPT protocol=TCP rulename=test' ) assert result.rc == 0 # Make sure they are in the DB now for hostname in ('frontend-0-0', 'backend-0-0'): result = host.run(f'stack list host firewall {hostname} output-format=json') assert result.rc == 0 rule = jmespath.search("[?name=='test']", json.loads(result.stdout)) assert rule == [{ 'host': hostname, 'name': 'test', 'table': 'filter', 'service': '1234', 'protocol': 'TCP', 'chain': 'INPUT', 'action': 'ACCEPT', 'network': None, 'output-network': None, 'flags': None, 'comment': None, 'source': 'H', 'type': 'var' }]
def format_resource(self, r): details = {} for k in r: if isinstance(k, (list, dict)): continue details[k] = r[k] for f in self.fields: value = jmespath.search(f['expr'], r) if not value: continue details[f['key']] = value for k, v in details.items(): if isinstance(v, datetime): v = v.isoformat() elif isinstance(v, (list, dict)): v = dumps(v) elif isinstance(v, (int, float, bool)): v = str(v) else: continue details[k] = v details['c7n:resource-type'] = self.manager.type other = { 'Type': 'Other', 'Id': self.manager.get_arns([r])[0], 'Region': self.manager.config.region, 'Details': {'Other': filter_empty(details)} } tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags: other['Tags'] = tags return other
def __init__(self, endpoint, data): super(Table, self).__init__(endpoint, data) self._id = data detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} data = endpoint.call(detail_op, **params) self.data = jmespath.search(detail_path, data)
def validate_healthcheck(self, data): for procType, healthcheck in data.items(): if healthcheck is None: continue for key, value in healthcheck.items(): if value is None: continue if key not in ['livenessProbe', 'readinessProbe']: raise serializers.ValidationError( "Healthcheck keys must be either livenessProbe or readinessProbe") try: jsonschema.validate(value, PROBE_SCHEMA) except jsonschema.ValidationError as e: raise serializers.ValidationError( "could not validate {}: {}".format(value, e.message)) # http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_probe # liveness only supports successThreshold=1, no other value # This is not in the schema since readiness supports other values threshold = jmespath.search('livenessProbe.successThreshold', healthcheck) if threshold is not None and threshold != 1: raise serializers.ValidationError( 'livenessProbe successThreshold can only be 1' ) return data
def _extract_fields(source, mappings): result = {} for mapping in mappings: result[mapping[1]] = jmespath.search(mapping[0], source) return result
def __call__(self, execution_result): json_value = execution_result.get_output_in_json() actual_result = jmespath.search(self._query, json_value, jmespath.Options(collections.OrderedDict)) if not re.match(self._expected_result, str(actual_result), re.IGNORECASE): raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result, execution_result.output)
def get_trail_ids(cls, event, mode): """extract resources ids from a cloud trail event.""" resource_ids = () event_name = event['detail']['eventName'] event_source = event['detail']['eventSource'] for e in mode.get('events', []): if not isinstance(e, dict): # Check if we have a short cut / alias info = CloudWatchEvents.match(event) if info: return info['ids'].search(event) continue if event_name != e.get('event'): continue if event_source != e.get('source'): continue id_query = e.get('ids') if not id_query: raise ValueError("No id query configured") evt = event # be forgiving for users specifying with details or without if not id_query.startswith('detail.'): evt = event.get('detail', {}) resource_ids = jmespath.search(id_query, evt) if resource_ids: break return resource_ids
def test_all_parameters(self, host): # Add the rule result = host.run( 'stack add host firewall frontend-0-0 table=nat service=1234 ' 'chain=INPUT action=ACCEPT protocol=TCP network=private ' 'output-network=private flags=test_flag comment=test_comment ' 'rulename=test' ) assert result.rc == 0 # Make sure it is in the DB now result = host.run('stack list host firewall frontend-0-0 output-format=json') assert result.rc == 0 rule = jmespath.search("[?name=='test']", json.loads(result.stdout)) assert rule == [{ 'host': 'frontend-0-0', 'name': 'test', 'table': 'nat', 'service': '1234', 'protocol': 'TCP', 'chain': 'INPUT', 'action': 'ACCEPT', 'network': 'private', 'output-network': 'private', 'flags': 'test_flag', 'comment': 'test_comment', 'source': 'H', 'type': 'var' }]
def test_no_rulename(self, host): # Add the rule result = host.run( 'stack add host firewall frontend-0-0 service=1234 chain=INPUT ' 'action=ACCEPT protocol=TCP network=private' ) assert result.rc == 0 # Make sure it is in the DB now result = host.run('stack list host firewall frontend-0-0 output-format=json') assert result.rc == 0 rule = jmespath.search("[?service=='1234']", json.loads(result.stdout)) # Make sure our rule name was a UUID and then remove it for the match assert re.match( r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', rule[0]['name'] ) del rule[0]['name'] assert rule == [{ 'host': 'frontend-0-0', 'table': 'filter', 'service': '1234', 'protocol': 'TCP', 'chain': 'INPUT', 'action': 'ACCEPT', 'network': 'private', 'output-network': None, 'flags': None, 'comment': None, 'source': 'H', 'type': 'var' }]
def retrieve_candidate_values(self, service, operation, param): if service not in self._services_with_completions: return # Example call: # service='ec2', operation='terminate-instances', # param='--instance-ids'. # We need to convert this to botocore syntax. # First try to load the resource model. LOG.debug("Called with: %s, %s, %s", service, operation, param) # Now convert operation to the name used by botocore. client = self._get_client(service) api_operation_name = client.meta.method_to_api_mapping.get( operation.replace('-', '_')) if api_operation_name is None: return # Now we need to convert the param name to the # casing used by the API. completer = self._get_completer_for_service(service) result = completer.describe_autocomplete( service, api_operation_name, param) if result is None: return # DEBUG:awsshell.resource.index:RESULTS: # ServerCompletion(service=u'ec2', operation=u'DescribeInstances', # params={}, path=u'Reservations[].Instances[].InstanceId') try: response = getattr(client, xform_name(result.operation, '_'))() except Exception: return results = jmespath.search(result.path, response) return results
def __call__(self, execution_result): json_value = execution_result.get_output_in_json() actual_result = jmespath.search(self._query, json_value, jmespath.Options(collections.OrderedDict)) if not actual_result: raise JMESPathCheckAssertionError(self._query, 'some value', actual_result, execution_result.output)
def process(self, resources): related_resources = dict( zip(jmespath.search('[].%s' % self.data['key'], resources), resources)) related_ids = set(related_resources) related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids) missing_related_tags = related_ids.difference(related_tag_map.keys()) if not self.data.get('skip_missing', True) and missing_related_tags: raise PolicyExecutionError( "Unable to find all %d %s related resources tags %d missing" % ( len(related_ids), self.data['resource'], len(missing_related_tags))) # rely on resource manager tag action implementation as it can differ between resources tag_action = self.manager.action_registry.get('tag')({}, self.manager) tag_action.id_key = tag_action.manager.get_model().id client = tag_action.get_client() stats = Counter() for related, r in related_resources.items(): if related in missing_related_tags or not related_tag_map[related]: stats['missing'] += 1 elif self.process_resource( client, r, related_tag_map[related], self.data['tags'], tag_action): stats['tagged'] += 1 else: stats['unchanged'] += 1 self.log.info( 'Tagged %d resources from related, missing-skipped %d unchanged %d', stats['tagged'], stats['missing'], stats['unchanged'])
def process(self, resources, event=None): ec2 = local_session(self.manager.session_factory).client('ec2') cidrs = jmespath.search( "PrefixLists[].Cidrs[]", ec2.describe_prefix_lists()) cidrs = [parse_cidr(cidr) for cidr in cidrs] results = [] check_egress = self.data.get('egress', True) check_ingress = self.data.get('ingress', True) present = self.data.get('present', False) for r in resources: matched = {cidr: None for cidr in cidrs} for entry in r['Entries']: if entry['Egress'] and not check_egress: continue if not entry['Egress'] and not check_ingress: continue entry_cidr = parse_cidr(entry['CidrBlock']) for c in matched: if c in entry_cidr and matched[c] is None: matched[c] = ( entry['RuleAction'] == 'allow' and True or False) if present and all(matched.values()): results.append(r) elif not present and not all(matched.values()): results.append(r) return results
def __init__(self, client, data, query=None): super(Table, self).__init__(client, data, query) self._id = data detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} data = client.call(detail_op, **params) self.data = jmespath.search(detail_path, data)
def _operation_type(cls, message_data): ''' We only care about _events_ that alter assets. Maintaining this list is going to be annoying. Long term, this entire class should be replaced when google provides a real-time event delivery solution ''' method_name = jmespath.search('protoPayload.methodName', message_data) or '' last = method_name.split('.')[-1].lower() # For batch methods, look for the verb after the word 'batch' if last.startswith('batch'): last = last[5:] read_prefixes = ('get', 'list') if last.startswith(read_prefixes): return 'read' write_prefixes = ( 'create', 'update', 'insert', 'patch', 'set', 'debug', 'enable', 'disable', 'expand', 'deactivate', 'activate' ) if last.startswith(write_prefixes): return 'write' delete_prefixes = ('delete') if last.startswith(delete_prefixes): return 'delete' else: return 'unknown'
def list_docker_images(): extra_globals = { "global.baseDomain": "foo.com", "blackboxExporterEnabled": True, "postgresqlEnabled": True, "prometheusPostgresExporterEnabled": True, "pspEnabled": True, "veleroEnabled": True, } public_repo_docs = render_chart(values={"global": extra_globals}) # Listing docker images search_string = "spec.template.spec.containers[*].image" docker_images = [] for public_repo_doc in public_repo_docs: docker_image_sublist = jmespath.search(search_string, public_repo_doc) if docker_image_sublist is not None: docker_images = docker_image_sublist + docker_images return docker_images
def list_spot_requests(ec2_conn): """Returns a list of object-like spot instance requests # sir.status: # open The request is waiting to be fulfilled. # active The request is fulfilled and has an associated Instance # failed The request has one or more bad parameters. # closed The Spot Instance was interrupted or terminated. # cancelled You cancelled the request, or the request expired. """ spot_instances = list_instances(ec2_conn, spot=True, state=["running", "terminated"]) sirs = ec2_conn.describe_spot_instance_requests()['SpotInstanceRequests'] requests = [] for req in sirs: try: launch, state, hostname = jmespath.search( "[?SpotInstanceRequestId=='%s'][][LaunchTime,State.Name," "Tags[?Key=='Name'].Value|[0]]" % req['SpotInstanceRequestId'], spot_instances)[0] except IndexError: launch, state, hostname = list('-' * 3) launch_specification = req['LaunchSpecification'] sir = types.SimpleNamespace( ami_id=launch_specification['ImageId'], az=req['LaunchedAvailabilityZone'], status_code=req['Status']["Code"], status_timestamp=req['Status']["UpdateTime"], hostname=hostname, id=req['SpotInstanceRequestId'], instance_id=req.get('InstanceId'), instance_status=state, instance_type=launch_specification['InstanceType'], instance_launch=launch, state=req['State'], subnet=launch_specification['NetworkInterfaces'][0]['SubnetId'], duration=req.get('BlockDurationMinutes')) requests.append(sir) return requests
def test_deregister_delete_snaps(self): factory = self.replay_flight_data('test_ami_deregister_delete_snap') p = self.load_policy( { 'name': 'deregister-snap', 'resource': 'ami', 'actions': [{ 'type': 'deregister', 'delete-snapshots': True }] }, session_factory=factory, config={'account_id': '644160558196'}) resources = p.run() self.assertEqual(len(resources), 1) client = factory().client('ec2') snap_ids = jmespath.search('BlockDeviceMappings[].Ebs.SnapshotId', resources[0]) self.assertRaises(ClientError, client.describe_snapshots, SnapshotIds=snap_ids, OwnerIds=['self'])
def _build_url(self, resource): """ Compose URL with query string parameters. Will not lose existing static parameters in the URL string but does not support 'duplicate' parameter entries """ if not self.query_params: return self.url evaluated_params = { k: jmespath.search(v, resource) for k, v in self.query_params.items() } url_parts = list(parse.urlparse(self.url)) query = dict(parse.parse_qsl(url_parts[4])) query.update(evaluated_params) url_parts[4] = parse.urlencode(query) return parse.urlunparse(url_parts)
def main(): parser = argparse.ArgumentParser() parser.add_argument('expression') parser.add_argument('-f', '--filename', help=('The filename containing the input data. ' 'If a filename is not given then data is ' 'read from stdin.')) parser.add_argument('--ast', action='store_true', help=('Pretty print the AST, do not search the data.')) args = parser.parse_args() expression = args.expression if args.ast: # Only print the AST expression = jmespath.compile(args.expression) sys.stdout.write(pformat(expression.parsed)) sys.stdout.write('\n') return 0 if args.filename: with open(args.filename, 'r') as f: data = json.load(f) else: data = sys.stdin.read() data = json.loads(data) try: sys.stdout.write(json.dumps( jmespath.search(expression, data), indent=4)) sys.stdout.write('\n') except exceptions.ArityError as e: sys.stderr.write("invalid-arity: %s\n" % e) return 1 except exceptions.JMESPathTypeError as e: sys.stderr.write("invalid-type: %s\n" % e) return 1 except exceptions.UnknownFunctionError as e: sys.stderr.write("unknown-function: %s\n" % e) return 1 except exceptions.ParseError as e: sys.stderr.write("syntax-error: %s\n" % e) return 1
def test_kafka_cluster_kms_filter(self): session_factory = self.replay_flight_data('test_kafka_cluster_kms_filter') kms = session_factory().client('kms') expression = 'EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId' p = self.load_policy( { 'name': 'kafka-kms-filter', 'resource': 'kafka', 'filters': [ { 'type': 'kms-key', 'key': 'c7n:AliasName', 'value': 'alias/aws/kafka' } ] }, session_factory=session_factory ) resources = p.run() self.assertTrue(len(resources), 1) aliases = kms.list_aliases(KeyId=(jmespath.search(expression, resources[0]))) self.assertEqual(aliases['Aliases'][0]['AliasName'], 'alias/aws/kafka')
def extract_tag_pageing_api(self, response_json, playlist_url): results = [] itemList = jmespath.search("itemList[].data", response_json) has_more = True if jmespath.search("nextPageUrl", response_json) else False for ele in itemList: data = jmespath.search("content.data", ele) if data is None: data = ele try: data['title'] except: continue tag = jmespath.search("tags[].name", data) try: upload_ts = data['releaseTime'] // 1000 except: upload_ts = None try: result = { "cover": jmespath.search("cover.feed", data), "play_addr": jmespath.search("playUrl", data), "tag": tag, "category": data['category'], "description": data['description'], "title": data['title'], "author": data['author']['name'], "author_id": data['author']['id'], "upload_ts": upload_ts, "vid": data['id'], "duration": data['duration'], "from": self.from_, "playlist_url": playlist_url, "webpage_url": jmespath.search("webUrl.forWeibo", data), } except: traceback.print_exc() continue results.append(result) else: return results, has_more, {}
def test_externalelasticsearch_with_awsServiceAccountAnnotation(self, kube_version): """Test External ElasticSearch with eks iam roles passed as Service Account Annotation.""" docs = render_chart( kube_version=kube_version, values={ "global": { "customLogging": { "enabled": True, "scheme": "https", "host": "esdemo.example.com", "awsServiceAccountAnnotation": "arn:aws:iam::xxxxxxxx:role/customrole", } } }, show_only=[ "charts/external-es-proxy/templates/external-es-proxy-deployment.yaml", "charts/external-es-proxy/templates/external-es-proxy-serviceaccount.yaml", "charts/external-es-proxy/templates/external-es-proxy-service.yaml", ], ) assert len(docs) == 3 doc = docs[0] expected_env = [{"name": "ENDPOINT", "value": "https://esdemo.example.com"}] assert expected_env == doc["spec"]["template"]["spec"]["containers"][1]["env"] doc = docs[1] assert "arn:aws:iam::xxxxxxxx:role/customrole" == jmespath.search( 'metadata.annotations."eks.amazonaws.com/role-arn"', doc ) assert "Service" == jmespath.search("kind", docs[2]) assert "release-name-external-es-proxy" == jmespath.search( "metadata.name", docs[2] ) assert "ClusterIP" == jmespath.search("spec.type", docs[2]) assert { "name": "secure-http", "protocol": "TCP", "port": 9200, } in jmespath.search("spec.ports", docs[2]) assert {"name": "http", "protocol": "TCP", "port": 9201} in jmespath.search( "spec.ports", docs[2] )
def format_resource(self, r): details = {} for k in r: if isinstance(k, (list, dict)): continue details[k] = r[k] for f in self.fields: value = jmespath.search(f['expr'], r) if not value: continue details[f['key']] = value for k, v in details.items(): if isinstance(v, datetime): v = v.isoformat() elif isinstance(v, (list, dict)): v = dumps(v) elif isinstance(v, (int, float, bool)): v = str(v) else: continue details[k] = v[:SECHUB_VALUE_SIZE_LIMIT] details['c7n:resource-type'] = self.manager.type other = { 'Type': self.resource_type, 'Id': self.manager.get_arns([r])[0], 'Region': self.manager.config.region, 'Partition': get_partition(self.manager.config.region), 'Details': { self.resource_type: filter_empty(details) } } tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags: other['Tags'] = tags return other
def test_asg_propagate_tag_action(test, aws_asg): factory = test.replay_flight_data('test_asg_propagate_tag_action') p = test.load_policy( { 'name': 'asg-tagger', 'resource': 'aws.asg', 'filters': [{ 'AutoScalingGroupName': aws_asg['aws_autoscaling_group.bar.id'] }, { 'tag:Owner': 'absent' }], 'actions': [{ 'type': 'tag', 'key': 'Owner', 'value': 'Kapil', 'propagate': True }, { 'type': 'propagate-tags', 'tags': ['Owner'] }] }, session_factory=factory) resources = p.run() test.assertEqual(len(resources), 1) client = factory().client('ec2') itags = { t['Key']: t['Value'] for t in jmespath.search( 'Reservations[0].Instances[0].Tags', client.describe_instances( InstanceIds=[resources[0]['Instances'][0]['InstanceId']])) } assert 'Owner' in itags assert itags['Owner'] == 'Kapil'
def load_expected_data(source_file_name, dir_path): expected_path = os.path.join(dir_path, source_file_name) if not os.path.exists(expected_path): return None with open(expected_path, "r") as f: expected_data = json.load(f) # Convert to absolute path: "buckets/bucket.tf[main.tf#0]" # ^^^^^^^^^^^^^^^^^ ^^^^^^^ # HERE & HERE # resolved_pattern = re.compile(r"(.+)\[(.+)#(\d+)]") # groups: location (1), referrer (2), index (3) # Expected files should have the filenames relative to their base directory, but the parser will # use the absolute path. This loop with replace relative filenames with absolute. keys = list(expected_data.keys()) for key in keys: # NOTE: Sometimes keys have module referrers, sometimes they don't match = resolved_pattern.match(key) if match: new_key = _make_module_ref_absolute(match, dir_path) else: if os.path.isabs(key): continue new_key = os.path.join(dir_path, key) expected_data[new_key] = expected_data[key] del expected_data[key] for resolved_list in jmespath.search("*.module[].*[].__resolved__", expected_data): for list_index in range(0, len(resolved_list)): match = resolved_pattern.match(resolved_list[list_index]) assert match is not None, f"Unexpected module resolved data: {resolved_list[list_index]}" resolved_list[list_index] = _make_module_ref_absolute(match, dir_path) # print(f"{match[0]} -> {resolved_list[list_index]}") return expected_data
def build_iterator(self, **kwargs) -> List: results = [] for feed_name, feed in self.feed_name_to_config.items(): r = requests.get(url=feed.get('url', self.url), verify=self.verify, auth=self.auth, cert=self.cert, headers=self.headers, **kwargs) try: r.raise_for_status() data = r.json() result = jmespath.search(expression=feed.get('extractor'), data=data) results.append({feed_name: result}) except ValueError as VE: raise ValueError( f'Could not parse returned data to Json. \n\nError massage: {VE}' ) return results
def should_notify(replica: Replica, subscription: dict, metadata_document: dict, key: str) -> bool: """ Check if a notification should be attempted for subscription and key """ jmespath_query = subscription.get(SubscriptionData.JMESPATH_QUERY) if not jmespath_query: return True else: try: if jmespath.search(jmespath_query, metadata_document): return True else: return False except JMESPathError: logger.error("jmespath query failed for owner={} replica={} uuid={} jmespath_query='{}' key={}".format( subscription[SubscriptionData.OWNER], subscription[SubscriptionData.REPLICA], subscription[SubscriptionData.UUID], subscription[SubscriptionData.JMESPATH_QUERY], key )) return False
def restore_security_group(instance_id, instance_to_mimic_id): try: client = boto3.client('rds') response = client.describe_db_instances( DBInstanceIdentifier=instance_to_mimic_id) security_group_groups = jmespath.search( "DBInstances[*].VpcSecurityGroups[*].VpcSecurityGroupId", response) # the outer group should correspond to the unique instance specified, so should have # length==1 if len(security_group_groups) == 1: security_group_ids = security_group_groups[0] client.modify_db_instance(DBInstanceIdentifier=instance_id, VpcSecurityGroupIds=security_group_ids) except botocore.exceptions.ClientError as e: print("Could not add security group(s) to %s. Error %s" % (instance_id, str(e)))
def test_keda_enabled(self, executor, is_created): """ ScaledObject should only be created when set to enabled and executor is Celery or CeleryKubernetes """ docs = render_chart( values={ "workers": { "keda": { "enabled": True }, "persistence": { "enabled": False } }, 'executor': executor, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) if is_created: assert jmespath.search("metadata.name", docs[0]) == "RELEASE-NAME-worker" else: assert docs == []
def retrieve_os_users(client, instances): ec2 = client ami_query = jmespath.search( "Images[].{id: ImageId, name: Name, loc: ImageLocation}", ec2.describe_images( ImageIds=[instance.get("ami") for instance in instances]), ) ami_query = {ami_dic["id"]: ami_dic for ami_dic in ami_query} instance_dics = [] for instance in instances: ami = instance["ami"] ami_info = ami_query.get(ami) os_user = _osuser_from_ami(ami_info) instance["os_user"] = os_user instance_dics.append(instance) return instance_dics
def valid_available_instances(client): ec2 = client inst_dict = jmespath.search( "Reservations[].Instances[].{name: Tags[?Key=='Name'].Value | [0], id: InstanceId, avz: Placement.AvailabilityZone, ami: ImageId, ip: PublicIpAddress}", ec2.describe_instances(), ) if inst_dict is None: return has_name, no_name = partition( lambda dic: dic.get("name") is None or dic.get("name") == "", inst_dict) no_whitespace, whitespace = partition(lambda dic: " " in dic.get("name"), has_name) valid_instances = list(no_whitespace) invalid_instances = list(whitespace) + list(no_name) return valid_instances, invalid_instances
def get_tmpl_context(ml, fname): if fname not in datadict: objs = cldf.objects(table_map[fname]) \ if table_map[fname] else cldf.iter_rows(fname, 'id') datadict[fname] = { r.id if isinstance(r, orm.Object) else r['id']: r for r in objs } tmpl_context = { k: True if v[0] == '' else v[0] for k, v in ml.parsed_url_query.items() } for k in tmpl_context: if k.startswith('with_') and (tmpl_context[k] in ['0', 'false', 'False']): tmpl_context[k] = False if table_map[fname] == 'Metadata': tmpl_context['ctx'] = jmespath.search(ml.objid, datadict[fname]) else: tmpl_context['ctx'] = list(datadict[fname].values()) \ if ml.all else datadict[fname][ml.objid] tmpl_context['cldf'] = cldf return tmpl_context
def verify_transform(subject, mapping): """ Determines if a key from mapping exists in subject and if so verifies that subject[k] is of type mapping[k] """ import jmespath for k in mapping.keys(): result = jmespath.search(k, subject) if result is None: raise AttributeError('The property "{}" is required'.format(k)) if not isinstance(result, mapping[k]): supplemental_info = "" if mapping[k] == dict: wiki_link = "https://github.com/Azure/azure-iot-cli-extension/wiki/Tips" supplemental_info = "Review inline JSON examples here --> {}".format( wiki_link) raise TypeError( 'The property "{}" must be of {} but is {}. Input: {}. {}'. format(k, str(mapping[k]), str(type(result)), result, supplemental_info))
def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim( self): docs = render_chart( values={ "dags": { "persistence": { "enabled": True, "existingClaim": "test-claim" } } }, show_only=["templates/pod-template-file.yaml"], ) self.assertIn( { "name": "dags", "persistentVolumeClaim": { "claimName": "test-claim" } }, jmespath.search("spec.volumes", docs[0]), )
def build_identifiers(identifiers_def, parent, params, raw_response): """ Builds a mapping of identifier names to values based on the identifier source location, type, and target. Identifier values may be scalars or lists depending on the source type and location. :type identifiers_def: list :param identifiers_def: List of identifier definitions :type parent: ServiceResource :param parent: The resource instance to which this action is attached. :type params: dict :param params: Request parameters sent to the service. :type response: dict :param response: Low-level operation response. """ results = {} for identifier in identifiers_def: source = identifier.get('source', '') source_type = identifier.get('sourceType') target = identifier.get('target') if source_type == 'responsePath': value = jmespath.search(source, raw_response) results[xform_name(target)] = value elif source_type in ['identifier', 'dataMember']: value = getattr(parent, xform_name(source)) results[xform_name(target)] = value elif source_type == 'requestParameter': value = params[source] results[xform_name(target)] = value else: raise NotImplementedError( 'Unsupported source type: {0}'.format(source_type)) return results
def recurse_access_key(data, query): """ Search for something in the given data using the given query. Example: >>> recurse_access_key({'a': 'b'}, 'a') 'b' >>> recurse_access_key({'a': {'b': ['c', 'd']}}, 'a.b[0]') 'c' Args: data (dict, list): Data to search in query (str): Query to run Returns: object: Whatever was found by the search """ try: from_jmespath = jmespath.search(query, data) except jmespath.exceptions.ParseError as e: logger.error("Error parsing JMES query") try: _deprecated_recurse_access_key(data, query.split(".")) except (IndexError, KeyError): logger.debug("Nothing found searching using old method") else: # If we found a key using 'old' style searching logger.warning( "Something was found using 'old style' searching in the response - please change the query to use jmespath instead - see http://jmespath.org/ for more information" ) raise exceptions.JMESError("Invalid JMES query") from e return from_jmespath
def get_regions(profile=None, region=None): while True: try: session = boto3.Session(profile_name=profile) response = session.client('ec2', region_name=region, config=config).describe_regions() return jmespath.search('Regions[].RegionName', response) except awsClientError as error: if error.response['Error']['Code'] in [ "RequestTimeout", "RequestTimeoutException", "PriorRequestNotComplete", "ConnectionError", "HTTPClientError", "Throttling", "ThrottlingException", "ThrottledException", "RequestThrottledException", "TooManyRequestsException", "ProvisionedThroughputExceededException", "TransactionInProgressException", "RequestLimitExceeded", "BandwidthLimitExceeded", "LimitExceededException", "RequestThrottled", "SlowDown", "EC2ThrottledException" ]: logger.error( "Encountered Error: {} !! Sleeping for 20 seconds".format( error.response['Error']['Code'])) logger.info(traceback.format_exc()) time.sleep(20) elif error.response['Error']['Code'] in ["UnauthorizedOperation"]: logger.error( "Encountered Error: {} !! Sleeping for 20 seconds".format( error.response['Error']['Code'])) logger.info(traceback.format_exc()) region = "ap-south-1" time.sleep(20) else: logger.error( "Encountered Error: {} !! Ommitting region {} for profile {}" .format(error.response['Error']['Code'], region, profile)) logger.info(traceback.format_exc()) return []
def process(self, resources): related_resources = list( zip( jmespath.search( '[].[%s || "c7n:NotFound"]|[]' % self.data['key'], resources), resources)) related_ids = set([r[0] for r in related_resources]) related_ids.discard('c7n:NotFound') related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids) missing_related_tags = related_ids.difference(related_tag_map.keys()) if not self.data.get('skip_missing', True) and missing_related_tags: raise PolicyExecutionError( "Unable to find all %d %s related resources tags %d missing" % (len(related_ids), self.data['resource'], len(missing_related_tags))) # rely on resource manager tag action implementation as it can differ between resources tag_action = self.manager.action_registry.get('tag')({}, self.manager) tag_action.id_key = tag_action.manager.get_model().id client = tag_action.get_client() stats = Counter() for related, r in related_resources: if related in missing_related_tags or not related_tag_map[related]: stats['missing'] += 1 elif self.process_resource(client, r, related_tag_map[related], self.data['tags'], tag_action): stats['tagged'] += 1 else: stats['unchanged'] += 1 self.log.info( 'Tagged %d resources from related, missing-skipped %d unchanged %d', stats['tagged'], stats['missing'], stats['unchanged'])
def save_response(test_executor: TestExecutor, r, response_dict): """ processes the http request to that mentioned in YAML :param test_executor: The TestExecutor object to give context for execution :type test_executor: TestExecutor :type r: request :param response_dict: response dict fetched from YAML :type response_dict: dict :returns; Status of Execution and error stack """ try: logger.info(r.json()) if 'save' in response_dict: for entry in response_dict['save']: b = entry['save in'] # if key is not present in dict, whole json will be saved to the 'save in' parameter if 'key' in entry: test_executor.variable_dictionary[b] = jmespath.search( entry['key'], r.json()) else: test_executor.variable_dictionary[b] = r.text logger.info( str(test_executor.variable_dictionary) + " new key, value pairs added") except JSONDecodeError: logger.info(r.text) # save the text response in the variable_dict if 'save' in response_dict: for entry in response_dict['save']: b = entry['save in'] test_executor.variable_dictionary[b] = r.text logger.info( str(test_executor.variable_dictionary) + " new (key,value) pairs added;" " value contains the whole text received") return True, None
def _process_for_equipment_reports(self, data, stop_area_equipments_list): """ For each stop_area_equipments within equipment_reports response, the structure 'equipment_details' is updated if the corresponding code is present :param data: equipments data received from the webservice :param stop_area_equipments_list: list of stop_area_equipments from the protobuf response """ for sae in stop_area_equipments_list: """ Codes might be duplicated across different stop points. Because we report equipments on a stop area basis, we don't want them duplicated """ unique_codes = { six.text_type(code): code for st in sae.stop_area.stop_points for code in st.codes } for code in unique_codes.values(): if code.type in self.code_types: equipments_list = jmespath.search( "equipments_details[?id=='{}']".format(code.value), data) if equipments_list: equipment = equipments_list[0] # Fill PB equipment_details = sae.equipment_details.add() self._fill_equipment_details( equipment_form_web_service=equipment, equipment_details=equipment_details) else: equipment_details = sae.equipment_details.add() self._fill_default_equipment_details( id=code.value, embedded_type=self._embedded_type(code.type), equipment_details=equipment_details, )
def extract(self, response, url, page): results = [] data = jmespath.search("result", response) count = int(response["count"]) max_page = count // len(data) + 1 for ele in data: try: results.append({ "vid": ele['id'], "cover": ele['thumb'], "title": ele['title'], "author": ele['editor'], "tag": ele['keyword'], "from": self.from_, "playlist_url": url, "webpage_url": 'https://www.topys.cn/article/' + ele['id'] + '.html', "view_count": ele["view_count"], "comment_count": ele["comment"], "upload_ts": ele["add_time"], "category": ele["cname"], }) except: continue else: return results, page < max_page, {}
def custom_build_iterator(client: Client, feed: Dict, limit: int = 0, **kwargs) -> List: url = feed.get('url', client.url) fetch_time = feed.get('fetch_time') start_date, end_date = parse_date_range(fetch_time, utc=True, to_timestamp=True) integration_context = get_integration_context() last_fetch = integration_context.get(f"{feed.get('indicator_type')}_fetch_time") params = {'lastUpdatedFrom': last_fetch if last_fetch else start_date} result: List[Dict] = [] should_continue = True while should_continue: r = requests.get( url=url, verify=client.verify, auth=client.auth, cert=client.cert, headers=client.headers, params=params, **kwargs ) try: r.raise_for_status() data = r.json() current_result = jmespath.search(expression=feed.get('extractor'), data=data) if current_result: result = result + current_result # gets next page reference and handles paging. should_continue = len(result) < limit if result else True should_continue = should_continue or data.get('cursorNext') != params.get('cursor') params['cursor'] = data.get('cursorNext') if should_continue else '' except ValueError as VE: raise ValueError(f'Could not parse returned data to Json. \n\nError massage: {VE}') set_integration_context({f"{feed.get('indicator_type')}_fetch_time": str(end_date)}) return result
def extract_user_video_page(self, response, container_id, webpage_url): results = [] cards = jmespath.search("cards[].mblog", response) has_next = jmespath.search("cardlistInfo.since_id", response) since_id = jmespath.search("cardlistInfo.since_id", response) for ele in cards: try: results.append({ "title": None, "vid": ele['idstr'], "description": ele['text'], "author": jmespath.search("user.name", ele), "author_id": jmespath.search("user.idstr", ele), "webpage_url": jmespath.search("url_struct[0].short_url", ele), "from": self.from_, "playlist_url": webpage_url, "cover": jmespath.search("page_info.pic_info.pic_big.url", ele), }) except: continue else: return results, True if has_next else False, {"since_id": since_id, "container_id": container_id}