def update(self, body): # Use JSONPath to extract out what we need passphrase = parse('config.ssh.passphrase').find(body) public_key = parse('config.ssh.publicKey').find(body) if passphrase: ssh = self.cluster['config'].setdefault('ssh', {}) ssh['passphrase'] = passphrase[0].value if public_key: public_key = public_key[0].value if not _validate_key(public_key): raise RestException('Invalid key format', 400) ssh = self.cluster['config'].setdefault('ssh', {}) ssh['publicKey'] = public_key self.cluster = self.model('cluster', 'cumulus').save(self.cluster) # Don't return the access object del self.cluster['access'] # Don't return the log del self.cluster['log'] # Don't return the passphrase if parse('config.ssh.passphrase').find(self.cluster): del self.cluster['config']['ssh']['passphrase'] return self.cluster
def _process_experimental(self, doc): facility_used = parse('experiment.experimentalEnvironment.facilityUsed').find(doc)[0].value experiments = parse('experiment.experiments').find(doc)[0].value experiment_model = self.model('experimental', 'molecules') experiments_list = [] for experiment in experiments: spectrum_type = experiment['spectrumType'] experimental_technique = experiment['experimentalTechnique'] id = experiment['id'] molecular_formula = experiment['molecularFormula'] instenisty_units = parse('measuredSpectrum.unitsY').find(experiment)[0].value frequency_units = parse('measuredSpectrum.unitsX').find(experiment)[0].value data_points = parse('measuredSpectrum.dataPoints').find(experiment)[0].value frequencies = data_points[::2] intensities = data_points[1::2] measured_spectrum = { 'frequencies': { 'units': frequency_units, 'values': frequencies }, 'intensities': { 'units': instenisty_units, 'values': intensities } } experiments_list.append(experiment_model.create( facility_used, spectrum_type, experimental_technique, id, molecular_formula, measured_spectrum)) return experiments_list
def extract_phones_urls_from_all_files(outer_folder= '/Users/mayankkejriwal/datasets/memex/user-worker-dig3-full-phase4-2017/'): """ Will write out strict phones in outer_folder+'strict_phones.txt' and relaxed_phones in outer_folder+'relaxed_phones.txt' Note that while we do 'set' checking for each individual json there can be many repeated phones across the file. Each line will be tab-delimited and represent the phones per json. :param outer_folder: :return: """ jpath_relaxed = parse('fields.phone.relaxed[*].name') jpath_strict = parse('fields.phone.strict[*].name') listOfFiles = glob.glob(outer_folder + 'gz/*.txt.gz') out_strict = codecs.open(outer_folder+'strict_phones_urls.txt', 'w', 'utf-8') out_relaxed = codecs.open(outer_folder + 'relaxed_phones_urls.txt', 'w', 'utf-8') for fi in listOfFiles: print 'processing file: ',fi with gzip.open(fi, 'rb') as f: for line in f: j = json.loads(re.split('\t', line[0:-1])[1]) strict_phones = list(set([match.value for match in jpath_strict.find(j)])) if len(strict_phones) != 0: strict_phones = '\t'.join(strict_phones) out_strict.write(j['url']+'\t'+strict_phones) out_strict.write('\n') relaxed_phones = list(set([match.value for match in jpath_relaxed.find(j)])) if len(relaxed_phones) != 0: relaxed_phones = '\t'.join(relaxed_phones) out_relaxed.write(j['url']+'\t'+relaxed_phones) out_relaxed.write('\n') out_strict.close() out_relaxed.close()
def process_class(class_name, args, listPath=None): "process on one class" args1 = list(args) while True: response_json = call_cli(args1) if (listPath): jsonpath_expr = parse(listPath[0]) node_list = jsonpath_expr.find(response_json) for node in node_list: response_json = node.value if (len(listPath )> 1 and len(response_json) > 0): jsonpath_expr = parse(listPath[1]) for first_node_list in response_json: node_list2 = jsonpath_expr.find(first_node_list) for node2 in node_list2: process_object(class_name, node2.value) else: process_object(class_name, node.value) else: process_object(class_name, response_json) if ('NextToken' in response_json): args1 = list(args) args1.append("--starting-token") args1.append(u2s(response_json['NextToken'])) else: break
def get_primary_score(exp): primary = parse('$.details[0].value') primary_with_phrase = parse('$.details[0].details[0].details[0].value') if has_phrase_rescore(exp): return get_score_from_path(exp, primary_with_phrase) else: return get_score_from_path(exp, primary)
def cross_references(self): """ Extract all source and destination cross references i.e. parameters with on 'SourceEnd' and 'TargetEnd' parameters :return: pd.DataFrame """ tag_dependency_list_query = 'DependencyList.DependencyListObj.Category.Dependency[*]' tag_dependency_json_list = parse(tag_dependency_list_query).find(self._cm_json) tag_xref_list_query = '*..Attribute[*]' source_end_list = [] target_end_list = [] for dependent_json in tag_dependency_json_list: for param in parse(tag_xref_list_query).find(dependent_json): v = param.value.values() if 'SourceEnd' in v: source_end_list.append(v[1]) continue if 'TargetEnd' in v: target_end_list.append(v[1]) continue continue df = pd.DataFrame({'SourceEnd': source_end_list, 'TargetEnd': target_end_list}) df['SourceTagname'] = df.SourceEnd.apply(lambda source_end: source_end.split('.')[0]) df['TargetTagname'] = df.TargetEnd.apply(lambda target_end: target_end.split('.')[0]) df.drop_duplicates(inplace=True) return self._drop_empty_columns(dataframe=df)
def _update(url, request): request_body = json.loads(request.body) passphrase = parse('config.ssh.passphrase').find(request_body) public_key = parse('config.ssh.publicKey').find(request_body) status = request_body['status'] == 'created' self._update = passphrase and public_key and status return httmock.response(200, None, {}, request=request)
def validate_args(kwargs): required = ['cluster.config.paraview.installDir', 'sessionKey'] for r in required: if not parse(r).find(kwargs): raise Exception('Required parameter %s not provide to taskflow.' % r) if not parse('dataDir') and not parse('input.file.id'): raise Exception('\'dataDir\' or \'input.file.id\' must be provided.')
def submit_paraview_job(task, cluster, job, *args, **kwargs): task.taskflow.logger.info('Submitting job to cluster.') girder_token = task.taskflow.girder_token params = {} if 'dataDir' in kwargs: params['dataDir'] = kwargs['dataDir'] if 'fileName' in kwargs: params['fileName'] = kwargs['fileName'] if 'simulationJobId' in kwargs: params['simulationJobId'] = kwargs['simulationJobId'] if 'sessionKey' in kwargs: params['sessionKey'] = kwargs['sessionKey'] # Save the sessionKey so we can clean up the proxy entry task.taskflow.set_metadata('sessionKey', kwargs['sessionKey']) parallel_environment \ = parse('config.parallelEnvironment').find(cluster) if parallel_environment: parallel_environment = parallel_environment[0].value params['parallelEnvironment'] = parallel_environment params['numberOfSlots'] = 1 job_output_dir = get_cluster_job_output_dir(cluster) if job_output_dir: params['jobOutputDir'] = job_output_dir paraview_install_dir \ = parse('config.paraview.installDir').find(cluster) if paraview_install_dir: paraview_install_dir = paraview_install_dir[0].value params['paraviewInstallDir'] = paraview_install_dir # Does the cluster have GPUs? params['gpu'] = has_gpus(cluster) or kwargs.get('numberOfGpusPerNode', 0) > 0 job['params'] = params # Create proxy entry if cluster['type'] == 'ec2': create_proxy_entry(task, cluster, job) # Before we submit the job upload any file we may have been given upload_input(task, cluster, job, *args, **kwargs) submit_job(cluster, job, log_write_url=None, girder_token=girder_token, monitor=False) monitor_paraview_job.delay(cluster, job, *args, **kwargs)
def get_assetstore_id(girder_token, cluster): if "assetstoreId" not in cluster: headers = {"Girder-Token": girder_token} url_base = get_assetstore_url_base(cluster) create_url = "%s/%s" % (cumulus.config.girder.baseUrl, url_base) body = { "name": cluster["_id"], "host": cluster["config"]["host"], "machine": cluster["config"]["host"], "authKey": cluster["_id"], } user = parse("config.ssh.user").find(cluster) if user: body["user"] = user[0].value r = requests.post(create_url, json=body, headers=headers) check_status(r) cluster["assetstoreId"] = r.json()["_id"] cluster_url = "%s/clusters/%s" % (cumulus.config.girder.baseUrl, cluster["_id"]) body = {"assetstoreId": cluster["assetstoreId"]} r = requests.patch(cluster_url, json=body, headers=headers) check_status(r) return cluster["assetstoreId"]
def test_text_fields(pattern): owl_entities = set() if 'classes' in pattern.keys(): owl_entities.update(set(pattern['classes'].keys())) if 'relations' in pattern.keys(): owl_entities.update(set(pattern['relations'].keys())) expr = parse('*..text') text_fields = [match for match in expr.find(pattern)] stat=True if text_fields: for field in text_fields: # Test for even number single quotes val = field.value m = re.findall("'", val) if divmod(len(m), 2)[1]: warnings.warn("text field '%s' has an odd number of single quotes." % val) stat = False # Test that single quoted strings are OWL entities in dict. m = re.findall("'(.+?)'", val) quoted = set(m) if not owl_entities.issuperset(quoted): warnings.warn("%s has values (%s) not found in owl entity dictionaries t (%s): " % (field.full_path, str(quoted.difference(owl_entities)), str(owl_entities))) stat = False else: warnings.warn("Pattern has no text fields") return stat
def test_vars(pattern): """Tests whether variable names in any field with key 'vars' is in the vars list for the pattern""" if 'vars' in pattern.keys(): vars = set(pattern['vars'].keys()) else: warnings.warn("Pattern has no vars") return True ## If this is to be compulsory, should be spec'd as such in json_schema if 'data_vars' in pattern.keys(): vars.update(set(pattern['data_vars'].keys())) if 'substitutions' in pattern.keys(): subvars = [X['out'] for X in pattern['substitutions']] vars.update(set(subvars)) expr = parse('*..vars') var_fields = [match for match in expr.find(pattern)] stat = True if var_fields: for field in var_fields: val = set(field.value) if not vars.issuperset(val): warnings.warn("%s has values (%s) not found in pattern variable list (%s): " % (field.full_path, str(val.difference(vars)), str(vars))) stat = False else: warnings.warn("Pattern has no var fields") return stat
def __init__(self, name, trait_cfg): self.cfg = trait_cfg self.name = name type_name = trait_cfg.get('type', 'string') if 'fields' not in trait_cfg: raise EventDefinitionException( _("Required field in trait definition not specified: " "'%s'") % 'fields', self.cfg) fields = trait_cfg['fields'] if not isinstance(fields, six.string_types): # NOTE(mdragon): if not a string, we assume a list. if len(fields) == 1: fields = fields[0] else: fields = '|'.join('(%s)' % path for path in fields) try: self.fields = jsonpath_rw.parse(fields) except Exception as e: raise EventDefinitionException( _("Parse error in JSONPath specification " "'%(jsonpath)s' for %(trait)s: %(err)s") % dict(jsonpath=fields, trait=name, err=e), self.cfg) self.trait_type = type_name if self.trait_type is None: raise EventDefinitionException( _("Invalid trait type '%(type)s' for trait %(trait)s") % dict(type=type_name, trait=name), self.cfg)
def validate(self): profile_id = parse('profileId').find(self.volume)[0].value profile = self.model('aws', 'cumulus').load(profile_id, user=getCurrentUser()) if not profile: raise ValidationException('Invalid profile id') valid_fs = ['ext2', 'ext3', 'ext4'] if 'fs' in self.volume and self.volume['fs'] not in valid_fs: raise ValidationException('Unsupported file system type', 'fs') try: int(self.volume['size']) except ValueError: raise ValidationException('size number in an integer', 'size') # Name should be unique user = getCurrentUser() query = { 'name': self.volume['name'], 'userId': user['_id'] } if '_id' in self.volume: query['_id'] = {'$ne': self.volume['_id']} volume = self.model('volume', 'cumulus').findOne(query) if volume: raise ValidationException('A volume with that name already exists', 'name') return self.volume
def main(argv=None): '''this is called if run from command line''' try: argv = argv or sys.argv parser = argparse.ArgumentParser() parser.add_argument('path', type=str, help='see python module jsonpath_rw') args = parser.parse_args() path = args.path expr = parse(path) for line in sys.stdin: try: (url, repn) = line.split('\t') obj = json.loads(repn) vals = expr.find(obj) val = vals[0].value print >> sys.stdout, "%s\t%s" % (url, json.dumps(val)) except Exception as e: print >> sys.stderr, "inner %r" % e # Failure to parse one line: continue to next pass except Exception as e: print >> sys.stderr, "outer %r" % e # Any other failure: die with no output pass
def edit(json, path, value, convert=False, multi=False): '''Updates the value at the JSONPath endpoint. :param json JSONConfigParser: The JSON representation to act on. :param path str: A string representation of a JSONpath endpoint :param value: The value to be placed at the endpoint :param convert str: A string representing the steps to convert the value to it's final form. ''' expr = parse(path) matches = expr.find(json) if len(matches) > 1 and not multi: raise AttributeError( 'Multiple endpoints found for {}. ' 'Please specify the multi flag if this is intended.' ''.format(str(path)) ) if convert: converter = build_converter(convert) value = converter(value) for m in matches: set_on_path(json, str(m.full_path), value)
def fetch_values(self): """ fetch the data from providers and select the final values with jsonpath rules """ # use grid-proxy for authentication ckey, cert = get_key_cert() handler = HTTPSClientAuthHandler(ckey, cert) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) # request list of possible values params = {} encoded_data = urllib.urlencode(params, doseq=True) service = self.cfg url = service['url'] + encoded_data print(str(url)) req = urllib2.Request(url) # ensure we get json (sitedb is messed up and randomly returns xml) if service['jsonpath_selector']: req.add_header('Accept', 'application/json') #print req.get_full_url() stream = urllib2.urlopen(req) if service['jsonpath_selector']: response = json.load(stream) jsonpath_expr = parse(service['jsonpath_selector']) results = jsonpath_expr.find(response) stream.close() return ({'value': v.value} for v in results) return []
def validate_response_body(cls, test_output_so_far, test_config_data, config_response_body, response_body): if type(config_response_body) == dict: for k,v in config_response_body.items(): if TapiExprEvaluator.is_tapi_expr(v): if not TapiExprEvaluator.get_response_tapi_expr(v, test_output_so_far, test_config_data, response_body): return False, 'Script {0} validation failed'.format(v) elif TapiExprEvaluator.is_jsonpath_expr(k): json_expr = parse(k) try: matches = json_expr.find(json.loads(response_body)) except: return False, 'Json path expression {0} did not find any match against {1}'.format(k, response_body) if v == '*': return True, None if not (len(matches) and matches[0].value == v): return False, 'Json path expression {0} did not validate against {1}'.format(k, v) else: if TapiExprEvaluator.is_tapi_expr(config_response_body): if not TapiExprEvaluator.get_response_tapi_expr(config_response_body, test_output_so_far, test_config_data, response_body): return False, 'Script {0} validation failed'.format(config_response_body) return True, None
def validate(self, cluster): if not cluster['name']: raise ValidationException('Name must not be empty.', 'name') if not cluster['type']: raise ValidationException('Type must not be empty.', 'type') scheduler_type = parse('config.scheduler.type').find(cluster) if scheduler_type: scheduler_type = scheduler_type[0].value else: scheduler_type = QueueType.SGE config = cluster.setdefault('config', {}) scheduler = config.setdefault('scheduler', {}) scheduler['type'] = scheduler_type if not queue.is_valid_type(scheduler_type): raise ValidationException('Unsupported scheduler.', 'type') # If inserting, ensure no other clusters have the same name field amd # type if '_id' not in cluster: query = { 'name': cluster['name'], 'userId': getCurrentUser()['_id'], 'type': cluster['type'] } if self.findOne(query): raise ValidationException('A cluster with that name already ' 'exists', 'name') adapter = get_cluster_adapter(cluster) return adapter.validate()
def series_key(self, sdmxobj): # pull down dataset key dataset_dim = parse( '$.structure.dimensions.dataSet[*]').find(sdmxobj._elem) full_key_ids = [d.value['id'] for d in dataset_dim] full_key_values = [d.value['values'][0]['id'] for d in dataset_dim] key_idx = [int(i) for i in sdmxobj._elem.value['_key'].split(':')] struct_dim = parse('$.structure.dimensions.series').find( sdmxobj._elem)[0].value series_key_ids = [d['id'] for d in struct_dim] series_key_values = [d['values'][i]['id'] for i, d in zip(key_idx, struct_dim)] full_key_ids.extend(series_key_ids) full_key_values.extend(series_key_values) SeriesKeyTuple = namedtuple_factory('SeriesKey', full_key_ids) return SeriesKeyTuple._make(full_key_values)
def get_value(doc, key): if not key or not isinstance(doc, dict): raise ValueError() jsonpath_expr = parse(key) matches = jsonpath_expr.find(doc) value = None if len(matches) < 1 else matches[0].value return value
def meets_condition(response, condition): if condition.get('status_code') and response.status_code != condition['status_code']: return False if condition.get('json') and response.json() != condition['json']: return False if condition.get('text') and response.text != condition['text']: return False if condition.get('jsonpath'): for jsonpath in condition['jsonpath']: if isinstance(jsonpath['expression'], string_types): expression = jsonpath_rw.parse(jsonpath['expression']) else: expression = jsonpath['expression'] value = jsonpath['value'] results = expression.find(response.json()) if not results: return False elif len(results) == 1: if results[0].value != value: return False else: if [result.value for result in results] != value: return False if condition.get('callback'): if condition['callback'](response) == False: return False return True
def __init__(self, node): """ `node` is deserialized JSON i.e. it is a native python structure """ import jsonpath_rw self.node = jsonpath_rw.parse('`this`').find(node)[0]
def create(self, params): body = getBodyJson() self.requireParams(['name', 'type', 'size', 'profileId'], body) if not VolumeType.is_valid_type(body['type']): raise RestException('Invalid volume type.', code=400) profile_id = parse('profileId').find(body) if not profile_id: raise RestException('A profile id must be provided', 400) profile_id = profile_id[0].value profile, secret_key = _get_profile(profile_id) if not profile: raise RestException('Invalid profile', 400) if 'zone' in body: zone = body['zone'] else: zone = profile['availabilityZone'] volume = self._create_ebs(body, zone) cherrypy.response.status = 201 cherrypy.response.headers['Location'] = '/volumes/%s' % volume['_id'] return self._model.filter(volume, getCurrentUser())
def path(self, path): from jsonpath_rw import parse value = self.value if isinstance(self.value, dict) else json.loads(self.value) jsonpath_expr = parse(path) result = [match.value for match in jsonpath_expr.find(value)] result = result[0] if len(result) == 1 else result return result
def get_elements (self, json_string, expr): """ Возвращает список элементов из _json_string_, соответствующих [http://goessner.net/articles/JsonPath/|JSONPath] выражению. *Args:*\n _json_string_ - json-строка;\n _expr_ - JSONPath выражение; *Return:*\n Список найденных элементов. Если элементы не найдены, то возвращается ``None`` *Example:*\n | *Settings* | *Value* | | Library | JsonValidator | | Library | OperatingSystem | | *Test Cases* | *Action* | *Argument* | *Argument* | | Get json elements | ${json_example}= | OperatingSystem.Get File | ${CURDIR}${/}json_example.json | | | ${json_elements}= | Get elements | ${json_example} | $.store.book[*].author | =>\n | [u'Nigel Rees', u'Evelyn Waugh', u'Herman Melville', u'J. R. R. Tolkien'] """ load_input_json=self.string_to_json (json_string) # парсинг jsonpath jsonpath_expr=parse(expr) # список возвращаемых элементов value_list=[] for match in jsonpath_expr.find(load_input_json): value_list.append(match.value) if not value_list: return None else: return value_list
def parse_json( uuid, date, json_string ): print "Parsing Json: %s" % json_string weights = [match.value for match in parse( '[*].weight' ).find( json_string )] dates = [match.value for match in parse( '[*].date' ).find( json_string )] ids = [match.value for match in parse( '[*].id' ).find( json_string )] print "weightparse=%s" % weights length = len( weights ) result = list() for idx,weight in enumerate( weights ): if date == str( dates[idx] ).replace("-",""): result.append((weight, dates[idx], ids[idx])) if len(result) >1 : print( "WARN:%s HAS %s weights for %s" % (uuid, len(result), date)) return result
def parse(self, response): # x_path test if self.scraper.checker_type == '4': self.log("No 404. Item kept.", log.INFO) return if self.scraper.detail_page_content_type == 'J': json_resp = json.loads(response.body_as_unicode()) try: jsonpath_expr = parse(self.scraper.checker_x_path) except JsonPathLexerError: raise CloseSpider("Invalid checker JSONPath!") test_select = [match.value for match in jsonpath_expr.find(json_resp)] #self.log(unicode(test_select), log.INFO) else: try: test_select = response.xpath(self.scraper.checker_x_path).extract() except ValueError: self.log('Invalid checker XPath!', log.ERROR) return if len(test_select) > 0 and self.scraper.checker_x_path_result == '': self.log("Elements for XPath found on page (no result string defined).", log.INFO) if self.conf['DO_ACTION']: self._del_ref_object() return elif len(test_select) > 0 and test_select[0] == self.scraper.checker_x_path_result: self.log("XPath result string '" + self.scraper.checker_x_path_result + "' found on page.", log.INFO) if self.conf['DO_ACTION']: self._del_ref_object() return else: self.log("XPath result string not found. Item kept.", log.INFO) return
def lookup(obj, ref, qlang): def itemval(e): if isinstance(e, etree._Element): return e.text else: return e if qlang==QL.XPath: if isinstance(ref, tuple): lval = [] for iref in list(ref): out = obj.xpath(iref) if isinstance(out, list): out = [itemval(e) for e in out] else: out = itemval(out) lval.append(out) return tuple(lval) else: out = obj.xpath(ref) if isinstance(out, list): return [itemval(e) for e in out] else: return itemvalue(out) elif qlang==QL.JSONPath: out = [o.value for o in parse(ref).find(obj)] return out elif qlang==QL.CSV: return obj.get(str(ref))
def parse(self, response): # x_path test checker = response.request.meta['checker'] rpt = response.request.meta['rpt'] if checker.checker_type == '4': self.log("No 404 (%s)." % unicode(checker), log.INFO) return if rpt.content_type == 'J': json_resp = json.loads(response.body_as_unicode()) try: jsonpath_expr = parse(checker.checker_x_path) except JsonPathLexerError: raise CloseSpider("Invalid checker JSONPath (%s)!" % unicode(checker)) test_select = [match.value for match in jsonpath_expr.find(json_resp)] #self.log(unicode(test_select), log.INFO) else: try: test_select = response.xpath(checker.checker_x_path).extract() except ValueError: self.log("Invalid checker XPath (%s)!" % unicode(checker), log.ERROR) return if len(test_select) > 0 and checker.checker_x_path_result == '': self.log("Elements for XPath found on page (no result string defined) (%s). Delete reason." % unicode(checker), log.INFO) if self.conf['DO_ACTION']: self._del_ref_object() return elif len(test_select) > 0 and test_select[0] == checker.checker_x_path_result: self.log("XPath result string '%s' found on page (%s). Delete reason." % (checker.checker_x_path_result, unicode(checker)), log.INFO) if self.conf['DO_ACTION']: self._del_ref_object() return else: self.log("XPath result string not found (%s)." % unicode(checker), log.INFO) return
def _load_ref(self, ref_uri, session): """Resolve "$ref" URI ``uri``""" if self.verbose: print('Resolving $ref URI {}'.format(ref_uri), file=sys.stderr) parsed_ref_uri = self._parse_ref_uri(ref_uri) ref_file = parsed_ref_uri.netloc + parsed_ref_uri.path if not ref_file: # must be relative to current doc pass # is already in cache elif ref_file not in self.cache: self.cache[ref_uri] = self._load_for_cache(parsed_ref_uri, session) ref_json = self.cache[ref_uri] expr = jsonpath_rw.parse('$' + '.'.join(parsed_ref_uri.fragment.split('/'))) for match in expr.find(ref_json): return match.value # return first match only # If we reach here, resolution failed raise RefResolutionException( 'Could not resolve reference URI "{}"'.format(ref_uri))
def get_data_for_key(self, row): depend_data = self.data.get_depend_key(row) print "depend_data", depend_data if depend_data != None: response_data = self.run_dependent() print "response_data", type(response_data) #设置规则,在层级关系中查找需要的数据 try: json_exe = parse(depend_data) #查找内容 madle = json_exe.find(response_data) print "madle", madle if madle != None: return [math.value for math in madle][0] else: print "nothing is find" except Exception, e: raise e
def get_data_for_key(self, row): """ 获取响应数据中的依赖返回值 :param row: 行数 :return: 依赖返回值 """ depend_data = self.data.get_depend_key(row) response_data = self.run_dependent() if "validateCookie" in depend_data: cookie = OperationCookie(response_data) cookie.write_cookie() code = OperationJson('../data_config/cookie.json') return code.get_data(depend_data)[8:] else: json_rules = parse(depend_data) result = json_rules.find(json.loads(response_data.text)) return [math.value for math in result][0]
def _validate_zone(self, client, doc): try: client = get_ec2_client(doc) client.describe_availability_zones( ZoneNames=[doc['availabilityZone']]) except ClientError as ce: code = parse('Error.Code').find(ce.response) if code: code = code[0].value else: raise if code == ClientErrorCode.InvalidParameterValue: raise ValidationException( 'Invalid zone', 'availabilityZone') except EndpointConnectionError as ece: raise ValidationException(ece.message)
def _build_request_entity_selectors(self, path_item: PathItem) -> typing.Dict: res = {} for method_name in self.methods: method: Operation = getattr(path_item, method_name) operation = ( ApiOperation.UNKNOWN if method is None else ApiOperation( get_x(method, "x-hmt-operation", ApiOperation.UNKNOWN))) if operation == ApiOperation.UPSERT or operation == ApiOperation.INSERT: request_body = typing.cast(RequestBody, method.requestBody) if "application/json" in request_body.content: schema = request_body.content["application/json"].schema if schema is not None: entity_path = self._find_entity( convert_from_openapi(schema), "$") if res is not None: res[method_name] = parse(entity_path) return res
def _extract_envelope(envelope_schema, json_payload): """Extract envelope key/values from the original payload Args: schema (dict): Parsing schema envelope_schema (dict): Envelope keys to be extracted json_payload (dict): The parsed json data Returns: dict: Key/values extracted from the log to be used as the envelope """ if not isinstance(json_payload, dict): json_payload = json.loads(json_payload) LOGGER.debug('Parsing envelope keys') envelope_keys = envelope_schema.keys() envelope_jsonpath = jsonpath_rw.parse("$." + ",".join(envelope_keys)) envelope_matches = [match.value for match in envelope_jsonpath.find(json_payload)] return dict(zip(envelope_keys, envelope_matches))
def __init__(self, filename): self.items = [] data = load(open(filename)) expr = parse("dependencies[*].vulnerabilities.`parent`") for item in expr.find(data): title = f"Vulnerable dependency {item.value['fileName']}" description = f"{item.value.get('description', '')}" _severity, steps_to_reproduce = self.steps_to_reproduce(item) severity = cwe_to_severity(_severity) file_path = item.value['filePath'] self.items.append({ "title": title, "description": description, "severity": severity, "file_path": file_path, "steps_to_reproduce": steps_to_reproduce })
def find_json(json_dict, json_path, condition=[]): ''' 查找符合条件的json列表 :param json_dict: json字典结构 :param json_path: json_path :param condition: 筛选条件 [==|=], !=, >, <, >=, <= , :return: ''' jsonpath_expr = parse(json_path) male = jsonpath_expr.find(json_dict) data = [match.value for match in male] new_data = [] EXPRESS = ['==', '!=', '<=', '>=', '>', '<'] for d_temp in data: #d = unicode_dict_list_to_str(d_temp) d = d_temp temp = 0 for i in condition: for e in EXPRESS: flag = False if e in i and e+'=' not in i: k, v = i.split(e) #执行条件筛选 if isinstance(d, (dict)): b = d[k] if isinstance(d[k], (unicode)): value = d[k].encode('utf-8') if isinstance(value, basestring): exec ("if \"%s\" %s %s: flag=True" % (d[k].encode('utf-8'), e, v)) else: exec ("if %s %s %s: flag=True" % (d[k].encode('utf-8'), e, v)) else: exec ("if d['%s'] %s %s: flag=True" % (k, e, v)) if flag: #条件有几个temp就加几次,temp数(符合条件数)和条件数相等才能添加到new_data里面 temp += 1 break if not flag: break #如果连之前的条件都不满足就没有必要再比下去了直接break if temp == len(condition): new_data.append(d) return utils.pretty_dict2(new_data), new_data
def main(arguments): """Program entry point""" arguments = docopt(__doc__, version="0") # Load spec from file with open(arguments["SPEC_FILE"], "r") as spec_file: spec = json.loads(spec_file.read()) # Create default dir structure for i in ["resources", "responses"]: os.makedirs(os.path.join(arguments["OUT_DIR"], i), exist_ok=True) # Generate resources for component_name, component_spec in spec["components"]["schemas"].items( ): with open( os.path.join(arguments["OUT_DIR"], "resources", component_name + ".json"), "w", ) as out_file: out_file.write( json.dumps( generate_resource_example(component_spec["properties"], [component_name]))) # Pull out responses match_expr = parse( "paths.*.*.(response|(responses.*)).content.*.(example|(examples.*.value))" ) for match in match_expr.find(spec): if "patch" in str(match.full_path): # PATCHes are not FHIR resources, so we should not be validating them continue with open( os.path.join( arguments["OUT_DIR"], "responses", str(match.full_path).replace("/", "_") + ".json", ), "w", ) as out_file: out_file.write(json.dumps(match.value))
def on_message(mosq, userdata, msg): """ This is invoked when we get our own message back. Verify that it is actually our message and if so, we've completed a round-trip. """ #print "on_message", msg.topic, str(msg.payload) global message global status payload = msg.payload if module_jsonpath_rw and module_json: if args.mqtt_jsonpath is not None: try: jspayload = json.loads(payload) jspath = parse(args.mqtt_jsonpath) extractpayload = [match.value for match in jspath.find(jspayload)] payload = extractpayload[0] except: payload = '' pass print "on_message", msg.topic, str(payload) elapsed = (time.time() - userdata['start_time']) userdata['have_response'] = True status = 2 if args.short_output == True: message = "value=%s | response_time=%.2f value=%s" % (str(payload), elapsed, str(payload)) else: message = "message from %s at %s in %.2fs | response_time=%.2f value=%s" % (args.check_subscription, args.mqtt_host, elapsed, elapsed, str(payload)) try: if (args.mqtt_operator == 'lt' or args.mqtt_operator == 'lessthan') and float(payload) < float(args.mqtt_value): status = 0 if (args.mqtt_operator == 'gt' or args.mqtt_operator == 'greaterthan') and float(payload) > float(args.mqtt_value): status = 0 if (args.mqtt_operator == 'eq' or args.mqtt_operator == 'equal') and str(payload) == args.mqtt_value: status = 0 if (args.mqtt_operator == 'ct' or args.mqtt_operator == 'contains') and str(payload).find(args.mqtt_value) != -1: status = 0 except: pass
def main(*argv): parser = argparse.ArgumentParser( description= 'Search JSON files (or stdin) according to a JSONPath expression.', formatter_class=argparse.RawTextHelpFormatter, epilog=""" Quick JSONPath reference (see more at https://github.com/kennknowles/python-jsonpath-rw) atomics: $ - root object `this` - current object operators: path1.path2 - same as xpath / path1|path2 - union path1..path2 - somewhere in between fields: fieldname - field with name * - any field [_start_?:_end_?] - array slice [*] - any array index """) parser.add_argument('expression', help='A JSONPath expression.') parser.add_argument('files', metavar='file', nargs='*', help='Files to search (if none, searches stdin)') args = parser.parse_args(argv[1:]) expr = parse(args.expression) glob_patterns = args.files if len(glob_patterns) == 0: # stdin mode print_matches(find_matches_for_file(expr, sys.stdin)) else: # file paths mode for pattern in glob_patterns: for filename in glob.glob(pattern): with open(filename) as f: print_matches(find_matches_for_file(expr, f))
def get_value(expression, body=None, value_type="string", get_tag=False, expression_instead_none=False): if isinstance(body, str): body = loads(body) if not expression: return '' positions = search(r'\$\{(?:(.*))\}', expression) if positions is not None: p1 = positions.regs[-1][0] p2 = positions.regs[-1][1] else: p1 = 0 p2 = len(expression) target_str = str(expression[p1:p2]) if get_tag: return target_str full_value = None try: if isinstance(body, dict) and target_str.split()[0] in body: if value_type.lower() == "string": full_value = expression[0:max(abs(p1 - 2), 0)] + body[ target_str.split()[0]] + expression[p2 + 1:len(expression)] else: full_value = body.get(target_str.split()[0]) elif isinstance(body, dict): jsonpath_expression = parse(target_str) jsonpath_match = jsonpath_expression.find(body) if jsonpath_match: full_value = jsonpath_match[0].value elif isinstance(body, (str, bytes)): search_result = search(expression, body) if search_result.groups(): full_value = search_result.group(0) elif expression_instead_none: full_value = expression except Exception as e: log.exception(e) finally: return full_value
def __enter__(self): # Do we need to get the session id for this user if not self._newt_session_id: headers = {'Girder-Token': self._girder_token} url = '%s/newt/sessionId' % cumulus.config.girder.baseUrl r = requests.get(url, headers=headers) check_status(r) session_id = parse('sessionId').find(r.json()) if not session_id: raise Exception('No NEWT session ID present') self._session = requests.Session() self._newt_session_id = session_id[0].value self._session.cookies.set('newt_sessionid', self._newt_session_id) return self
def get_datasources(self, item): # Query datasources. _finder = parse("$..datasource") _datasources = _finder.find(item) # Compute unique list of datasources. datasources = [] for _ds in _datasources: if not _ds.value: continue if isinstance(_ds.value, Munch): value = dict(_ds.value) else: value = str(_ds.value) if value not in datasources: datasources.append(value) return datasources
def notify(request): if request.method == 'POST': rulesPath = os.path.join(os.getcwd(), "rules") requestRule = request.body bFinRule = False matchRule = '' for (dirpath, dirnames, filenames) in walk(rulesPath): for filename in filenames: if filename.endswith(".rle"): filePath = os.path.join(rulesPath, filename) f = open(filePath, 'r') fdata = "" while 1: line = f.readline() if not line: break fdata += line f.close() # compare simple strings if requestRule.replace(' ', '') == fdata.replace(' ', ''): bFinRule = True matchRule = filename break else: # compare complex rule try: complexRule = parse(fdata) js = json.loads(requestRule) m = [match.value for match in complexRule.find(js)] if not len(m) == 0: bFinRule = True matchRule = filename break except Exception, e: pass else: pass finally: pass
def step_impl(context, jsonpath, thing, value): if not context.content_json: ## Apparently no JSON at all... assert True is False else: jsonpath_expr = jsonpath_rw.parse(jsonpath) res = jsonpath_expr.find(context.content_json) if not res[0]: assert True is False else: if thing == "string": assert res[0].value == value elif thing == "integer": assert res[0].value == int(value) elif thing == "float": assert res[0].value == float(value) else: ## Not a thing we know how to deal with yet. assert True is False
def find_calc_types(self, params): fields = ['access', 'properties.calculationTypes'] query = {} if 'moleculeId' in params: query['moleculeId'] = ObjectId(params['moleculeId']) calcs = self._model.find(query, fields=fields) allTypes = [] for types in calcs: calc_types = parse('properties.calculationTypes').find(types) if calc_types: calc_types = calc_types[0].value allTypes.extend(calc_types) typeSet = set(allTypes) return list(typeSet)
def convert(cur_format, new_format, content_data, field_name): content_data_loaded = json.loads(str(content_data)) if isinstance(content_data_loaded, float) or isinstance( content_data_loaded, int): new_value = convert_map[cur_format][new_format](content_data_loaded) new_content_data = json.dumps(new_value) else: pattern = parse(field_name) match = pattern.find(content_data_loaded) if match: cur_value = match[0].value new_value = convert_map[cur_format][new_format](cur_value) new_content_data = update_json(content_data_loaded, get_path(match[0]), new_value) new_content_data = json.dumps(new_content_data) else: raise Exception('No match found') return new_content_data
def parse_fields(self, field, message): fval = self.cfg.get(field) if not fval: return if isinstance(fval, six.integer_types): return fval try: parts = jsonpath_rw.parse(fval) except Exception as e: raise MeterDefinitionException( _LE("Parse error in JSONPath specification " "'%(jsonpath)s': %(err)s") % dict(jsonpath=parts, err=e), self.cfg) values = [ match.value for match in parts.find(message) if match.value is not None ] if values: return values[0]
def headmark(self): dt = '' r = get( 'https://api.xuangubao.cn/api/pc/msgs?%slimit=50&subjids=9,469,35,10' % dt) jo = r.json() jsonpath_expr = parse('$.NewMsgs[*]') res = OrderedDict() for i, match in enumerate(jsonpath_expr.find(jo)): jo = match.value ts, title, summary = jo['CreatedAt'], jo['Title'], jo['Summary'] stocks = jo.get('Stocks', ) res['[%s]%s' % (ts, title)] = { 'title': title, 'ts': ts, 'summary': summary, 'stocks': stocks } return res
def _handle_function(input_data, func): function = getattr(functions, func.name, None) assert function is not None, ("{} is not an available function").format( func.name) values = [] for value in func.values: expr = parse(value) parsed_value = [m.value for m in expr.find(input_data)][0] values.append(parsed_value) assert len(func.values) == len(values), "Unable parse some values" args = values + func.args _required_no_of_args = inspect.getfullargspec(function).args assert len(_required_no_of_args) == len( args), "Number of arguments mismatch. {} != {}".format( len(_required_no_of_args), len(args)) return function(*args)
def get_table_from_jpath(jobj, jpath, minrow, mincol): if jobj is None: return [] if '_id' in jobj: cdr_id = jobj['_id'] elif 'cdr_id' in jobj: cdr_id = jobj['_id'] else: raise Exception('cdr_id not found in {}'.format(jobj)) my_parser = parse(jpath) res = [] for match in my_parser.find(jobj): val = match.value if val is not None: row, col = get_table_dim(val) if row >= minrow or col >= mincol: val['cdr_id'] = cdr_id res.append(val) return res
def capture_box_score_json_experiment(self): uri = "http://api.nfldata.apiphany.com/nfl/v2/json/BoxScores/2015PRE/4?key=%s" % srkey print uri resp = requests.get(uri) if resp.status_code != 200: raise Exception(resp.status_code) #clean it up? load and then return to string parsed = json.loads(resp.text) tostr = json.dumps(parsed) # attempt raw parse jsonpath_expr = parse(tostr) items = resp.json() fname = datetime.now().strftime('%Mminute-%Hhour') f = open('%s.json' % fname, 'w') f.write(str(items)) f.close()
def get_response_data(self): ''' 取出依赖的多个字段 :return: ''' depend_fields_list = self.get_response_field() depend_case_ids = self.get_case_id() fields_list = [] for i, depend_case_id in enumerate(depend_case_ids): response_data = self.send_depend_request(depend_case_id) fields = [] if depend_fields_list[i]: for depend_field in depend_fields_list[i]: depend_field = parse(depend_field) madle = depend_field.find(response_data) res = [match.value for match in madle][0] fields.append(res) fields_list.append(fields) return fields_list
def count_tables(jobj, jpath, minrow, mincol): # print 'here we are!' if jobj is None: return 0 if '_id' in jobj: cdr_id = jobj['_id'] elif 'cdr_id' in jobj: cdr_id = jobj['cdr_id'] else: raise Exception('cdr_id not found in {}'.format(jobj)) count = 0 my_parser = parse(jpath) for match in my_parser.find(jobj): val = match.value if val is not None: row, col = get_table_dim(val) if row >= minrow or col >= mincol: count += 1 return count
def analyze_record_delta(self, file_name, test_event): """Provide some additional context on why this test failed. This will perform some analysis of the test record to determine which keys are missing or which unnecessary keys are causing the test to fail. Any errors are appended to a list of errors so they can be printed at the end of the test run. Args: file_name (str): Name of file containing the test event test_event (dict): Actual record data being tested """ base_message = ('Invalid test event in file \'{}.json\' with description ' '\'{}\'.'.format(file_name, test_event['description'])) if not self.check_log_declared_in_sources(base_message, test_event): return log_type = test_event['log'] if log_type not in self.cli_config['logs']: message = ('{} Log (\'{}\') declared in test event does not exist in ' 'logs.json'.format(base_message, log_type)) self.status_messages.append(StatusMessage(StatusMessage.FAILURE, message)) return config_log_info = self.cli_config['logs'][log_type] schema_keys = config_log_info['schema'] envelope_keys = config_log_info.get('configuration', {}).get('envelope_keys') if envelope_keys: if self.report_envelope_key_error(base_message, envelope_keys, test_event['data']): return # Check is a json path is used for nested records json_path = config_log_info.get('configuration', {}).get('json_path') if json_path: records_jsonpath = jsonpath_rw.parse(json_path) for match in records_jsonpath.find(test_event['data']): self.report_record_delta(base_message, log_type, schema_keys, match.value) return self.report_record_delta(base_message, log_type, schema_keys, test_event['data'])
def send_requests(s, testdata): '''封装requests请求''' method = testdata["请求方式"] url = testdata["url"] header = eval(testdata["header"]) data = eval(testdata["请求数据"]) test_nub = testdata['ID'] expect = eval(testdata['预期结果']) depend_case = testdata['依赖id'] depend_data = testdata['依赖数据'] if depend_case == "": print("*******正在执行用例:----- %s ----**********" % test_nub) print("请求方式:%s, 请求url:%s" % (method, url)) print("预期结果: %s" % expect) res = s.request(method=method, url=url, headers=header, data=json.dumps(data)) print("请求响应结果: %s" % res.text) else: dependdata = dependent_data(depend_case) depend_response_data = dependdata.run_dependent(depend_case) print(depend_response_data) jsonpath_expr = parse(depend_data) madle = jsonpath_expr.find(depend_response_data) depend_response_data_new = [match.value for match in madle][0] depend_data_new = testdata['依赖数据所属字段'] header[depend_data_new] = depend_response_data_new print("*******正在执行用例:----- %s ----**********" % test_nub) print("请求方式:%s, 请求url:%s" % (method, url)) print("预期结果: %s" % expect) print("依赖ID: %s" % depend_case) print("依赖数据: %s" % depend_data) print("依赖数据的值: %s" % depend_response_data_new) print(type(header)) print(header) print("===============================") print(json.dumps(data)) print("===============================") print(method) print("===============================") print(url) print("===============================") res1 = s.request(method=method, url=url, headers=header, data=json.dumps(data)) print("请求响应结果: %s" % res1.text) return res1
def extract_json(self, text, path): try: if not isinstance(text, string_types): logging.warning("[extract_json] unknow text type %s" % type(text)) return [] data_json = json.loads(text) logging.debug("[extract_json] Parse json text successfully.") jsonpath_expr = parse(path) extract = [match.value for match in jsonpath_expr.find(data_json)] # 转换为string list for idx, e in enumerate(extract): if not isinstance(e, string_types): extract[idx] = json.dumps(e) return extract except Exception: logging.exception("extract_json error: %s" % path) raise
def run(host, keyword, resultname, basicQuery, repeatQuery, jsonPath): result = "" if jsonPath != "": jsonpath_expr = parse(jsonPath) with open(os.path.join(keyword), 'r') as f: query = host + basicQuery tempQ = query req = requests.get(query) ret = req.text ret = json.loads(ret) if req.status_code == 200: result += (tempQ + " : OK\n") result += ("검색결과 갯수 : ") result += str(ret["response"]["rows"]) result += "\n" else: result += (tempQ + " : Not OK\n") repeatQuery = repeatQuery.split("&") basicQuery = basicQuery.split("&") while (True): line = f.readline() if not line: break for basicQ in basicQuery: for repeatQ in repeatQuery: temp = basicQ.split("=") temp = temp[0] if repeatQ == temp: tempQ = query.replace(basicQ, (repeatQ + "=" + line)) req = requests.get(tempQ) ret = req.text ret = json.loads(ret) if req.status_code == 200: result += (tempQ + " : OK\n") result += ("검색결과 갯수 : ") result += str(ret["response"]["rows"]) result += "\n" else: result += (tempQ + " : Not OK\n") print result with open(os.path.join(BASE_DIR, resultname), 'w+') as result_file: result_file.write(result)
def _on_response_filter_changed(self, entry: Gtk.SearchEntry): filter_text = entry.get_text() if filter_text == '': self._set_response_text() ct = get_content_type(self.last_response) try: if ct == 'application/json': path_expr = jsonpath_rw.parse(filter_text) j = self.last_response.json() match_text = json.dumps( [match.value for match in path_expr.find(j)], indent=4) or 'No matches found' self.response_text.get_buffer().set_text(match_text) elif ct in {'text/xml', 'application/xml'}: root = etree.fromstring(self.last_response.text) matches = root.xpath(filter_text) matches_root = etree.Element('matches') for m in matches: matches_root.append(m) matches_html = etree.tostring(matches_root, encoding='unicode', pretty_print=True) self.response_text.get_buffer().set_text(matches_html) elif ct == 'text/html': root = html.fromstring(self.last_response.text) matches = root.xpath(filter_text) matches_root = etree.Element('matches') for m in matches: matches_root.append(m) matches_html = etree.tostring(matches_root, encoding='unicode', pretty_print=True) self.response_text.get_buffer().set_text(matches_html) else: log.warning( 'Got unexpected content type %s when filtering response.', ct) except Exception as e: log.debug('Failed to filter response json %s', e)