def clean_hourly(forecast): parser = jp.parse("[*].fecha") clean_data = {} for match in parser.find(forecast): [value_path, *_] = [_ for _ in get_path(match)] clean_data[value_path] = match.value path = "[*].{}" for sensor, [jpath, field] in HOURLY_SENSORS_CONVERT.items(): parser = jp.parse(path.format(jpath)) for match in parser.find(forecast): value = match.value [value_path, *_] = [_ for _ in get_path(match)] fecha = clean_data[value_path] periodo = f"{fecha[:10]}T{value['periodo']}:00:00" if periodo not in clean_data.keys(): clean_data[periodo] = {ATTR_FORECAST_TIME: periodo} if value.get(field, None) is not None: clean_data[periodo][sensor] = value[field] valores = [ valor for valor in clean_data.values() if not isinstance(valor, str) ] for valor in valores: valor[ATTR_FORECAST_WIND_BEARING] = valor[ ATTR_FORECAST_WIND_BEARING][0] valor[ATTR_FORECAST_WIND_SPEED] = float( valor[ATTR_FORECAST_WIND_SPEED][0]) return valores
def clean_daily(forecast): parser = jp.parse("[*].fecha") clean_data = [{ ATTR_FORECAST_TIME: match.value } for match in parser.find(forecast)] path = "[*].{}" for sensor, [jpath, field] in DAILY_SENSORS_CONVERT.items(): parser = jp.parse(path.format(jpath)) for match in parser.find(forecast): value = match.value [value_path, *_] = [_ for _ in get_path(match)] if isinstance(value, dict): periodo = value.get("periodo", None) if periodo: if periodo not in clean_data[value_path].keys(): clean_data[value_path][periodo] = {} clean_data[value_path][periodo][sensor] = value[ field] else: clean_data[value_path][sensor] = value[field] else: clean_data[value_path][sensor] = value return clean_data
def generate_json_paths(self): """Given a scalar or ordered list of strings parse them to generate JSONPaths""" if isinstance(self.input_fields, basestring): try: self.jsonpaths = parse(self.input_fields) except Exception as exception: print "input_fields failed {}".format(self.input_fields) raise exception elif isinstance(self.input_fields, types.ListType): self.jsonpaths = list() for input_field in self.input_fields: self.jsonpaths.append(parse(input_field))
def get_json_elements(self, json_string, expr): """ Get list of elements from json_string for matching expression. == Arguments == - json_string : JSON string;\n - expr : JSONPath expression;\n == Returns == List of found elements or ``None`` if no elements were found == Example Test Cases == 1. To Get Json Elements from a file using "Load JSON From File" keyword: | ${json_data} | Load JSON From File | ../JSON/DataSetFiles/OpenAPIData/JSonFiles/json_example.json | | ${json_String_data} | Json To String | ${json_data} | | @{json_elements} | Get Json Elements | ${json_String_data} | $..number | => | [ 90-4567-8888 | 0123-4567-8910 ] """ jsonpath_expr = parse(expr) load_input_json = self.string_to_json(json_string) value_list = [] if not jsonpath_expr.find(load_input_json): raise JsonValidatorError( "Invalid Input Error ! \nPath {} doesn't exist".format(expr)) for match in jsonpath_expr.find(load_input_json): value_list.append(match.value) if not value_list: return None else: return value_list
def delete_object_from_json(self, json_data, json_path): """Deletes Object From Json data using json_path expression == Arguments == - json_data: json data as a dictionary object.\n - json_path: jsonpath expression\n == Returns == Returns a new json_data == Example Test Cases == | ${json_data} | Load JSON From File | ../JSON/DataSetFiles/OpenAPIData/JSonFiles/json_example.json | | ${new_json_data} | Delete Object From Json | ${json_data} | $..address.streetAddress | """ json_path_expr = parse(json_path) if not json_path_expr.find(json_data): raise JsonValidatorError( "Invalid Input Error ! \nPath {} doesn't exist".format( json_path)) for match in json_path_expr.find(json_data): path = match.path if isinstance(path, Index): del (match.context.value[match.path.index]) elif isinstance(path, Fields): del (match.context.value[match.path.fields[0]]) return json_data
def add_object_to_json(self, json_data, json_path, object_to_add): """ Adds a dictionary object to json_data using json_path == Arguments == - json_data: json data as a dictionary object.\n - json_path: jsonpath expression.\n - object_to_add: dictionary object to add to json_data which is matched by json_path.\n == Returns == Returns a new json data object. == Example Test Cases == | ${json_data} | Load JSON From File | ../JSON/DataSetFiles/OpenAPIData/JSonFiles/example.json | &{dict} | Create Dictionary | latitude=13.1234 | longitude=130.1234 | | ${json} | Add Object To Json | ${json_data} | $..address | ${dict} | """ json_path_expr = parse(json_path) if not json_path_expr.find(json_data): raise JsonValidatorError( "Invalid Input Error ! \nPath {} doesn't exist".format( json_path)) for match in json_path_expr.find(json_data): if type(match.value) in [dict, OrderedDict]: match.value.update(object_to_add) else: raise JsonValidatorError( "Invalid Object type. Object is neither Dictionary nor List" ) if isinstance(json_data, list): return json_data[0] else: return json_data
def search_value(self, xpath: str, default: Optional[Any] = None, fail_if_no_match: Optional[bool] = False) -> Optional[Any]: """ Try to find a value in the result. see https://github.com/kennknowles/python-jsonpath-rw#jsonpath-syntax :param str xpath: a xpath filter :param any default: default value if not found :param bool fail_if_no_match: Raise a ValidationError if no matches :return: the value found or None """ matches = [ match.value for match in jsonpath_rw_ext.parse(xpath).find(self.retval) ] if len(matches) == 0: if fail_if_no_match is True: raise ValidationError("No value found for xpath: '{}'".format( xpath )) else: return default elif len(matches) == 1: return matches[0] else: return matches
def select_json_objects(self, json_string, expr): """ Return list of elements from json_string, matching expression. == Arguments == - json_string : JSON string;\n - expr : ObjectPath expression;\n == Returns == List of found elements. If no elements were found, empty list will be returned 1. To Select Json Object from a file using "Load JSON From File" keyword: | ${json_data} | Load JSON From File | ../JSON/DataSetFiles/OpenAPIData/JSonFiles/json_example.json | | ${json_String_data} | Json To String | ${json_data} | | @{json_elements} | Select Json Objects | ${json_String_data} | $..number | | Should Be Equal As Strings | @{Json_Objects}[1] | 0123-4567-8910 | """ jsonpath_expr = parse(expr) load_input_json = self.string_to_json(json_string) load_input_json1 = json.loads(json.dumps(load_input_json)) value_list = [] if not jsonpath_expr.find(load_input_json1): raise JsonValidatorError( "Invalid Input Error ! \nPath {} doesn't exist".format(expr)) for match in jsonpath_expr.find(load_input_json1): value_list.append(match.value) if not value_list: return None else: return value_list
def get_elements(self, json_string, expr): """ Get list of elements from _json_string_, matching [http://goessner.net/articles/JsonPath/|JSONPath] expression. *Args:*\n _json_string_ - JSON string;\n _expr_ - JSONPath expression; *Returns:*\n List of found elements or ``None`` if no elements were found *Example:*\n | *Settings* | *Value* | | Library | JsonValidator | | Library | OperatingSystem | | *Test Cases* | *Action* | *Argument* | *Argument* | | Get json elements | ${json_example}= | OperatingSystem.Get File | ${CURDIR}${/}json_example.json | | | ${json_elements}= | Get elements | ${json_example} | $.store.book[*].author | =>\n | [u'Nigel Rees', u'Evelyn Waugh', u'Herman Melville', u'J. R. R. Tolkien'] """ load_input_json = self.string_to_json(json_string) # parsing jsonpath jsonpath_expr = parse(expr) # list of returned elements value_list = [] for match in jsonpath_expr.find(load_input_json): value_list.append(match.value) if not value_list: return None else: return value_list
def update_value_to_json(self, json_object, json_path, new_value): """Update value to JSON using JSONPath Arguments: - json_object: json as a dictionary object. - json_path: jsonpath expression - new_value: value to update Return new json_object Examples: | ${json_object}= | Update Value To Json | ${json} | $..address.streetAddress | Ratchadapisek Road | """ json_path_expr = parse(json_path) if new_value.lower().startswith('(int)'): new_value = int(new_value.lstrip('(int)')) elif new_value.lower().startswith('(bool)'): if new_value.lower().find('true'): new_value = True else: new_value = False else: pass if new_value == u"None": new_value = None for match in json_path_expr.find(json_object): path = match.path if isinstance(path, Index): match.context.value[match.path.index] = new_value elif isinstance(path, Fields): match.context.value[match.path.fields[0]] = new_value return json_object
def build(self) -> None: try: self._jsonpath = jsonpath_rw_ext.parse(self.expr) self.built = True except (JsonPathLexerError, Exception) as exc: # jsonpath_rw.parser.JsonPathParser.p_error raises exc of Exception type raise ExprError(extractor=self, exc=exc) from exc
def get_results(self): """Parse the JSON and yield a structured response""" # pylint: disable=too-many-nested-blocks for data in self.get_data(): if "jsonpath" in self.conf: jsonpaths = self.conf["jsonpath"] if not isinstance(jsonpaths, list): jsonpaths = [jsonpaths] for jsonpath_conf in jsonpaths: new_data = OrderedDict() for (key, jsonpath) in jsonpath_conf.items(): expr = jsonpath_rw_ext.parse(jsonpath) for match in expr.find(data): if key in new_data: if not isinstance(new_data[key], list): new_data[key] = [new_data[key]] new_data[key].append(match.value) else: new_data.update({key: match.value}) # print(new_data) yield new_data else: if self.conf.get("multi_json", False): for result in data: yield result else: yield data
def _json_path_search(self, json_dict, expr): """ Scan JSON dictionary with using json-path passed sting of the format of $.element..element1[index] etc. *Args:*\n _json_dict_ - JSON dictionary;\n _expr_ - string of fuzzy search for items within the directory;\n *Returns:*\n List of DatumInContext objects: ``[DatumInContext(value=..., path=..., context=[DatumInContext])]`` - value - found value - path - value selector inside context.value (in implementation of jsonpath-rw: class Index or Fields) *Raises:*\n JsonValidatorError """ path = parse(expr) results = path.find(json_dict) if len(results) is 0: raise JsonValidatorError("Nothing found in the dictionary {0} using the given path {1}".format( str(json_dict), str(expr))) return results
def __init__(self, jsonpath_expression): """ :param str jsonpath_expression: JSONPath expression """ try: self.expression = parse(jsonpath_expression) except (JsonPathLexerError, TypeError) as error: raise InvalidRoutingConditionError(str(error)) from error
def find(self, path): jpath = jsonpath.parse(path) for found in jpath.find(self.parse()): try: return str(found.value) except (AttributeError, KeyError): pass return None
def getResourceItem(self, extrapolated_properties, *args): error = None properties = None file = args[0] try: with open(file, 'r') as conf_file: dataStr = conf_file.read() try: jsObj = None if dataStr is not None: jsObj = json.loads(dataStr) if not any(extrapolated_properties): properties = self.generateProperties(jsObj) else: for property in extrapolated_properties: if property: values = None try: parser = jp.parse(property) if jsObj is not None: for match in parser.find(jsObj): if match.value is not None: if values is None: values = [] values.append(match.value) if not properties: properties = [] if values is None: properties.append( Property( str(property), values, Error(type=Error. MISSING_PROPERTY, message="No property: " + str(property)))) else: properties.append( Property(str(property), values)) except Exception as e: print e if not properties: properties = [] properties.append( Property( str(property), values, Error( type=Error.SYNTAX_ERROR, message= "Parser error, check jsonpath for property '" + str(property)))) except Error: error = Error(type=Error.PARSER_ERROR, message="Invalid json file '" + file + "'") except IOError as e: print e error = Error(Error.FILE_NOT_FOUND, "File not found") return Item(file, properties, error)
def _update_value_to_json(self, json_data, json_path, new_value): """Update value to JSON using JSONPath == Arguments == - json_data: json data as a dictionary object.\n - json_path: jsonpath expression\n - new_value: value to update\n == Returns == Returns new json_object """ json_path_expr = parse(json_path) if not json_path_expr.find(json_data): raise JsonValidatorError( "Invalid Input Error ! \nPath {} doesn't exist".format( json_path)) for match in json_path_expr.find(json_data): path = match.path if isinstance(path, Fields): fields_type = match.context.value[match.path.fields[0]] if isinstance(fields_type, str): match.context.value[match.path.fields[0]] = new_value elif isinstance(fields_type, bool): if new_value.lower() == "true": match.context.value[match.path.fields[0]] = True elif new_value.lower() == "false": match.context.value[match.path.fields[0]] = False else: raise AssertionError( "Please provide a valid boolean value as true or false. The provided value to be updated is: {}" .format(new_value)) elif isinstance(fields_type, int): match.context.value[match.path.fields[0]] = int(new_value) elif isinstance(fields_type, float): match.context.value[match.path.fields[0]] = float( new_value) elif isinstance(path, Index): path_type = match.context.value[match.path.index] if isinstance(path_type, str): match.context.value[match.path.index] = new_value elif isinstance(path_type, bool): if new_value.lower() == "true": match.context.value[match.path.index] = True elif new_value.lower() == "false": match.context.value[match.path.index] = False else: raise AssertionError( "Please provide a valid boolean value as true or false. The provided value to be updated is: {}" .format(new_value)) elif isinstance(path_type, int): match.context.value[match.path.index] = int(new_value) elif isinstance(path_type, float): match.context.value[match.path.index] = float(new_value) return json_data
def get_value(data, path): path_expr = parse(path) temp = path_expr.find(data) result = [] for val in temp: result.append(val.value) if result: return result[0] else: return None
def get_value_from_json(self, json_object, json_path): """Get Value From JSON using JSONPath Arguments: - json_object: json as a dictionary object. - json_path: jsonpath expression Return array of values Examples: | ${values}= | Get Value From Json | ${json} | $..phone_number | """ json_path_expr = parse(json_path) return [match.value for match in json_path_expr.find(json_object)]
def extract(body, path, multiple=False): jsonpath_expr = deepcopy(parse( path)) # Use parse from jsonpath-rw-ext here to parse with filters. result = [match.value for match in jsonpath_expr.find(body)] if multiple: return result else: try: return result[0] except IndexError as e: raise DataError( "Could not find requested test data in files. Check test data." )
def __init__(self, expr: str) -> None: super(JSONExtractor, self).__init__(expr) if _missing_jsonpath_rw_ext: _missing_dependency("jsonpath-rw-ext") # Third Party Library from jsonpath_rw.lexer import JsonPathLexerError try: self._jsonpath = jsonpath_rw_ext.parse(self.expr) except (JsonPathLexerError, Exception) as exc: # jsonpath_rw.parser.JsonPathParser.p_error raises exc of Exception type raise ExprError(extractor=self, exc=exc) from exc
def findall(self, query) -> list: ''' Find all elements with JsonPath query. Arguments: query: JsonPath Query string Returns: python list of found Json elements. ''' jsonpath_expr = parse(query) return [Json.from_object(match.value, allow_any=True) for match in jsonpath_expr.find(self._container)]
def _parse(self, expr: str) -> JSONPath: """ Parse JSONPath expression and store it into the cache. *Args:*\n _expr_ - JSONPath expression; *Returns:*\n Parsed JSONPath expression. """ if expr not in self._parser_cache: self._parser_cache[expr] = parse(expr) return self._parser_cache[expr]
def json_path_parse_public(self, json_path, json_obj): if json_path: # 定义要获取的key json_exe = parse(json_path) # 定义响应数据,key从响应数据里获取 madle = json_exe.find(json_obj) # math.value返回的是一个list,可以使用索引访问特定的值jsonpath_rw的作用就相当于从json里面提取响应的字段值 try: math_value = [i.value for i in madle] return math_value except IndexError as indexerror: return None else: return None
def delete_object_from_json(self, json_object, json_path): """Delete Object From JSON using json_path Arguments: - json_object: json as a dictionary object. - json_path: jsonpath expression Return new json_object Examples: | ${json_object}= | Delete Object From Json | ${json} | $..address.streetAddress | """ json_path_expr = parse(json_path) for match in json_path_expr.find(json_object): path = match.path if isinstance(path, Index): del (match.context.value[match.path.index]) elif isinstance(path, Fields): del (match.context.value[match.path.fields[0]]) return json_object
def update_value_to_json(self, json_object, json_path, new_value): """Update value to JSON using JSONPath Arguments: - json_object: json as a dictionary object. - json_path: jsonpath expression - new_value: value to update Return new json_object Examples: | ${json_object}= | Update Value To Json | ${json} | $..address.streetAddress | Ratchadapisek Road | """ json_path_expr = parse(json_path) for match in json_path_expr.find(json_object): path = match.path if isinstance(path, Index): match.context.value[match.path.index] = new_value elif isinstance(path, Fields): match.context.value[match.path.fields[0]] = new_value return json_object
def add_object_to_json(self, json_object, json_path, object_to_add): """Add an dictionary or list object to json object using json_path Arguments: - json_object: json as a dictionary object. - json_path: jsonpath expression - object_to_add: dictionary or list object to add to json_object which is matched by json_path Return new json object. Examples: | ${dict}= | Create Dictionary | latitude=13.1234 | longitude=130.1234 | | ${json}= | Add Object To Json | ${json} | $..address | ${dict} | """ json_path_expr = parse(json_path) for match in json_path_expr.find(json_object): if type(match.value) is dict: match.value.update(object_to_add) if type(match.value) is list: match.value.append(object_to_add) return json_object
def _load_config(self, filename): config = self.default_config if filename: try: with open(filename, 'r') as fd: yml_config = yaml.safe_load(fd.read()) if len(yml_config): config = [(list(item.keys())[0], list(item.values())[0]) for item in yml_config] else: LOG.warning('Invalid config file {file}. Using default ' 'configuration'.format(file=filename)) except (IOError, yaml.scanner.ScannerError) as err: LOG.warning('Error: {err}. Using default ' 'configuration'.format(err=err)) self.parsers = {} for col, path in config: self.parsers[col] = jp.parse(path) return config
def getResourceItem(self, extrapolated_properties, *args): properties = None error = None file = args[0] try: with open(file, 'r') as conf_file: dataStr = conf_file.read() if dataStr is not None: pattern = re.compile('^include.*', flags=re.MULTILINE) dataStr = pattern.sub('', dataStr) try: with make_loader() as loader: confDict = None if dataStr is not None: confDict = loader.loads(dataStr) for property in extrapolated_properties: if property: parser = jp.parse(property) values = [] if confDict is not None: for match in parser.find(confDict): values.append(match.value) if not properties: properties = [] if values is None: properties.append( Property( str(property), values, Error(type=Error. MISSING_PROPERTY, message="No property: " + str(property)))) else: properties.append( Property(str(property), values)) except Error: error = Error(type=Error.PARSER_ERROR, message="Invalid conf file '" + file + "'") except IOError as e: print e error = Error(Error.FILE_NOT_FOUND, "File not found") return Item(file, properties, error)
def _parse_operator(cls, rule_operator_value): """ Parses part of rule, which contains rule, operator and comparison value :param rule_operator_value: string which contains rule, operator and comparison value :return: pattern, operator, comparison_value, parsed pattern :raise Exception in case of wrong pattern or WardenFilterRuleFormatError """ # go through all supported operators and split rule with operator, which was used for operator in cls.SUPPORTED_OPERATORS: if operator in rule_operator_value: # operator found, get pattern and compared value by split and return it with found operator pattern, comparison_value = rule_operator_value.split(operator) # parse pattern parsed_pattern = jsonpath_rw_ext.parse("$." + pattern.strip()) return parsed_pattern, operator, comparison_value.strip() else: raise WardenFilterRuleFormatError( "Rule uses unsupported operator! Supported opperators " "are {}!".format(", ".join(cls.SUPPORTED_OPERATORS)))
def generate_instance_types_json(): r = requests.get(URL, stream=True) r.encoding = 'utf-8' costs = json.load(codecs.getreader("utf-8")(r.raw)) products = [ p.value for p in parse('products.*').find(costs)] # Only interested in Linux linux_products = [ p.value for p in parse('@[?(attributes.operatingSystem == "Linux")]').find(products)] instance_types_json = {} for p in linux_products: attributes = p['attributes'] # For now exclude dedicated and host tenancy if attributes['tenancy'] != 'Shared': continue if attributes['location'] not in REGIONS_MAP: continue location = instance_types_json.setdefault(REGIONS_MAP[attributes['location']], {}) instance = { 'id': attributes['instanceType'], 'cpu': attributes['vcpu'], 'memory': attributes['memory'], 'storage': attributes['storage'], 'family': attributes['instanceFamily'] } # N.B. Only do this partial in jsonpath otherwise we get errors :-( # The following path doesn't work for some reason. # 'terms.OnDemand.%s.*.priceDimensions.*.pricePerUnit.USD' % p['sku'] on_demand = costs['terms']['OnDemand'] path = '*.priceDimensions.*.pricePerUnit.USD' instance['price'] = parse(path).find(on_demand[p['sku']])[0].value gpu = 0 if gpu in attributes: gpu = attributes['gpu'] instance['gpu'] = gpu family = location.setdefault(attributes['instanceFamily'], []) family.append(instance) class sort_key(object): def __init__(self, obj, *args): self.obj = obj def _cmp(self, other): if int(self.obj['cpu']) < int(other['cpu']): return -1 elif int(self.obj['cpu']) > int(other['cpu']): return 1 if self.obj['memory'] < other['memory']: return -1 elif self.obj['memory'] > other['memory']: return 1 if int(self.obj['gpu']) < int(other['gpu']): return -1 elif int(self.obj['gpu']) > int(other['gpu']): return 1 return 0 def __lt__(self, other): return self._cmp(other.obj) < 0 def __gt__(self, other): return self._cmp(other.obj) > 0 def __eq__(self, other): return self._cmp(other.obj) == 0 def __le__(self, other): return self._cmp(other.obj) <= 0 def __ge__(self, other): return self._cmp(other.obj) >= 0 def __ne__(self, other): return self._cmp(other.obj) != 0 # Sort the families based on number of cpus and memory for region in instance_types_json: for family in instance_types_json[region]: instance_types_json[region][family].sort(key=sort_key) print (json.dumps(instance_types_json))
def jp(raw_json, jsonpath): jsonpath = parse(jsonpath) return [i.value for i in jsonpath.find(raw_json)]
def test_string_fields_config(self): cfg = dict(fields='payload.test') t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr) self.assertPathsEqual(t.getter.__self__, jsonpath_rw_ext.parse('payload.test'))
def test_list_fields_config(self): cfg = dict(fields=['payload.test', 'payload.other']) t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr) self.assertPathsEqual( t.getter.__self__, jsonpath_rw_ext.parse('(payload.test)|(payload.other)'))
def test_list_fields_config(self): cfg = dict(fields=["payload.test", "payload.other"]) t = converter.TraitDefinition("test_trait", cfg, self.fake_plugin_mgr) self.assertPathsEqual(t.getter.__self__, jsonpath_rw_ext.parse("(payload.test)|(payload.other)"))
import json import codecs from jsonpath_rw_ext import parse import compiler from types import ModuleType __name__ = "ConstraintConsistency" name = __name__ clause_jsonpath = parse("filters[*]|clauses[*]") def load_json_file(file_name): rules = json.load(codecs.open(file_name, 'r', 'utf-8')) return rules class IdentityConstraintConsistency(object): name = "IdentityConstraintConsistency" component_type = __name__ def preprocess(self, query): return query class ConstraintTypeTransformations(object): name = "ConstraintTypeTransformations" component_type = __name__
def jsonpath(expr, objs): return jsonpath_rw_ext.parse(expr).find(objs)