def __init__(self): self.pre_path = os.path.join(swagger_dir, BUILD_CONFIG['swagger']['pre']) self.post_path = os.path.join(os.path.dirname(static.__file__), BUILD_CONFIG['swagger']['post']) with open(self.pre_path, 'r', encoding='utf8') as f: self.data = json.load(f) # define pointers self.vb_pointer = BUILD_CONFIG['swagger']['pointers']['vb'] self.pn_pointer = BUILD_CONFIG['swagger']['pointers']['pn'] self.aff_pointer = BUILD_CONFIG['swagger']['pointers']['aff'] self.affopt_pointer = BUILD_CONFIG['swagger']['pointers']['ao'] # define tags self.vb_tags = sorted(list(set([slugify(x['tag']) for x in verb_data]))) self.pn_tags = sorted( list(set([slugify(x['tag']) for x in pronoun_data]))) self.aff_tags = sorted( list(set([slugify(x['tag']) for x in affix_data]))) aff_options = AFFIX_OPTIONS['AFFIX_OPTIONS'] self.affopt_tags = sorted( list(set([slugify(x['tag']) for x in aff_options]))) # set urls set_pointer(self.data, '/servers', BUILD_CONFIG['swagger']['urls'])
async def recv_and_process(): sock = ctx.socket(zmq.REP) sock.bind(url) with open("/home/slovak/awesomeconfig/config/conf_test.yaml") as iff: jobj = yaml.safe_load(iff) while True: msg = await sock.recv_multipart() responce = None try: request = msp.unpackb(msg[0]) pprint(request) if request["cmd"] == "get": responce = resolve_pointer(jobj, request["key"]) elif request["cmd"] == "set": set_pointer(jobj, request["key"], request["value"]) responce = True except: pass print(responce) await sock.send(msp.packb(responce))
def set_light(self, *args, **kwargs): results = self._bridge.set_light(*args, **kwargs)[0] state = copy.deepcopy(self._state) for result in results: if 'success' in result: k, v = result['success'].popitem() jsonpointer.set_pointer(state, k, v) self.set_state(state)
def replace_suggestions_urls(data, replacement_url_root) -> int: replaced_pointer_count = 0 for match in find_suggestion_urls(data): pointer = json_path_to_pointer(f"{match.full_path}.url") replacement_url = match.value['url'].format( suggestions_url_root=replacement_url_root) set_pointer(data, pointer, replacement_url) replaced_pointer_count += 1 return replaced_pointer_count
def remove_suggestions_urls(data) -> int: removed_pointer_count = 0 for match in find_suggestion_urls(data): parent_pointer = json_path_to_pointer(str(match.context.full_path)) parent_object = match.context.value del parent_object["suggestions"] set_pointer(data, parent_pointer, parent_object) removed_pointer_count += 1 return removed_pointer_count
def redact_words(self, refs: List[str]): def word_path(ref: str) -> str: # Remove leading '#'. return ref[1:] for ref in refs: r = word_path(ref) word = resolve_pointer(self.ocr_result, r) word["text"] = first_char(word["text"]) set_pointer(self.ocr_result, r, word)
def apply_pointers(self, content, params): """ Let's apply all the json pointers! Valid params in Nulecule: param1: - /spec/containers/0/ports/0/hostPort - /spec/containers/0/ports/0/hostPort2 or param1: - /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2 Args: content (str): content of artifact file params (dict): list of params with pointers to replace in content Returns: str: content with replaced pointers Todo: In the future we need to change this to detect haml, yaml, etc as we add more providers Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393 """ obj = anymarkup.parse(content) if type(obj) != dict: logger.debug( "Artifact file not json/haml, assuming it's $VARIABLE substitution" ) return content if params is None: # Nothing to do here! return content for name, pointers in params.items(): if not pointers: logger.warning("Could not find pointer for %s" % name) continue for pointer in pointers: try: resolve_pointer(obj, pointer) set_pointer(obj, pointer, name) logger.debug("Replaced %s pointer with %s param" % (pointer, name)) except JsonPointerException: logger.debug("Error replacing %s with %s" % (pointer, name)) logger.debug("Artifact content: %s", obj) raise NuleculeException( "Error replacing pointer %s with %s." % (pointer, name)) return anymarkup.serialize(obj, format="json")
def test_alttypes(self): JsonPointer.alttypes = True class Node(object): def __init__(self, name, parent=None): self.name = name self.parent = parent self.left = None self.right = None def set_left(self, node): node.parent = self self.left = node def set_right(self, node): node.parent = self self.right = node def __getitem__(self, key): if key == 'left': return self.left if key == 'right': return self.right raise KeyError("Only left and right supported") def __setitem__(self, key, val): if key == 'left': return self.set_left(val) if key == 'right': return self.set_right(val) raise KeyError("Only left and right supported: %s" % key) root = Node('root') root.set_left(Node('a')) root.left.set_left(Node('aa')) root.left.set_right(Node('ab')) root.set_right(Node('b')) root.right.set_left(Node('ba')) root.right.set_right(Node('bb')) self.assertEqual(resolve_pointer(root, '/left').name, 'a') self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab') self.assertEqual(resolve_pointer(root, '/right').name, 'b') self.assertEqual(resolve_pointer(root, '/right/left').name, 'ba') newroot = set_pointer(root, '/left/right', Node('AB'), inplace=False) self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab') self.assertEqual(resolve_pointer(newroot, '/left/right').name, 'AB') set_pointer(root, '/left/right', Node('AB')) self.assertEqual(resolve_pointer(root, '/left/right').name, 'AB')
def sub( obj: Any, context: Dict[str, Any] = None, values: Dict[str, Any] = None, g: Union[Dict[str, Any], Type[Any]] = globals.ExpressionGlobals, ) -> Any: context = context or {} values = values or {} for pointer, value in values.items(): jsonpointer.set_pointer(context, "/" + pointer, value) return _sub_i(obj, context, _globals(g))
def data(self, value): """ Modify the data associated for this resource. Note that while a root DataRep can be set without triggering a pull, setting a fragment requires accessing the fragment's root.data. """ if self.fragment: # Access .root.data rather than ._data to ensure that # we have pulled it at least once. set_pointer(self.root.data, self.fragment, value) else: self._data = value
def render(self, dict_to_render, list_item_id): """ Transform the current schema json to a fully rendered dictionary """ rendered_data = deepcopy(dict_to_render) pointers = find_pointers_containing(rendered_data, "placeholders") for pointer in pointers: rendered_text = self.render_pointer(rendered_data, pointer, list_item_id) set_pointer(rendered_data, pointer, rendered_text) return rendered_data
def update_file_key(self, key): """Update file key if condition is valid.""" if self.condition: jsonpointer.set_pointer(self.instance, self.validator_value['file_key'], key) else: data, part = jsonpointer.JsonPointer( self.validator_value['file_key']).to_last(self.instance) if data: if part is not None: del data[part] else: del data
def writeNewData(self): set_pointer(self.data, self.vb_pointer, self.vb_tags) set_pointer(self.data, self.pn_pointer, self.pn_tags) set_pointer(self.data, self.aff_pointer, self.aff_tags) set_pointer(self.data, self.affopt_pointer, self.affopt_tags) with open(self.post_path, 'w') as f: json.dump(self.data, f)
def render(self, dict_to_render, list_item_id): """ Transform the current schema json to a fully rendered dictionary """ dict_to_render = QuestionnaireSchema.get_mutable_deepcopy( dict_to_render) pointers = find_pointers_containing(dict_to_render, "placeholders") for pointer in pointers: rendered_text = self.render_pointer(dict_to_render, pointer, list_item_id) set_pointer(dict_to_render, pointer, rendered_text) return dict_to_render
def apply_pointers(self, content, params): """ Let's apply all the json pointers! Valid params in Nulecule: param1: - /spec/containers/0/ports/0/hostPort - /spec/containers/0/ports/0/hostPort2 or param1: - /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2 Args: content (str): content of artifact file params (dict): list of params with pointers to replace in content Returns: str: content with replaced pointers Todo: In the future we need to change this to detect haml, yaml, etc as we add more providers Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393 """ obj = anymarkup.parse(content) if type(obj) != dict: logger.debug("Artifact file not json/haml, assuming it's $VARIABLE substitution") return content if params is None: # Nothing to do here! return content for name, pointers in params.items(): if not pointers: logger.warning("Could not find pointer for %s" % name) continue for pointer in pointers: try: resolve_pointer(obj, pointer) set_pointer(obj, pointer, name) logger.debug("Replaced %s pointer with %s param" % (pointer, name)) except JsonPointerException: logger.debug("Error replacing %s with %s" % (pointer, name)) logger.debug("Artifact content: %s", obj) raise NuleculeException("Error replacing pointer %s with %s." % (pointer, name)) return anymarkup.serialize(obj, format="json")
def main(): module = AnsibleModule( argument_spec=dict( data=dict(required=True, type='dict'), set_data=dict(type='json'), ), supports_check_mode=True, ) if jsonpointer is None: module.fail_json(msg='jsonpointer module is not available') data = module.params['data'] j_data = json.loads(module.params['set_data']) for j in j_data: pointer = "/{}".format(j) s_data = j_data[j] msg = ("pointer: {} set_data: {}".format(pointer, s_data)) try: data = jsonpointer.set_pointer(data, pointer, s_data) except jsonpointer.JsonPointerException as err: module.fail_json(msg=str(err)) module.exit_json(changed=True, result=data)
def redact_lines(self, refs: List[str]): def line_path(ref: str) -> str: end = ref.find("/word") # Remove leading '#' and trailing word path. return ref[1:end] for ref in refs: r = line_path(ref) line = resolve_pointer(self.ocr_result, r) tokens = line["text"].split(' ') word_id = int(ref.split('/')[-1]) tokens[word_id] = first_char(tokens[word_id]) line["text"] = ' '.join(tokens) set_pointer(self.ocr_result, r, line)
def _json_set(query: str, value: str, input_filename: str = '-', output_filename: str = '-') -> None: doc = _load_json(input_filename) value = json.loads(value) result = set_pointer(doc, query, value) _save_json(result, output_filename)
def __set__(self, obj, val): # TODO: Abstract JsonPointerException when settings a member with missing nested parent dict # We need to deepcopy data, so that SQLAlchemy can detect changes. Otherwise nested changes would mutate the dict in-place and SQLAlchemy cannot perform comparison. data = copy.deepcopy(self.ensure_valid_data(obj)) val = self.converter.serialize(val) if val is not None: # Do some basic data validation, don't let first class objects slip through if type(val) not in (str, float, bool, int, dict): raise BadJSONData("Cannot update field at {} as it has unsupported type {} for JSONB data".format(self.pointer, type(val))) jsonpointer.set_pointer(data, self.pointer, val) set_attribute(obj, self.data_field, data)
def test_command_release_field_is_null(pointer, monkeypatch): data = json.loads(read('release_minimal.json')) parts = pointer.split('/') for i, part in enumerate(parts, 1): if i < len(parts): if parts[i] == '0': value = [{}] else: value = {} else: value = None set_pointer(data, '/' + '/'.join(parts[:i]), value) stdin = json.dumps(data).encode('utf-8') # Should not raise an error. run_streaming(monkeypatch, main, ['upgrade', '1.0:1.1'], stdin)
def filter_values(data, keys): data = copy.deepcopy(data) for key in keys: try: key_data = jsonpointer.resolve_pointer(data, key) except jsonpointer.JsonPointerException: # Data does not exist anyway, nothing to do! continue key_status = key_data.get("status", "") if isinstance(key_data, dict) else "" if isinstance(key_status, str) and key_status.strip().lower() == "public": pass else: jsonpointer.set_pointer(data, key, None) return data
def graft(model, pointer, branch): '''Add branch to model at pointer. Note, pointer must have already support in model. Ie, there is no 'mkdir'. ''' if not pointer: return branch return jsonpointer.set_pointer(model, pointer, branch)
def write_swagger(dictionaries: List[Dictionary]): static_dir = os.path.dirname(static.__file__) pre_path = os.path.join(os.path.dirname(swagger_dir.__file__), 'swagger-pre.json') post_path = os.path.join(static_dir, 'swagger.json') with open(pre_path, 'r', encoding='utf8') as f: data = json.load(f) languages_pointer = '/components/schemas/LanguageTags/enum' language_tags = sorted(list(set([d.name for d in dictionaries]))) new_data = set_pointer(data, languages_pointer, language_tags) with open(post_path, 'w', encoding='utf8') as f: json.dump(new_data, f)
def set_json_value(self, json_string, json_pointer, json_value): """ Set the target node of the JSON document `json_string` specified by JSON Pointer `json_pointer` to `json_value`. Example: | ${result}= | Set Json Value | {"foo": {"bar": [1,2,3]}} | /foo | 12 | | Should Be Equal | ${result} | {"foo": 12} | | | """ value = load_json(json_value) p = jsonpointer.set_pointer(json_string, json_pointer, value) return json_string
def test_alttypes(self): Node = self.Node root = Node('root') root.set_left(Node('a')) root.left.set_left(Node('aa')) root.left.set_right(Node('ab')) root.set_right(Node('b')) root.right.set_left(Node('ba')) root.right.set_right(Node('bb')) self.assertEqual(resolve_pointer(root, '/left').name, 'a') self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab') self.assertEqual(resolve_pointer(root, '/right').name, 'b') self.assertEqual(resolve_pointer(root, '/right/left').name, 'ba') newroot = set_pointer(root, '/left/right', Node('AB'), inplace=False) self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab') self.assertEqual(resolve_pointer(newroot, '/left/right').name, 'AB') set_pointer(root, '/left/right', Node('AB')) self.assertEqual(resolve_pointer(root, '/left/right').name, 'AB')
def test_datarep_data_setter_fragment(mock_service): root_data = copy.deepcopy(ANY_DATA) root = datarep.DataRep(mock_service, ANY_URI, ANY_DATA_SCHEMA, data=root_data) fragment = datarep.DataRep(root=root, fragment=ANY_FRAGMENT_PTR) modified_data = set_pointer(ANY_DATA, ANY_FRAGMENT_PTR, 42, inplace=False) root.pull = mock.Mock() fragment.pull = mock.Mock() fragment.data = 42 assert fragment._data is datarep.DataRep.FRAGMENT assert fragment.data == 42 assert root.data == modified_data assert not root.pull.called assert not fragment.pull.called
def main(): module = AnsibleModule( argument_spec=dict( data=dict(required=True, type='dict'), pointer=dict(required=True), action=dict(required=True, choices=['append', 'extend', 'update']), update=dict(type='str'), extend=dict(type='list'), append=dict(), ), supports_check_mode=True, ) if jsonpointer is None: module.fail_json(msg='jsonpointer module is not available') action = module.params['action'] data = module.params['data'] pointer = module.params['pointer'] if isinstance(data, str): data = json.loads(str) try: res = jsonpointer.resolve_pointer(data, pointer) except jsonpointer.JsonPointerException as err: module.fail_json(msg=str(err)) if action == 'append': res.append(module.params['append']) if action == 'extend': res.extend(module.params['extend']) elif action == 'update': res = jsonpointer.set_pointer(data, pointer, module.params['update']) # res.update(module.params['update']) module.exit_json(changed=True, result=data)
def uri(self, value): """Set new uri value in record. It will not change the location of the underlying file! """ jsonpointer.set_pointer(self.record, self.pointer, value)
def _patch(obj, jsonptr, value): """Returns patched object.""" return jsonpointer.set_pointer(obj, jsonptr, value, inplace=False)
def setin(self, path, value): """ Set a value using JSON-pointer syntax """ set_pointer(self, path, value)
def parse_schema(self, resp): """Function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str. """ #pylint: disable=maybe-no-member jsonpath_expr = jsonpath_rw.parse('$.."$ref"') matches = jsonpath_expr.find(resp.dict) respcopy = resp.dict typeregex = '([#,@].*?\.)' if matches: for match in matches: fullpath = str(match.full_path) jsonfile = match.value.split('#')[0] jsonpath = match.value.split('#')[1] listmatch = None found = None if 'redfish.dmtf.org' in jsonfile: if 'odata' in jsonfile: jsonpath = jsonpath.replace(jsonpath.split('/')[-1], \ 'odata' + jsonpath.split('/')[-1]) jsonfile = 'Resource.json' found = re.search(typeregex, fullpath) if found: repitem = fullpath[found.regs[0][0]:found.regs[0][1]] schemapath = '/' + fullpath.replace(repitem, '~').\ replace('.', '/').replace('~', repitem) else: schemapath = '/' + fullpath.replace('.', '/') if '.json' in jsonfile: itempath = schemapath if self.is_redfish: if resp.request.path[-1] == '/': newpath = '/'.join(resp.request.path.split('/')\ [:-2]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')\ [:-1]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')[:-1]) \ + '/' + jsonfile if 'href.json' in newpath: continue if not newpath.lower() in self._visited_urls: self.load(newpath, skipcrawl=True, includelogs=False, \ skipinit=True, loadtype='ref') instance = list() #deprecated type "string" for Type.json if 'string' in self.types: for item in self.itertype('string'): instance.append(item) if 'object' in self.types: for item in self.itertype('object'): instance.append(item) for item in instance: if jsonfile in item.resp._rest_request._path: if 'anyOf' in fullpath: break dictcopy = item.resp.dict listmatch = re.search('[[][0-9]+[]]', itempath) if listmatch: start = listmatch.regs[0][0] end = listmatch.regs[0][1] newitempath = [ itempath[:start], itempath[end:] ] start = jsonpointer.JsonPointer(newitempath[0]) end = jsonpointer.JsonPointer(newitempath[1]) del start.parts[-1], end.parts[-1] vals = start.resolve(respcopy) count = 0 for val in vals: try: if '$ref' in six.iterkeys( end.resolve(val)): end.resolve(val).pop('$ref') end.resolve(val).update(dictcopy) replace_pointer = jsonpointer.\ JsonPointer(end.path + jsonpath) data = replace_pointer.resolve(val) set_pointer(val, end.path, data) start.resolve(respcopy)[count].\ update(val) break except: count += 1 else: itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in six.iterkeys( itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(dictcopy) break if jsonpath: if 'anyOf' in fullpath: continue if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = schemapath.replace('/$ref', '') if re.search('\[\d]', schemapath): schemapath = schemapath.translate(None, '[]') schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: if not listmatch: schemapath = schemapath.replace('/$ref', '') replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.json(respcopy) else: resp.json(respcopy)
def _set_value(self, pointer, value): set_pointer(self.resolver, pointer, value)
def parse_schema(self, resp): """Function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str. """ jsonpath_expr = jsonpath_rw.parse(u'$.."$ref"') matches = jsonpath_expr.find(resp.dict) respcopy = resp.dict if matches: for match in matches: fullpath = str(match.full_path) jsonfile = match.value.split('#')[0] jsonpath = match.value.split('#')[1] if '@odata' in fullpath: schemapath = '/' + fullpath.replace('@odata.', '~').\ replace('.', '/').replace('~', '@odata.') else: schemapath = '/' + fullpath.replace('.', '/') if '.json' in jsonfile: itempath = schemapath if self.is_redfish: if resp.request.path[-1] == '/': newpath = '/'.join(resp.request.path.split('/')\ [:-2]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')\ [:-1]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')[:-1]) \ + '/' + jsonfile if 'href.json' in newpath: continue if not newpath.lower() in self._visited_urls: self.load(newpath, skipcrawl=True, includelogs=False, \ skipinit=True, loadtype='ref') instance = list() if u'st' in self.types: for x in self.types[u'st'][u'Instances']: instance.append(x) if u'ob' in self.types: for x in self.types[u'ob'][u'Instances']: instance.append(x) for item in instance: if jsonfile in item.resp._rest_request._path: if 'anyOf' in fullpath: break dictcopy = item.resp.dict listmatch = re.search('[[][0-9]+[]]', itempath) if listmatch: start = listmatch.regs[0][0] end = listmatch.regs[0][1] newitempath = [itempath[:start],\ itempath[end:]] start = jsonpointer.JsonPointer(newitempath[0]) end = jsonpointer.JsonPointer(newitempath[1]) del start.parts[-1], end.parts[-1] vals = start.resolve(respcopy) count = 0 for val in vals: try: if '$ref' in end.resolve( val).iterkeys(): end.resolve(val).pop('$ref') end.resolve(val).update(dictcopy) replace_pointer = jsonpointer.\ JsonPointer(end.path + jsonpath) data = replace_pointer.resolve(val) set_pointer(val, end.path, data) start.resolve(respcopy)[count].\ update(val) break except: count += 1 else: itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in itempath.resolve(respcopy).\ iterkeys(): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(dictcopy) if jsonpath: if 'anyOf' in fullpath: continue if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = schemapath.replace('/$ref', '') schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: if not listmatch: schemapath = schemapath.replace('/$ref', '') replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.json(respcopy) else: resp.json(respcopy)
def ops_published_data_crawl(constituents, query, chunksize): if constituents != 'pub-number': raise ValueError('constituents "{0}" invalid or not implemented yet'.format(constituents)) real_constituents = constituents if constituents == 'pub-number': constituents = '' # fetch first chunk (1-chunksize) from upstream first_chunk = ops_published_data_search(constituents, query, '1-{0}'.format(chunksize)) #print first_chunk pointer_total_count = JsonPointer('/ops:world-patent-data/ops:biblio-search/@total-result-count') total_count = int(pointer_total_count.resolve(first_chunk)) log.info('ops_published_data_crawl total_count: %s', total_count) # The first 2000 hits are accessible from OPS. total_count = min(total_count, 2000) # collect upstream results begin_second_chunk = chunksize + 1 chunks = [first_chunk] for range_begin in range(begin_second_chunk, total_count + 1, chunksize): # countermeasure to robot flagging # <code>CLIENT.RobotDetected</code> # <message>Recent behaviour implies you are a robot. The server is at the moment busy to serve robots. Please try again later</message> time.sleep(5) range_end = range_begin + chunksize - 1 range_string = '{0}-{1}'.format(range_begin, range_end) log.info('ops_published_data_crawl range: ' + range_string) chunk = ops_published_data_search(constituents, query, range_string) #print 'chunk:', chunk chunks.append(chunk) #return chunks # merge chunks into single result """ <empty>: "ops:search-result" { » "ops:publication-reference": [ biblio: "ops:search-result" { » "exchange-documents": [ » "exchange-document": { abstract: "ops:search-result" { » "exchange-documents": [ » "exchange-document": { full-cycle: "ops:search-result" { » "exchange-documents": [ » "exchange-document": [ pub-number: "ops:search-result" { » "ops:publication-reference": [ { "@family-id": "6321653", "@system": "ops.epo.org", "document-id": { "@document-id-type": "docdb", "country": { "$": "DE" }, "doc-number": { "$": "3705908" }, "kind": { "$": "A1" } } }, """ pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/ops:publication-reference') #pointer_time_elapsed = JsonPointer('/ops:world-patent-data/ops:meta/@value') all_results = [] #time_elapsed = int(pointer_time_elapsed.resolve(first_chunk)) for chunk in chunks: # FIXME: use this for "real_constituents == 'pub-number'" only chunk_results = to_list(pointer_results.resolve(chunk)) # FIXME: implement other constituents #print 'chunk_results:', chunk_results all_results += chunk_results #time_elapsed += int(pointer_time_elapsed.resolve(chunk)) response = None if real_constituents == 'pub-number': response = first_chunk # delete upstream data del resolve_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:search-result')['ops:publication-reference'] # compute own representation publication_numbers = [] pointer_document_id = JsonPointer('/document-id') for entry in all_results: pubref = pointer_document_id.resolve(entry) #print entry, pubref pubref_number, pubref_date = _get_document_number_date(pubref, 'docdb') publication_numbers.append(pubref_number) # add own representation set_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:search-result/publication-numbers', publication_numbers, inplace=True) # amend metadata new_total_count = str(len(publication_numbers)) pointer_total_count.set(response, new_total_count) set_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:range', {'@begin': '1', '@end': new_total_count}) #pointer_time_elapsed.set(response, str(time_elapsed)) if not response: raise ValueError('constituents "{0}" invalid or not implemented yet'.format(constituents)) return response
def parse_schema_gen(self, resp): """Redfish general function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str. """ #pylint: disable=maybe-no-member getval = lambda inmat: getval(inmat.left) + '/' + str(inmat.right) \ if hasattr(inmat, 'left') else str(inmat) respcopy = resp.dict jsonpath_expr = jsonpath_rw.parse('$.."anyOf"') while True: matches = jsonpath_expr.find(respcopy) if not matches: break match = matches[0] newval = None schlist = match.value schlist = [ele for ele in list(schlist) if ele != {"type": "null"}] norefsch = [ele for ele in list(schlist) if isinstance(ele, dict) and \ len(ele.keys()) > 1] if norefsch: newval = norefsch[0] else: newsc = [ ele for ele in list(schlist) if not ele["$ref"].split('#')[0] ] newval = newsc[0] if newsc else None if not newval: schlist = [ele["$ref"] for ele in list(schlist) if "$ref" in ele.keys() and \ (ele["$ref"].split('#')[0].endswith('.json') and 'odata' not in \ ele["$ref"].split('#')[0])] maxsch = max(schlist) newval = {"$ref": maxsch} itempath = '/' + getval(match.full_path) if re.search('\[\d+]', itempath): itempath = itempath.translate(None, '[]') itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if 'anyOf' in six.iterkeys(itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('anyOf') itempath.resolve(respcopy).update(newval) jsonpath_expr = jsonpath_rw.parse('$.."$ref"') matches = jsonpath_expr.find(respcopy) if matches: for _, match in enumerate(matches): jsonfile = match.value.split('#')[0] jsonfile = '' if jsonfile.lower() == resp.request.path.lower( ) else jsonfile jsonpath = match.value.split('#')[1] schemapath = '/' + getval(match.full_path) if jsonfile: itempath = schemapath if '/' not in jsonfile: inds = -2 if resp.request.path[-1] == '/' else -1 jsonfile = '/'.join(resp.request.path.split('/')[:inds]) \ + '/' + jsonfile + '/' if jsonfile not in self.paths: self.load(jsonfile, crawl=False, includelogs=False, \ init=False, loadtype='ref') item = self.paths[ jsonfile] if jsonfile in self.paths else None # if not item: # if not 'anyOf' in schemapath: # raise "We got a situation :|" # continue if re.search('\[\d+]', itempath): itempath = itempath.translate(None, '[]') itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in six.iterkeys(itempath.resolve(respcopy)): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(item.dict) if jsonpath: schemapath = schemapath.replace('/$ref', '') if re.search('\[\d+]', schemapath): schemapath = schemapath.translate(None, '[]') if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.loaddict(respcopy) else: resp.loaddict(respcopy)
def test_set(self): doc = { "foo": ["bar", "baz"], "": 0, "a/b": 1, "c%d": 2, "e^f": 3, "g|h": 4, "i\\j": 5, "k\"l": 6, " ": 7, "m~n": 8 } origdoc = copy.deepcopy(doc) # inplace=False newdoc = set_pointer(doc, "/foo/1", "cod", inplace=False) self.assertEqual(resolve_pointer(newdoc, "/foo/1"), "cod") newdoc = set_pointer(doc, "/", 9, inplace=False) self.assertEqual(resolve_pointer(newdoc, "/"), 9) newdoc = set_pointer(doc, "/fud", {}, inplace=False) newdoc = set_pointer(newdoc, "/fud/gaw", [1, 2, 3], inplace=False) self.assertEqual(resolve_pointer(newdoc, "/fud"), {'gaw' : [1, 2, 3]}) newdoc = set_pointer(doc, "", 9, inplace=False) self.assertEqual(newdoc, 9) self.assertEqual(doc, origdoc) # inplace=True set_pointer(doc, "/foo/1", "cod") self.assertEqual(resolve_pointer(doc, "/foo/1"), "cod") set_pointer(doc, "/", 9) self.assertEqual(resolve_pointer(doc, "/"), 9) self.assertRaises(JsonPointerException, set_pointer, doc, "/fud/gaw", 9) set_pointer(doc, "/fud", {}) set_pointer(doc, "/fud/gaw", [1, 2, 3] ) self.assertEqual(resolve_pointer(doc, "/fud"), {'gaw' : [1, 2, 3]}) self.assertRaises(JsonPointerException, set_pointer, doc, "", 9)
def parse_schema(self, resp): """Function to get and replace schema $ref with data :param resp: response data containing ref items. :type resp: str. """ jsonpath_expr = jsonpath_rw.parse(u'$.."$ref"') matches = jsonpath_expr.find(resp.dict) respcopy = resp.dict if matches: for match in matches: fullpath = str(match.full_path) jsonfile = match.value.split('#')[0] jsonpath = match.value.split('#')[1] if '@odata' in fullpath: schemapath = '/' + fullpath.replace('@odata.', '~').\ replace('.', '/').replace('~', '@odata.') else: schemapath = '/' + fullpath.replace('.', '/') if '.json' in jsonfile: itempath = schemapath if self.is_redfish: if resp.request.path[-1] == '/': newpath = '/'.join(resp.request.path.split('/')\ [:-2]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')\ [:-1]) + '/' + jsonfile + '/' else: newpath = '/'.join(resp.request.path.split('/')[:-1]) \ + '/' + jsonfile if 'href.json' in newpath: continue if not newpath.lower() in self._visited_urls: self.load(newpath, skipcrawl=True, includelogs=False, \ skipinit=True, loadtype='ref') instance = list() if u'st' in self.types: for stitem in self.types[u'st'][u'Instances']: instance.append(stitem) if u'ob' in self.types: for obitem in self.types[u'ob'][u'Instances']: instance.append(obitem) for item in instance: if jsonfile in item.resp._rest_request._path: if 'anyOf' in fullpath: break dictcopy = item.resp.dict listmatch = re.search('[[][0-9]+[]]', itempath) if listmatch: start = listmatch.regs[0][0] end = listmatch.regs[0][1] newitempath = [itempath[:start], itempath[end:]] start = jsonpointer.JsonPointer(newitempath[0]) end = jsonpointer.JsonPointer(newitempath[1]) del start.parts[-1], end.parts[-1] vals = start.resolve(respcopy) count = 0 for val in vals: try: if '$ref' in end.resolve(val).iterkeys(): end.resolve(val).pop('$ref') end.resolve(val).update(dictcopy) replace_pointer = jsonpointer.\ JsonPointer(end.path + jsonpath) data = replace_pointer.resolve(val) set_pointer(val, end.path, data) start.resolve(respcopy)[count].\ update(val) break except: count += 1 else: itempath = jsonpointer.JsonPointer(itempath) del itempath.parts[-1] if '$ref' in itempath.resolve(respcopy).\ iterkeys(): itempath.resolve(respcopy).pop('$ref') itempath.resolve(respcopy).update(dictcopy) if jsonpath: if 'anyOf' in fullpath: continue if not jsonfile: replacepath = jsonpointer.JsonPointer(jsonpath) schemapath = schemapath.replace('/$ref', '') schemapath = jsonpointer.JsonPointer(schemapath) data = replacepath.resolve(respcopy) if '$ref' in schemapath.resolve(respcopy): schemapath.resolve(respcopy).pop('$ref') schemapath.resolve(respcopy).update(data) else: if not listmatch: schemapath = schemapath.replace('/$ref', '') replacepath = schemapath + jsonpath replace_pointer = jsonpointer.\ JsonPointer(replacepath) data = replace_pointer.resolve(respcopy) set_pointer(respcopy, schemapath, data) resp.json(respcopy) else: resp.json(respcopy)
def update(self, pointer: str, value): if not pointer.startswith("/"): pointer = "/" + pointer jsonpointer.set_pointer(self.data, pointer, value, inplace=True)
def modify_json(json_data, pointer, json_value, state='present', action='add'): is_root = False # special treatment - we cannot modify reference in place key_exists = False try: value = json.loads(json_value) except Exception: value = None if state == 'present': if action not in ['add', 'update', 'replace']: raise ValueError elif state == 'absent': pass else: raise ValueError # we store the original json document to compare it later original_json_data = copy.deepcopy(json_data) try: target = jsonpointer.resolve_pointer(json_data, pointer) if pointer == '': is_root = True key_exists = True except jsonpointer.JsonPointerException: key_exists = False if key_exists: if state == "present": if action == "add": if isinstance(target, dict) and isinstance(value, dict): # we keep old values and only append new ones value.update(target) result = jsonpointer.set_pointer(json_data, pointer, value, inplace=(not is_root)) if is_root: json_data = result elif isinstance(target, list) and isinstance(value, list): # we just append new items to the list for item in value: if item not in target: target.append(item) elif ((not isinstance(target, dict)) and (not isinstance(target, list))): # 'add' does not overwrite pass else: raise ValueError elif action == "update": if isinstance(target, dict) and isinstance(value, dict): # we append new values and overwrite the old ones target.update(value) elif isinstance(target, list) and isinstance(value, list): # we just append new items to the list - same as with 'add' for item in value: if item not in target: target.append(item) elif ((not isinstance(target, dict)) and (not isinstance(target, list))): # 'update' DOES overwrite if value is not None: result = jsonpointer.set_pointer( json_data, pointer, value) elif target != json_value: result = jsonpointer.set_pointer( json_data, pointer, json_value) else: raise ValueError else: raise ValueError elif action == "replace": # simple case when we don't care what was there before (almost) if value is not None: result = jsonpointer.set_pointer(json_data, pointer, value, inplace=(not is_root)) else: result = jsonpointer.set_pointer(json_data, pointer, json_value, inplace=(not is_root)) if is_root: json_data = result else: raise ValueError elif state == "absent": # we will delete the elements in the object or object itself if is_root: if json_value == '': # we just return empty json json_data = {} elif isinstance(target, dict) and isinstance(value, dict): for key in value: target.pop(key, None) else: raise ValueError else: # we must take a step back in the pointer, so we can edit it ppointer = pointer.split('/') to_delete = ppointer.pop() ppointer = '/'.join(ppointer) ptarget = jsonpointer.resolve_pointer(json_data, ppointer) if (((not isinstance(target, dict)) and (not isinstance(target, list)) and json_value == '') or (isinstance(target, dict) or isinstance(target, list)) and json_value == ''): # we simply delete the key with it's value (whatever it is) ptarget.pop(to_delete, None) target = ptarget # piece of self-defense elif isinstance(target, dict) and isinstance(value, dict): for key in value: target.pop(key, None) elif isinstance(target, list) and isinstance(value, list): for item in value: try: target.remove(item) except ValueError: pass else: raise ValueError else: raise ValueError else: # the simplest case - nothing was there before and pointer is not root # because in that case we would have key_exists = true if state == 'present': if value is not None: result = jsonpointer.set_pointer(json_data, pointer, value) else: result = jsonpointer.set_pointer(json_data, pointer, json_value) if json_data != original_json_data: changed = True else: changed = False if changed: msg = "JSON object '%s' was updated" % pointer else: msg = "No change to JSON object '%s'" % pointer return json_data, changed, msg