def __init__(self):
self.pre_path = os.path.join(swagger_dir,
BUILD_CONFIG['swagger']['pre'])
self.post_path = os.path.join(os.path.dirname(static.__file__),
BUILD_CONFIG['swagger']['post'])
with open(self.pre_path, 'r', encoding='utf8') as f:
self.data = json.load(f)
# define pointers
self.vb_pointer = BUILD_CONFIG['swagger']['pointers']['vb']
self.pn_pointer = BUILD_CONFIG['swagger']['pointers']['pn']
self.aff_pointer = BUILD_CONFIG['swagger']['pointers']['aff']
self.affopt_pointer = BUILD_CONFIG['swagger']['pointers']['ao']
# define tags
self.vb_tags = sorted(list(set([slugify(x['tag'])
for x in verb_data])))
self.pn_tags = sorted(
list(set([slugify(x['tag']) for x in pronoun_data])))
self.aff_tags = sorted(
list(set([slugify(x['tag']) for x in affix_data])))
aff_options = AFFIX_OPTIONS['AFFIX_OPTIONS']
self.affopt_tags = sorted(
list(set([slugify(x['tag']) for x in aff_options])))
# set urls
set_pointer(self.data, '/servers', BUILD_CONFIG['swagger']['urls'])
async def recv_and_process():
sock = ctx.socket(zmq.REP)
sock.bind(url)
with open("/home/slovak/awesomeconfig/config/conf_test.yaml") as iff:
jobj = yaml.safe_load(iff)
while True:
msg = await sock.recv_multipart()
responce = None
try:
request = msp.unpackb(msg[0])
pprint(request)
if request["cmd"] == "get":
responce = resolve_pointer(jobj, request["key"])
elif request["cmd"] == "set":
set_pointer(jobj, request["key"], request["value"])
responce = True
except:
pass
print(responce)
await sock.send(msp.packb(responce))
def set_light(self, *args, **kwargs):
results = self._bridge.set_light(*args, **kwargs)[0]
state = copy.deepcopy(self._state)
for result in results:
if 'success' in result:
k, v = result['success'].popitem()
jsonpointer.set_pointer(state, k, v)
self.set_state(state)
def replace_suggestions_urls(data, replacement_url_root) -> int:
replaced_pointer_count = 0
for match in find_suggestion_urls(data):
pointer = json_path_to_pointer(f"{match.full_path}.url")
replacement_url = match.value['url'].format(
suggestions_url_root=replacement_url_root)
set_pointer(data, pointer, replacement_url)
replaced_pointer_count += 1
return replaced_pointer_count
def remove_suggestions_urls(data) -> int:
removed_pointer_count = 0
for match in find_suggestion_urls(data):
parent_pointer = json_path_to_pointer(str(match.context.full_path))
parent_object = match.context.value
del parent_object["suggestions"]
set_pointer(data, parent_pointer, parent_object)
removed_pointer_count += 1
return removed_pointer_count
def redact_words(self, refs: List[str]):
def word_path(ref: str) -> str:
# Remove leading '#'.
return ref[1:]
for ref in refs:
r = word_path(ref)
word = resolve_pointer(self.ocr_result, r)
word["text"] = first_char(word["text"])
set_pointer(self.ocr_result, r, word)
def apply_pointers(self, content, params):
"""
Let's apply all the json pointers!
Valid params in Nulecule:
param1:
- /spec/containers/0/ports/0/hostPort
- /spec/containers/0/ports/0/hostPort2
or
param1:
- /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2
Args:
content (str): content of artifact file
params (dict): list of params with pointers to replace in content
Returns:
str: content with replaced pointers
Todo:
In the future we need to change this to detect haml, yaml, etc as we add more providers
Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393
"""
obj = anymarkup.parse(content)
if type(obj) != dict:
logger.debug(
"Artifact file not json/haml, assuming it's $VARIABLE substitution"
)
return content
if params is None:
# Nothing to do here!
return content
for name, pointers in params.items():
if not pointers:
logger.warning("Could not find pointer for %s" % name)
continue
for pointer in pointers:
try:
resolve_pointer(obj, pointer)
set_pointer(obj, pointer, name)
logger.debug("Replaced %s pointer with %s param" %
(pointer, name))
except JsonPointerException:
logger.debug("Error replacing %s with %s" %
(pointer, name))
logger.debug("Artifact content: %s", obj)
raise NuleculeException(
"Error replacing pointer %s with %s." %
(pointer, name))
return anymarkup.serialize(obj, format="json")
def test_alttypes(self):
JsonPointer.alttypes = True
class Node(object):
def __init__(self, name, parent=None):
self.name = name
self.parent = parent
self.left = None
self.right = None
def set_left(self, node):
node.parent = self
self.left = node
def set_right(self, node):
node.parent = self
self.right = node
def __getitem__(self, key):
if key == 'left':
return self.left
if key == 'right':
return self.right
raise KeyError("Only left and right supported")
def __setitem__(self, key, val):
if key == 'left':
return self.set_left(val)
if key == 'right':
return self.set_right(val)
raise KeyError("Only left and right supported: %s" % key)
root = Node('root')
root.set_left(Node('a'))
root.left.set_left(Node('aa'))
root.left.set_right(Node('ab'))
root.set_right(Node('b'))
root.right.set_left(Node('ba'))
root.right.set_right(Node('bb'))
self.assertEqual(resolve_pointer(root, '/left').name, 'a')
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(root, '/right').name, 'b')
self.assertEqual(resolve_pointer(root, '/right/left').name, 'ba')
newroot = set_pointer(root, '/left/right', Node('AB'), inplace=False)
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(newroot, '/left/right').name, 'AB')
set_pointer(root, '/left/right', Node('AB'))
self.assertEqual(resolve_pointer(root, '/left/right').name, 'AB')
def test_alttypes(self):
JsonPointer.alttypes = True
class Node(object):
def __init__(self, name, parent=None):
self.name = name
self.parent = parent
self.left = None
self.right = None
def set_left(self, node):
node.parent = self
self.left = node
def set_right(self, node):
node.parent = self
self.right = node
def __getitem__(self, key):
if key == 'left':
return self.left
if key == 'right':
return self.right
raise KeyError("Only left and right supported")
def __setitem__(self, key, val):
if key == 'left':
return self.set_left(val)
if key == 'right':
return self.set_right(val)
raise KeyError("Only left and right supported: %s" % key)
root = Node('root')
root.set_left(Node('a'))
root.left.set_left(Node('aa'))
root.left.set_right(Node('ab'))
root.set_right(Node('b'))
root.right.set_left(Node('ba'))
root.right.set_right(Node('bb'))
self.assertEqual(resolve_pointer(root, '/left').name, 'a')
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(root, '/right').name, 'b')
self.assertEqual(resolve_pointer(root, '/right/left').name, 'ba')
newroot = set_pointer(root, '/left/right', Node('AB'), inplace=False)
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(newroot, '/left/right').name, 'AB')
set_pointer(root, '/left/right', Node('AB'))
self.assertEqual(resolve_pointer(root, '/left/right').name, 'AB')
def sub(
obj: Any,
context: Dict[str, Any] = None,
values: Dict[str, Any] = None,
g: Union[Dict[str, Any], Type[Any]] = globals.ExpressionGlobals,
) -> Any:
context = context or {}
values = values or {}
for pointer, value in values.items():
jsonpointer.set_pointer(context, "/" + pointer, value)
return _sub_i(obj, context, _globals(g))
def data(self, value):
""" Modify the data associated for this resource.
Note that while a root DataRep can be set without triggering a pull,
setting a fragment requires accessing the fragment's root.data.
"""
if self.fragment:
# Access .root.data rather than ._data to ensure that
# we have pulled it at least once.
set_pointer(self.root.data, self.fragment, value)
else:
self._data = value
def render(self, dict_to_render, list_item_id):
"""
Transform the current schema json to a fully rendered dictionary
"""
rendered_data = deepcopy(dict_to_render)
pointers = find_pointers_containing(rendered_data, "placeholders")
for pointer in pointers:
rendered_text = self.render_pointer(rendered_data, pointer, list_item_id)
set_pointer(rendered_data, pointer, rendered_text)
return rendered_data
def data(self, value):
""" Modify the data associated for this resource.
Note that while a root DataRep can be set without triggering a pull,
setting a fragment requires accessing the fragment's root.data.
"""
if self.fragment:
# Access .root.data rather than ._data to ensure that
# we have pulled it at least once.
set_pointer(self.root.data, self.fragment, value)
else:
self._data = value
def update_file_key(self, key):
"""Update file key if condition is valid."""
if self.condition:
jsonpointer.set_pointer(self.instance,
self.validator_value['file_key'], key)
else:
data, part = jsonpointer.JsonPointer(
self.validator_value['file_key']).to_last(self.instance)
if data:
if part is not None:
del data[part]
else:
del data
def writeNewData(self):
set_pointer(self.data, self.vb_pointer, self.vb_tags)
set_pointer(self.data, self.pn_pointer, self.pn_tags)
set_pointer(self.data, self.aff_pointer, self.aff_tags)
set_pointer(self.data, self.affopt_pointer, self.affopt_tags)
with open(self.post_path, 'w') as f:
json.dump(self.data, f)
def render(self, dict_to_render, list_item_id):
"""
Transform the current schema json to a fully rendered dictionary
"""
dict_to_render = QuestionnaireSchema.get_mutable_deepcopy(
dict_to_render)
pointers = find_pointers_containing(dict_to_render, "placeholders")
for pointer in pointers:
rendered_text = self.render_pointer(dict_to_render, pointer,
list_item_id)
set_pointer(dict_to_render, pointer, rendered_text)
return dict_to_render
def apply_pointers(self, content, params):
"""
Let's apply all the json pointers!
Valid params in Nulecule:
param1:
- /spec/containers/0/ports/0/hostPort
- /spec/containers/0/ports/0/hostPort2
or
param1:
- /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2
Args:
content (str): content of artifact file
params (dict): list of params with pointers to replace in content
Returns:
str: content with replaced pointers
Todo:
In the future we need to change this to detect haml, yaml, etc as we add more providers
Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393
"""
obj = anymarkup.parse(content)
if type(obj) != dict:
logger.debug("Artifact file not json/haml, assuming it's $VARIABLE substitution")
return content
if params is None:
# Nothing to do here!
return content
for name, pointers in params.items():
if not pointers:
logger.warning("Could not find pointer for %s" % name)
continue
for pointer in pointers:
try:
resolve_pointer(obj, pointer)
set_pointer(obj, pointer, name)
logger.debug("Replaced %s pointer with %s param" % (pointer, name))
except JsonPointerException:
logger.debug("Error replacing %s with %s" % (pointer, name))
logger.debug("Artifact content: %s", obj)
raise NuleculeException("Error replacing pointer %s with %s." % (pointer, name))
return anymarkup.serialize(obj, format="json")
def main():
module = AnsibleModule(
argument_spec=dict(
data=dict(required=True, type='dict'),
set_data=dict(type='json'),
),
supports_check_mode=True,
)
if jsonpointer is None:
module.fail_json(msg='jsonpointer module is not available')
data = module.params['data']
j_data = json.loads(module.params['set_data'])
for j in j_data:
pointer = "/{}".format(j)
s_data = j_data[j]
msg = ("pointer: {} set_data: {}".format(pointer, s_data))
try:
data = jsonpointer.set_pointer(data, pointer, s_data)
except jsonpointer.JsonPointerException as err:
module.fail_json(msg=str(err))
module.exit_json(changed=True, result=data)
def redact_lines(self, refs: List[str]):
def line_path(ref: str) -> str:
end = ref.find("/word")
# Remove leading '#' and trailing word path.
return ref[1:end]
for ref in refs:
r = line_path(ref)
line = resolve_pointer(self.ocr_result, r)
tokens = line["text"].split(' ')
word_id = int(ref.split('/')[-1])
tokens[word_id] = first_char(tokens[word_id])
line["text"] = ' '.join(tokens)
set_pointer(self.ocr_result, r, line)
def _json_set(query: str,
value: str,
input_filename: str = '-',
output_filename: str = '-') -> None:
doc = _load_json(input_filename)
value = json.loads(value)
result = set_pointer(doc, query, value)
_save_json(result, output_filename)
def __set__(self, obj, val):
# TODO: Abstract JsonPointerException when settings a member with missing nested parent dict
# We need to deepcopy data, so that SQLAlchemy can detect changes. Otherwise nested changes would mutate the dict in-place and SQLAlchemy cannot perform comparison.
data = copy.deepcopy(self.ensure_valid_data(obj))
val = self.converter.serialize(val)
if val is not None:
# Do some basic data validation, don't let first class objects slip through
if type(val) not in (str, float, bool, int, dict):
raise BadJSONData("Cannot update field at {} as it has unsupported type {} for JSONB data".format(self.pointer, type(val)))
jsonpointer.set_pointer(data, self.pointer, val)
set_attribute(obj, self.data_field, data)
def test_command_release_field_is_null(pointer, monkeypatch):
data = json.loads(read('release_minimal.json'))
parts = pointer.split('/')
for i, part in enumerate(parts, 1):
if i < len(parts):
if parts[i] == '0':
value = [{}]
else:
value = {}
else:
value = None
set_pointer(data, '/' + '/'.join(parts[:i]), value)
stdin = json.dumps(data).encode('utf-8')
# Should not raise an error.
run_streaming(monkeypatch, main, ['upgrade', '1.0:1.1'], stdin)
def __set__(self, obj, val):
# TODO: Abstract JsonPointerException when settings a member with missing nested parent dict
# We need to deepcopy data, so that SQLAlchemy can detect changes. Otherwise nested changes would mutate the dict in-place and SQLAlchemy cannot perform comparison.
data = copy.deepcopy(self.ensure_valid_data(obj))
val = self.converter.serialize(val)
if val is not None:
# Do some basic data validation, don't let first class objects slip through
if type(val) not in (str, float, bool, int, dict):
raise BadJSONData("Cannot update field at {} as it has unsupported type {} for JSONB data".format(self.pointer, type(val)))
jsonpointer.set_pointer(data, self.pointer, val)
set_attribute(obj, self.data_field, data)
def filter_values(data, keys):
data = copy.deepcopy(data)
for key in keys:
try:
key_data = jsonpointer.resolve_pointer(data, key)
except jsonpointer.JsonPointerException:
# Data does not exist anyway, nothing to do!
continue
key_status = key_data.get("status", "") if isinstance(key_data,
dict) else ""
if isinstance(key_status,
str) and key_status.strip().lower() == "public":
pass
else:
jsonpointer.set_pointer(data, key, None)
return data
def graft(model, pointer, branch):
'''Add branch to model at pointer.
Note, pointer must have already support in model. Ie, there is no
'mkdir'.
'''
if not pointer:
return branch
return jsonpointer.set_pointer(model, pointer, branch)
def write_swagger(dictionaries: List[Dictionary]):
static_dir = os.path.dirname(static.__file__)
pre_path = os.path.join(os.path.dirname(swagger_dir.__file__), 'swagger-pre.json')
post_path = os.path.join(static_dir, 'swagger.json')
with open(pre_path, 'r', encoding='utf8') as f:
data = json.load(f)
languages_pointer = '/components/schemas/LanguageTags/enum'
language_tags = sorted(list(set([d.name for d in dictionaries])))
new_data = set_pointer(data, languages_pointer, language_tags)
with open(post_path, 'w', encoding='utf8') as f:
json.dump(new_data, f)
def set_json_value(self, json_string, json_pointer, json_value):
"""
Set the target node of the JSON document `json_string` specified by
JSON Pointer `json_pointer` to `json_value`.
Example:
| ${result}= | Set Json Value | {"foo": {"bar": [1,2,3]}} | /foo | 12 |
| Should Be Equal | ${result} | {"foo": 12} | | |
"""
value = load_json(json_value)
p = jsonpointer.set_pointer(json_string, json_pointer, value)
return json_string
def test_alttypes(self):
Node = self.Node
root = Node('root')
root.set_left(Node('a'))
root.left.set_left(Node('aa'))
root.left.set_right(Node('ab'))
root.set_right(Node('b'))
root.right.set_left(Node('ba'))
root.right.set_right(Node('bb'))
self.assertEqual(resolve_pointer(root, '/left').name, 'a')
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(root, '/right').name, 'b')
self.assertEqual(resolve_pointer(root, '/right/left').name, 'ba')
newroot = set_pointer(root, '/left/right', Node('AB'), inplace=False)
self.assertEqual(resolve_pointer(root, '/left/right').name, 'ab')
self.assertEqual(resolve_pointer(newroot, '/left/right').name, 'AB')
set_pointer(root, '/left/right', Node('AB'))
self.assertEqual(resolve_pointer(root, '/left/right').name, 'AB')
def test_datarep_data_setter_fragment(mock_service):
root_data = copy.deepcopy(ANY_DATA)
root = datarep.DataRep(mock_service, ANY_URI, ANY_DATA_SCHEMA,
data=root_data)
fragment = datarep.DataRep(root=root, fragment=ANY_FRAGMENT_PTR)
modified_data = set_pointer(ANY_DATA, ANY_FRAGMENT_PTR, 42, inplace=False)
root.pull = mock.Mock()
fragment.pull = mock.Mock()
fragment.data = 42
assert fragment._data is datarep.DataRep.FRAGMENT
assert fragment.data == 42
assert root.data == modified_data
assert not root.pull.called
assert not fragment.pull.called
def main():
module = AnsibleModule(
argument_spec=dict(
data=dict(required=True, type='dict'),
pointer=dict(required=True),
action=dict(required=True, choices=['append', 'extend', 'update']),
update=dict(type='str'),
extend=dict(type='list'),
append=dict(),
),
supports_check_mode=True,
)
if jsonpointer is None:
module.fail_json(msg='jsonpointer module is not available')
action = module.params['action']
data = module.params['data']
pointer = module.params['pointer']
if isinstance(data, str):
data = json.loads(str)
try:
res = jsonpointer.resolve_pointer(data, pointer)
except jsonpointer.JsonPointerException as err:
module.fail_json(msg=str(err))
if action == 'append':
res.append(module.params['append'])
if action == 'extend':
res.extend(module.params['extend'])
elif action == 'update':
res = jsonpointer.set_pointer(data, pointer, module.params['update'])
# res.update(module.params['update'])
module.exit_json(changed=True, result=data)
def uri(self, value):
"""Set new uri value in record.
It will not change the location of the underlying file!
"""
jsonpointer.set_pointer(self.record, self.pointer, value)
def _patch(obj, jsonptr, value):
"""Returns patched object."""
return jsonpointer.set_pointer(obj, jsonptr, value, inplace=False)
def setin(self, path, value):
"""
Set a value using JSON-pointer syntax
"""
set_pointer(self, path, value)
def parse_schema(self, resp):
"""Function to get and replace schema $ref with data
:param resp: response data containing ref items.
:type resp: str.
"""
#pylint: disable=maybe-no-member
jsonpath_expr = jsonpath_rw.parse('$.."$ref"')
matches = jsonpath_expr.find(resp.dict)
respcopy = resp.dict
typeregex = '([#,@].*?\.)'
if matches:
for match in matches:
fullpath = str(match.full_path)
jsonfile = match.value.split('#')[0]
jsonpath = match.value.split('#')[1]
listmatch = None
found = None
if 'redfish.dmtf.org' in jsonfile:
if 'odata' in jsonfile:
jsonpath = jsonpath.replace(jsonpath.split('/')[-1], \
'odata' + jsonpath.split('/')[-1])
jsonfile = 'Resource.json'
found = re.search(typeregex, fullpath)
if found:
repitem = fullpath[found.regs[0][0]:found.regs[0][1]]
schemapath = '/' + fullpath.replace(repitem, '~').\
replace('.', '/').replace('~', repitem)
else:
schemapath = '/' + fullpath.replace('.', '/')
if '.json' in jsonfile:
itempath = schemapath
if self.is_redfish:
if resp.request.path[-1] == '/':
newpath = '/'.join(resp.request.path.split('/')\
[:-2]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')\
[:-1]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')[:-1]) \
+ '/' + jsonfile
if 'href.json' in newpath:
continue
if not newpath.lower() in self._visited_urls:
self.load(newpath, skipcrawl=True, includelogs=False, \
skipinit=True, loadtype='ref')
instance = list()
#deprecated type "string" for Type.json
if 'string' in self.types:
for item in self.itertype('string'):
instance.append(item)
if 'object' in self.types:
for item in self.itertype('object'):
instance.append(item)
for item in instance:
if jsonfile in item.resp._rest_request._path:
if 'anyOf' in fullpath:
break
dictcopy = item.resp.dict
listmatch = re.search('[[][0-9]+[]]', itempath)
if listmatch:
start = listmatch.regs[0][0]
end = listmatch.regs[0][1]
newitempath = [
itempath[:start], itempath[end:]
]
start = jsonpointer.JsonPointer(newitempath[0])
end = jsonpointer.JsonPointer(newitempath[1])
del start.parts[-1], end.parts[-1]
vals = start.resolve(respcopy)
count = 0
for val in vals:
try:
if '$ref' in six.iterkeys(
end.resolve(val)):
end.resolve(val).pop('$ref')
end.resolve(val).update(dictcopy)
replace_pointer = jsonpointer.\
JsonPointer(end.path + jsonpath)
data = replace_pointer.resolve(val)
set_pointer(val, end.path, data)
start.resolve(respcopy)[count].\
update(val)
break
except:
count += 1
else:
itempath = jsonpointer.JsonPointer(itempath)
del itempath.parts[-1]
if '$ref' in six.iterkeys(
itempath.resolve(respcopy)):
itempath.resolve(respcopy).pop('$ref')
itempath.resolve(respcopy).update(dictcopy)
break
if jsonpath:
if 'anyOf' in fullpath:
continue
if not jsonfile:
replacepath = jsonpointer.JsonPointer(jsonpath)
schemapath = schemapath.replace('/$ref', '')
if re.search('\[\d]', schemapath):
schemapath = schemapath.translate(None, '[]')
schemapath = jsonpointer.JsonPointer(schemapath)
data = replacepath.resolve(respcopy)
if '$ref' in schemapath.resolve(respcopy):
schemapath.resolve(respcopy).pop('$ref')
schemapath.resolve(respcopy).update(data)
else:
if not listmatch:
schemapath = schemapath.replace('/$ref', '')
replacepath = schemapath + jsonpath
replace_pointer = jsonpointer.\
JsonPointer(replacepath)
data = replace_pointer.resolve(respcopy)
set_pointer(respcopy, schemapath, data)
resp.json(respcopy)
else:
resp.json(respcopy)
def _set_value(self, pointer, value):
set_pointer(self.resolver, pointer, value)
def parse_schema(self, resp):
"""Function to get and replace schema $ref with data
:param resp: response data containing ref items.
:type resp: str.
"""
jsonpath_expr = jsonpath_rw.parse(u'$.."$ref"')
matches = jsonpath_expr.find(resp.dict)
respcopy = resp.dict
if matches:
for match in matches:
fullpath = str(match.full_path)
jsonfile = match.value.split('#')[0]
jsonpath = match.value.split('#')[1]
if '@odata' in fullpath:
schemapath = '/' + fullpath.replace('@odata.', '~').\
replace('.', '/').replace('~', '@odata.')
else:
schemapath = '/' + fullpath.replace('.', '/')
if '.json' in jsonfile:
itempath = schemapath
if self.is_redfish:
if resp.request.path[-1] == '/':
newpath = '/'.join(resp.request.path.split('/')\
[:-2]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')\
[:-1]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')[:-1]) \
+ '/' + jsonfile
if 'href.json' in newpath:
continue
if not newpath.lower() in self._visited_urls:
self.load(newpath, skipcrawl=True, includelogs=False, \
skipinit=True, loadtype='ref')
instance = list()
if u'st' in self.types:
for x in self.types[u'st'][u'Instances']:
instance.append(x)
if u'ob' in self.types:
for x in self.types[u'ob'][u'Instances']:
instance.append(x)
for item in instance:
if jsonfile in item.resp._rest_request._path:
if 'anyOf' in fullpath:
break
dictcopy = item.resp.dict
listmatch = re.search('[[][0-9]+[]]', itempath)
if listmatch:
start = listmatch.regs[0][0]
end = listmatch.regs[0][1]
newitempath = [itempath[:start],\
itempath[end:]]
start = jsonpointer.JsonPointer(newitempath[0])
end = jsonpointer.JsonPointer(newitempath[1])
del start.parts[-1], end.parts[-1]
vals = start.resolve(respcopy)
count = 0
for val in vals:
try:
if '$ref' in end.resolve(
val).iterkeys():
end.resolve(val).pop('$ref')
end.resolve(val).update(dictcopy)
replace_pointer = jsonpointer.\
JsonPointer(end.path + jsonpath)
data = replace_pointer.resolve(val)
set_pointer(val, end.path, data)
start.resolve(respcopy)[count].\
update(val)
break
except:
count += 1
else:
itempath = jsonpointer.JsonPointer(itempath)
del itempath.parts[-1]
if '$ref' in itempath.resolve(respcopy).\
iterkeys():
itempath.resolve(respcopy).pop('$ref')
itempath.resolve(respcopy).update(dictcopy)
if jsonpath:
if 'anyOf' in fullpath:
continue
if not jsonfile:
replacepath = jsonpointer.JsonPointer(jsonpath)
schemapath = schemapath.replace('/$ref', '')
schemapath = jsonpointer.JsonPointer(schemapath)
data = replacepath.resolve(respcopy)
if '$ref' in schemapath.resolve(respcopy):
schemapath.resolve(respcopy).pop('$ref')
schemapath.resolve(respcopy).update(data)
else:
if not listmatch:
schemapath = schemapath.replace('/$ref', '')
replacepath = schemapath + jsonpath
replace_pointer = jsonpointer.\
JsonPointer(replacepath)
data = replace_pointer.resolve(respcopy)
set_pointer(respcopy, schemapath, data)
resp.json(respcopy)
else:
resp.json(respcopy)
def ops_published_data_crawl(constituents, query, chunksize):
if constituents != 'pub-number':
raise ValueError('constituents "{0}" invalid or not implemented yet'.format(constituents))
real_constituents = constituents
if constituents == 'pub-number':
constituents = ''
# fetch first chunk (1-chunksize) from upstream
first_chunk = ops_published_data_search(constituents, query, '1-{0}'.format(chunksize))
#print first_chunk
pointer_total_count = JsonPointer('/ops:world-patent-data/ops:biblio-search/@total-result-count')
total_count = int(pointer_total_count.resolve(first_chunk))
log.info('ops_published_data_crawl total_count: %s', total_count)
# The first 2000 hits are accessible from OPS.
total_count = min(total_count, 2000)
# collect upstream results
begin_second_chunk = chunksize + 1
chunks = [first_chunk]
for range_begin in range(begin_second_chunk, total_count + 1, chunksize):
# countermeasure to robot flagging
# <code>CLIENT.RobotDetected</code>
# <message>Recent behaviour implies you are a robot. The server is at the moment busy to serve robots. Please try again later</message>
time.sleep(5)
range_end = range_begin + chunksize - 1
range_string = '{0}-{1}'.format(range_begin, range_end)
log.info('ops_published_data_crawl range: ' + range_string)
chunk = ops_published_data_search(constituents, query, range_string)
#print 'chunk:', chunk
chunks.append(chunk)
#return chunks
# merge chunks into single result
"""
<empty>: "ops:search-result" { » "ops:publication-reference": [
biblio: "ops:search-result" { » "exchange-documents": [ » "exchange-document": {
abstract: "ops:search-result" { » "exchange-documents": [ » "exchange-document": {
full-cycle: "ops:search-result" { » "exchange-documents": [ » "exchange-document": [
pub-number: "ops:search-result" { » "ops:publication-reference": [
{
"@family-id": "6321653",
"@system": "ops.epo.org",
"document-id": {
"@document-id-type": "docdb",
"country": {
"$": "DE"
},
"doc-number": {
"$": "3705908"
},
"kind": {
"$": "A1"
}
}
},
"""
pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/ops:publication-reference')
#pointer_time_elapsed = JsonPointer('/ops:world-patent-data/ops:meta/@value')
all_results = []
#time_elapsed = int(pointer_time_elapsed.resolve(first_chunk))
for chunk in chunks:
# FIXME: use this for "real_constituents == 'pub-number'" only
chunk_results = to_list(pointer_results.resolve(chunk))
# FIXME: implement other constituents
#print 'chunk_results:', chunk_results
all_results += chunk_results
#time_elapsed += int(pointer_time_elapsed.resolve(chunk))
response = None
if real_constituents == 'pub-number':
response = first_chunk
# delete upstream data
del resolve_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:search-result')['ops:publication-reference']
# compute own representation
publication_numbers = []
pointer_document_id = JsonPointer('/document-id')
for entry in all_results:
pubref = pointer_document_id.resolve(entry)
#print entry, pubref
pubref_number, pubref_date = _get_document_number_date(pubref, 'docdb')
publication_numbers.append(pubref_number)
# add own representation
set_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:search-result/publication-numbers', publication_numbers, inplace=True)
# amend metadata
new_total_count = str(len(publication_numbers))
pointer_total_count.set(response, new_total_count)
set_pointer(response, '/ops:world-patent-data/ops:biblio-search/ops:range', {'@begin': '1', '@end': new_total_count})
#pointer_time_elapsed.set(response, str(time_elapsed))
if not response:
raise ValueError('constituents "{0}" invalid or not implemented yet'.format(constituents))
return response
def parse_schema_gen(self, resp):
"""Redfish general function to get and replace schema $ref with data
:param resp: response data containing ref items.
:type resp: str.
"""
#pylint: disable=maybe-no-member
getval = lambda inmat: getval(inmat.left) + '/' + str(inmat.right) \
if hasattr(inmat, 'left') else str(inmat)
respcopy = resp.dict
jsonpath_expr = jsonpath_rw.parse('$.."anyOf"')
while True:
matches = jsonpath_expr.find(respcopy)
if not matches:
break
match = matches[0]
newval = None
schlist = match.value
schlist = [ele for ele in list(schlist) if ele != {"type": "null"}]
norefsch = [ele for ele in list(schlist) if isinstance(ele, dict) and \
len(ele.keys()) > 1]
if norefsch:
newval = norefsch[0]
else:
newsc = [
ele for ele in list(schlist)
if not ele["$ref"].split('#')[0]
]
newval = newsc[0] if newsc else None
if not newval:
schlist = [ele["$ref"] for ele in list(schlist) if "$ref" in ele.keys() and \
(ele["$ref"].split('#')[0].endswith('.json') and 'odata' not in \
ele["$ref"].split('#')[0])]
maxsch = max(schlist)
newval = {"$ref": maxsch}
itempath = '/' + getval(match.full_path)
if re.search('\[\d+]', itempath):
itempath = itempath.translate(None, '[]')
itempath = jsonpointer.JsonPointer(itempath)
del itempath.parts[-1]
if 'anyOf' in six.iterkeys(itempath.resolve(respcopy)):
itempath.resolve(respcopy).pop('anyOf')
itempath.resolve(respcopy).update(newval)
jsonpath_expr = jsonpath_rw.parse('$.."$ref"')
matches = jsonpath_expr.find(respcopy)
if matches:
for _, match in enumerate(matches):
jsonfile = match.value.split('#')[0]
jsonfile = '' if jsonfile.lower() == resp.request.path.lower(
) else jsonfile
jsonpath = match.value.split('#')[1]
schemapath = '/' + getval(match.full_path)
if jsonfile:
itempath = schemapath
if '/' not in jsonfile:
inds = -2 if resp.request.path[-1] == '/' else -1
jsonfile = '/'.join(resp.request.path.split('/')[:inds]) \
+ '/' + jsonfile + '/'
if jsonfile not in self.paths:
self.load(jsonfile, crawl=False, includelogs=False, \
init=False, loadtype='ref')
item = self.paths[
jsonfile] if jsonfile in self.paths else None
# if not item:
# if not 'anyOf' in schemapath:
# raise "We got a situation :|"
# continue
if re.search('\[\d+]', itempath):
itempath = itempath.translate(None, '[]')
itempath = jsonpointer.JsonPointer(itempath)
del itempath.parts[-1]
if '$ref' in six.iterkeys(itempath.resolve(respcopy)):
itempath.resolve(respcopy).pop('$ref')
itempath.resolve(respcopy).update(item.dict)
if jsonpath:
schemapath = schemapath.replace('/$ref', '')
if re.search('\[\d+]', schemapath):
schemapath = schemapath.translate(None, '[]')
if not jsonfile:
replacepath = jsonpointer.JsonPointer(jsonpath)
schemapath = jsonpointer.JsonPointer(schemapath)
data = replacepath.resolve(respcopy)
if '$ref' in schemapath.resolve(respcopy):
schemapath.resolve(respcopy).pop('$ref')
schemapath.resolve(respcopy).update(data)
else:
replacepath = schemapath + jsonpath
replace_pointer = jsonpointer.\
JsonPointer(replacepath)
data = replace_pointer.resolve(respcopy)
set_pointer(respcopy, schemapath, data)
resp.loaddict(respcopy)
else:
resp.loaddict(respcopy)
def test_set(self):
doc = {
"foo": ["bar", "baz"],
"": 0,
"a/b": 1,
"c%d": 2,
"e^f": 3,
"g|h": 4,
"i\\j": 5,
"k\"l": 6,
" ": 7,
"m~n": 8
}
origdoc = copy.deepcopy(doc)
# inplace=False
newdoc = set_pointer(doc, "/foo/1", "cod", inplace=False)
self.assertEqual(resolve_pointer(newdoc, "/foo/1"), "cod")
newdoc = set_pointer(doc, "/", 9, inplace=False)
self.assertEqual(resolve_pointer(newdoc, "/"), 9)
newdoc = set_pointer(doc, "/fud", {}, inplace=False)
newdoc = set_pointer(newdoc, "/fud/gaw", [1, 2, 3], inplace=False)
self.assertEqual(resolve_pointer(newdoc, "/fud"), {'gaw' : [1, 2, 3]})
newdoc = set_pointer(doc, "", 9, inplace=False)
self.assertEqual(newdoc, 9)
self.assertEqual(doc, origdoc)
# inplace=True
set_pointer(doc, "/foo/1", "cod")
self.assertEqual(resolve_pointer(doc, "/foo/1"), "cod")
set_pointer(doc, "/", 9)
self.assertEqual(resolve_pointer(doc, "/"), 9)
self.assertRaises(JsonPointerException, set_pointer, doc, "/fud/gaw", 9)
set_pointer(doc, "/fud", {})
set_pointer(doc, "/fud/gaw", [1, 2, 3] )
self.assertEqual(resolve_pointer(doc, "/fud"), {'gaw' : [1, 2, 3]})
self.assertRaises(JsonPointerException, set_pointer, doc, "", 9)
def parse_schema(self, resp):
"""Function to get and replace schema $ref with data
:param resp: response data containing ref items.
:type resp: str.
"""
jsonpath_expr = jsonpath_rw.parse(u'$.."$ref"')
matches = jsonpath_expr.find(resp.dict)
respcopy = resp.dict
if matches:
for match in matches:
fullpath = str(match.full_path)
jsonfile = match.value.split('#')[0]
jsonpath = match.value.split('#')[1]
if '@odata' in fullpath:
schemapath = '/' + fullpath.replace('@odata.', '~').\
replace('.', '/').replace('~', '@odata.')
else:
schemapath = '/' + fullpath.replace('.', '/')
if '.json' in jsonfile:
itempath = schemapath
if self.is_redfish:
if resp.request.path[-1] == '/':
newpath = '/'.join(resp.request.path.split('/')\
[:-2]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')\
[:-1]) + '/' + jsonfile + '/'
else:
newpath = '/'.join(resp.request.path.split('/')[:-1]) \
+ '/' + jsonfile
if 'href.json' in newpath:
continue
if not newpath.lower() in self._visited_urls:
self.load(newpath, skipcrawl=True, includelogs=False, \
skipinit=True, loadtype='ref')
instance = list()
if u'st' in self.types:
for stitem in self.types[u'st'][u'Instances']:
instance.append(stitem)
if u'ob' in self.types:
for obitem in self.types[u'ob'][u'Instances']:
instance.append(obitem)
for item in instance:
if jsonfile in item.resp._rest_request._path:
if 'anyOf' in fullpath:
break
dictcopy = item.resp.dict
listmatch = re.search('[[][0-9]+[]]', itempath)
if listmatch:
start = listmatch.regs[0][0]
end = listmatch.regs[0][1]
newitempath = [itempath[:start], itempath[end:]]
start = jsonpointer.JsonPointer(newitempath[0])
end = jsonpointer.JsonPointer(newitempath[1])
del start.parts[-1], end.parts[-1]
vals = start.resolve(respcopy)
count = 0
for val in vals:
try:
if '$ref' in end.resolve(val).iterkeys():
end.resolve(val).pop('$ref')
end.resolve(val).update(dictcopy)
replace_pointer = jsonpointer.\
JsonPointer(end.path + jsonpath)
data = replace_pointer.resolve(val)
set_pointer(val, end.path, data)
start.resolve(respcopy)[count].\
update(val)
break
except:
count += 1
else:
itempath = jsonpointer.JsonPointer(itempath)
del itempath.parts[-1]
if '$ref' in itempath.resolve(respcopy).\
iterkeys():
itempath.resolve(respcopy).pop('$ref')
itempath.resolve(respcopy).update(dictcopy)
if jsonpath:
if 'anyOf' in fullpath:
continue
if not jsonfile:
replacepath = jsonpointer.JsonPointer(jsonpath)
schemapath = schemapath.replace('/$ref', '')
schemapath = jsonpointer.JsonPointer(schemapath)
data = replacepath.resolve(respcopy)
if '$ref' in schemapath.resolve(respcopy):
schemapath.resolve(respcopy).pop('$ref')
schemapath.resolve(respcopy).update(data)
else:
if not listmatch:
schemapath = schemapath.replace('/$ref', '')
replacepath = schemapath + jsonpath
replace_pointer = jsonpointer.\
JsonPointer(replacepath)
data = replace_pointer.resolve(respcopy)
set_pointer(respcopy, schemapath, data)
resp.json(respcopy)
else:
resp.json(respcopy)
def update(self, pointer: str, value):
if not pointer.startswith("/"):
pointer = "/" + pointer
jsonpointer.set_pointer(self.data, pointer, value, inplace=True)
def modify_json(json_data, pointer, json_value, state='present', action='add'):
is_root = False # special treatment - we cannot modify reference in place
key_exists = False
try:
value = json.loads(json_value)
except Exception:
value = None
if state == 'present':
if action not in ['add', 'update', 'replace']:
raise ValueError
elif state == 'absent':
pass
else:
raise ValueError
# we store the original json document to compare it later
original_json_data = copy.deepcopy(json_data)
try:
target = jsonpointer.resolve_pointer(json_data, pointer)
if pointer == '':
is_root = True
key_exists = True
except jsonpointer.JsonPointerException:
key_exists = False
if key_exists:
if state == "present":
if action == "add":
if isinstance(target, dict) and isinstance(value, dict):
# we keep old values and only append new ones
value.update(target)
result = jsonpointer.set_pointer(json_data,
pointer,
value,
inplace=(not is_root))
if is_root:
json_data = result
elif isinstance(target, list) and isinstance(value, list):
# we just append new items to the list
for item in value:
if item not in target:
target.append(item)
elif ((not isinstance(target, dict))
and (not isinstance(target, list))):
# 'add' does not overwrite
pass
else:
raise ValueError
elif action == "update":
if isinstance(target, dict) and isinstance(value, dict):
# we append new values and overwrite the old ones
target.update(value)
elif isinstance(target, list) and isinstance(value, list):
# we just append new items to the list - same as with 'add'
for item in value:
if item not in target:
target.append(item)
elif ((not isinstance(target, dict))
and (not isinstance(target, list))):
# 'update' DOES overwrite
if value is not None:
result = jsonpointer.set_pointer(
json_data, pointer, value)
elif target != json_value:
result = jsonpointer.set_pointer(
json_data, pointer, json_value)
else:
raise ValueError
else:
raise ValueError
elif action == "replace":
# simple case when we don't care what was there before (almost)
if value is not None:
result = jsonpointer.set_pointer(json_data,
pointer,
value,
inplace=(not is_root))
else:
result = jsonpointer.set_pointer(json_data,
pointer,
json_value,
inplace=(not is_root))
if is_root:
json_data = result
else:
raise ValueError
elif state == "absent":
# we will delete the elements in the object or object itself
if is_root:
if json_value == '':
# we just return empty json
json_data = {}
elif isinstance(target, dict) and isinstance(value, dict):
for key in value:
target.pop(key, None)
else:
raise ValueError
else:
# we must take a step back in the pointer, so we can edit it
ppointer = pointer.split('/')
to_delete = ppointer.pop()
ppointer = '/'.join(ppointer)
ptarget = jsonpointer.resolve_pointer(json_data, ppointer)
if (((not isinstance(target, dict)) and
(not isinstance(target, list)) and json_value == '') or
(isinstance(target, dict) or isinstance(target, list))
and json_value == ''):
# we simply delete the key with it's value (whatever it is)
ptarget.pop(to_delete, None)
target = ptarget # piece of self-defense
elif isinstance(target, dict) and isinstance(value, dict):
for key in value:
target.pop(key, None)
elif isinstance(target, list) and isinstance(value, list):
for item in value:
try:
target.remove(item)
except ValueError:
pass
else:
raise ValueError
else:
raise ValueError
else:
# the simplest case - nothing was there before and pointer is not root
# because in that case we would have key_exists = true
if state == 'present':
if value is not None:
result = jsonpointer.set_pointer(json_data, pointer, value)
else:
result = jsonpointer.set_pointer(json_data, pointer,
json_value)
if json_data != original_json_data:
changed = True
else:
changed = False
if changed:
msg = "JSON object '%s' was updated" % pointer
else:
msg = "No change to JSON object '%s'" % pointer
return json_data, changed, msg
