def test_syntax_error(self):
expected_message = re.escape('''\
error: syntax error, unexpected '*', expecting $end
**
1 compile error''')
with self.assertRaisesRegex(ValueError, expected_message):
pyjq.compile('**')
def test_syntax_error(self):
expected_message = re.escape('''\
error: syntax error, unexpected '*', expecting $end
**
1 compile error''')
with self.assertRaisesRegex(ValueError, expected_message):
pyjq.compile('**')
def add_missing_value_aux(config, config_aux, path, value):
get_script = pyjq.compile(path)
if get_script.first(config) == None:
aux_val = get_script.first(config_aux)
if aux_val != None:
v = aux_val
else:
v = value() if callable(value) else value
set_script = pyjq.compile(path + '=$v', vars={'v': v})
config = set_script.first(config)
if aux_val == None:
config_aux = set_script.first(config_aux)
return config, config_aux
def test_conversion_between_python_object_and_jv():
objects = [
None,
False,
True,
1,
1.5,
"string",
[None, False, True, 1, 1.5, [None, False, True], {
"foo": "bar"
}],
{
"key1":
None,
"key2":
False,
"key3":
True,
"key4":
1,
"key5":
1.5,
"key6":
[None, False, True, 1, 1.5, [None, False, True], {
"foo": "bar"
}],
},
]
s = pyjq.compile(".")
for obj in objects:
assert [obj] == s.all(obj)
def __init__(
self,
filter_spec=".",
args={},
output_raw=False
):
"""
Construct a filter. Arguments:
filter_spec - The JQ script to be used for this filter. This
may be any subclass of basestring, a list or a dict. Strings
are interpreted directly, lists are stringified and joined
with newlines (to make multi-line scripts readable in JSON)
and dicts give their "script" and "output-raw" elements
extracted and used as if they were either of the other types.
args - A dictionary of variables to be pre-set in the script.
output_raw - True to produce raw output instead of JSON.
"""
self.output_raw = output_raw
if type(filter_spec) == dict:
self.output_raw = filter_spec.get("output-raw", output_raw)
filter_spec = filter_spec.get("script", ".")
if isinstance(filter_spec, list):
filter_spec = "\n".join([str(line) for line in filter_spec])
if not isinstance(filter_spec, basestring):
raise ValueError("Filter spec must be plain text, list or dict")
self.script = pyjq.compile(filter_spec, args)
def test_conversion_between_python_object_and_jv(self):
objects = [
None,
False,
True,
1,
1.5,
"string",
[None, False, True, 1, 1.5, [None, False, True], {
'foo': 'bar'
}],
{
'key1':
None,
'key2':
False,
'key3':
True,
'key4':
1,
'key5':
1.5,
'key6': [
None, False, True, 1, 1.5, [None, False, True], {
'foo': 'bar'
}
],
},
]
s = pyjq.compile('.')
for obj in objects:
self.assertEqual([obj], s.all(obj))
def __init__(self, value, transform=None):
self.value = value
try:
self.getter = pyjq.compile(value)
except (AttributeError, ValueError) as err:
self.getter = None
self.transform = Operation(
transform) if transform is not None else identity
def read_config(path: str) -> dict:
with open(path) as f:
cfg = yaml.load(f, Loader=yaml.SafeLoader)
# Pre-compile all JQ queries
for topic, items in cfg["topics"].items():
for item in items:
if "jq" in item:
item["jq"]["query"] = pyjq.compile(item["jq"]["query"])
return cfg
def main(argv):
opts = docopt(__doc__, argv)
entity = opts['<entity>']
params = dict(i.split(':', 1) for i in opts['<params>'])
debug = opts['--debug']
jq = None
if opts['--jq']:
try:
import pyjq
except ImportError:
print('pyjq package required for jq script support', file=sys.stderr)
sys.exit(1)
jq = pyjq.compile(opts['--jq'])
api = login(opts.get('--api'))
if 'filter' in params:
params['filter'] = dict(i.split(':', 1) for i in params['filter'].split('+'))
for name, val in params['filter'].items():
params['filter'][name] = val.split(',')
if 'search' in params:
params['search'] = dict(i.split(':', 1) for i in params['search'].split('+'))
for name, val in params['search'].items():
params['search'][name] = val.split(',')
for name, val in params.items():
if isinstance(val, str) and val.lower() in ['true', 'True', 'yes', 'Yes']:
params[name] = True
if 'limit' not in params:
params['limit'] = 10
if 'searchByAny' not in params:
params['searchByAny'] = False
if 'startSearch' not in params:
params['startSearch'] = True
if debug:
print('DEBUG params:')
json.dump(params, sys.stdout, indent=2)
print('')
Entity = getattr(objects, entity, None)
if not Entity:
print('unknown entity:', entity, file=sys.stderr)
sys.exit(1)
method = getattr(Entity, opts['<method>'], None)
if not method:
print('unsupported method:', opts['<method>'], file=sys.stderr)
sys.exit(1)
result = method(api, **params)
if jq:
result = jq.first([i.json() for i in result])
json.dump(result, sys.stdout, indent=2, default=lambda i: i.json())
def compileRules(self, conditions):
ok = []
for condition in conditions:
try:
condition["compiled"] = pyjq.compile(condition["expression"])
except Exception as err:
message = err.message.replace("\n", " -> ")
self.logging.error(
"Failed to compile jq rule '%s'. Skipped. Reason: '%s'" %
(condition["name"], message))
else:
ok.append(condition)
return ok
def __init__(
self,
filter_spec=".",
args={},
output_raw=False,
groom=False,
):
"""
Construct a filter. Arguments:
filter_spec - The JQ script to be used for this filter. This
may be any subclass of basestring, a list or a dict. Strings
are interpreted directly, lists are stringified and joined
with newlines (to make multi-line scripts readable in JSON)
and dicts give their "script" and "output-raw" elements
extracted and used as if they were either of the other types.
args - A dictionary of variables to be pre-set in the script.
output_raw - True to produce raw output instead of JSON.
groom - Move all 'import' and 'include' statements to the
top of the script before compiling. This allows filters
prepending functions to user-provided scripts that do
imports to compile properly.
"""
self.output_raw = output_raw
if type(filter_spec) == dict:
self.output_raw = filter_spec.get("output-raw", output_raw)
filter_spec = filter_spec.get("script", ".")
if isinstance(filter_spec, list):
filter_spec = "\n".join([str(line) for line in filter_spec])
if not isinstance(filter_spec, basestring):
raise ValueError("Filter spec must be plain text, list or dict")
if groom:
filter_spec = _groom(filter_spec)
self.script = pyjq.compile(filter_spec,
args,
library_paths=_library_path())
def __init__(self, topic_cfg: dict, templates_cfg: dict):
self.topic_cfg = topic_cfg
self.templates_cfg = templates_cfg
self.load_json = topic_cfg.get('load_json', False)
self.jq: Optional[pyjq._pyjq.Script] = None
self.jinja: Optional[jinja2.Template] = None
self.log = logging.getLogger('handler')
self.log.setLevel(logging.DEBUG)
if "jq_query" in topic_cfg:
self.jq = pyjq.compile(topic_cfg["jq_query"])
if "jinja_query" in topic_cfg:
self.jinja = jinja2.Template(topic_cfg['jinja_query'])
if self.jinja and self.jq:
raise ValueError(
"jinja_query and jq_query can't be both specified at the same time"
)
def test_conversion_between_python_object_and_jv(self):
objects = [
None,
False,
True,
1,
1.5,
"string",
[None, False, True, 1, 1.5, [None, False, True], {'foo': 'bar'}],
{
'key1': None,
'key2': False,
'key3': True,
'key4': 1,
'key5': 1.5,
'key6': [None, False, True, 1, 1.5,
[None, False, True], {'foo': 'bar'}],
},
]
s = pyjq.compile('.')
for obj in objects:
self.assertEqual([obj], s.all(obj))
def test_compile_dot(self):
s = pyjq.compile('.')
self.assertIsInstance(s, _pyjq.Script)
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
print(pyjq.compile(".openapi").first(content))
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
summaries = pyjq.compile(".paths[] | .get.summary").all(content)
for summary in summaries:
print(summary)
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
print(pyjq.compile(".info.license.name").first(content))
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
summaries = pyjq.compile(".paths[] | .delete.summary").all(content)
for summary in summaries:
print(summary)
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
from pprint import pprint
with open("openapi.json") as fin:
content = json.load(fin)
for parameters in pyjq.compile(
'.paths."/client/cluster/search".get.parameters').all(content):
pprint(parameters)
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
with open("openapi.json") as fin:
content = fin.read()
print(pyjq.compile(".openapi").all(content))
def test_compile_dot(self):
s = pyjq.compile('.')
self.assertIsInstance(s, pyjq._Script)
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
print(pyjq.compile(".foobar").all(content))
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
from pprint import pprint
with open("openapi.json") as fin:
content = json.load(fin)
search = pyjq.compile(".paths.\"/client/cluster/search\"").first(content)
pprint(search)
print(
"----------------------------------------------------------------------------"
)
search = pyjq.compile('.paths."/client/cluster/search"').first(content)
pprint(search)
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
for endpoint in pyjq.compile('.paths[]').all(content):
print(",".join(endpoint.keys()))
print("-------------------------")
for has_delete in pyjq.compile('.paths[] | has("delete")').all(content):
print(has_delete)
def test_syntax_error(self):
expected_message = re.escape(r"error: syntax error")
with self.assertRaisesRegexp(ValueError, expected_message):
pyjq.compile('**')
def test_compile_dot():
s = pyjq.compile(".")
assert isinstance(s, _pyjq.Script)
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
for endpoint in pyjq.compile(".paths[]").all(content):
print(",".join(endpoint.keys()))
print("-------------------------")
for has_get in pyjq.compile('.paths[] | has("get")').all(content):
print(has_get)
def test_syntax_error():
with pytest.raises(ValueError, match=r"error: syntax error"):
pyjq.compile("**")
#!/usr/bin/env python3
# vim: set fileencoding=utf-8
#
# (C) Copyright 2020 Pavel Tisnovsky
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
# Pavel Tisnovsky
#
# Demonstrační příklad k článku:
# Zpracování dat uložených ve formátu JSON knihovnou pyjq
import pyjq
import json
with open("openapi.json") as fin:
content = json.load(fin)
print("-----------------------------")
print(pyjq.compile('.paths."/"').first(content))
print("-----------------------------")
print(pyjq.compile('".paths./"').first(content))
print("-----------------------------")
print(pyjq.compile('.paths./').first(content))
def test_syntax_error(self):
expected_message = re.escape(r"error: syntax error")
with self.assertRaisesRegexp(ValueError, expected_message):
pyjq.compile('**')
