def test_non_json_query(self):
"""Test complex query"""
data = [{"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}}]
cmd = "map({id: x.a, data: x.b.d})"
expected = '[{"data": [1, 2, 3, 4], "id": 1}]'
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_hide(self):
"""Test complex query"""
data = [{"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}}]
cmd = 'map({"id": x.a, "data": x.b.d}), hide("data")'
expected = '[{"id": 1}]'
result = tolist(list(jf.run_query(cmd, data, "itertools", "modules")))
self.assertEqual(result, expected)
def test_last_two_with_sort(self):
"""Test fetching last two items using sort and first"""
data = [
{"a": 2, "b": "2018-01-30 16:28:40+00:00"},
{"a": 1, "b": "2018-01-30 15:12:35+00:00"},
{"a": 3, "b": "2018-01-10 15:12:35+00:00"},
{"a": 5, "b": "2018-01-30 16:06:59+03:00"},
]
cmd = "map({id: x.a, date: x.b}), sorted(age(.date), reverse=False), map(.id), first(2)"
expected = "[2, 1]"
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_sorting_query(self):
"""Test complex query"""
data = [
{"a": 2, "b": {"c": 632, "d": [1, 2, 3, 4]}},
{"a": 1, "b": {"c": 632, "d": [3, 4, 5, 6]}},
{"a": 5, "b": {"c": 632, "d": [5, 6, 7, 8]}},
]
cmd = "map({id: x.a, data: x.b.d[0]}), sorted(.id, reverse=True)"
expected = '[{"data": 5, "id": 5}, {"data": 1, "id": 2}, {"data": 3, "id": 1}]'
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_filter_query3(self):
"""Test complex query"""
data = [
{"a": 2, "b": {"c": 632, "d": [1, 2, 3, 4]}},
{"a": 1, "b": {"c": 632, "d": [3, 4, 5, 6]}},
{"a": 5, "b": {"c": 632, "d": [5, 6, 7, 8]}},
]
cmd = "map({id: x.a, data: x.b.d[0]}), filter(.id < .data)"
expected = '[{"data": 3, "id": 1}]'
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_age_tz(self):
"""Test age with items containing differing timezones"""
data = [
{"a": 2, "b": "2018-01-30 16:28:40+00:00"},
{"a": 1, "b": "2018-01-30 15:12:35+00:00"},
{"a": 3, "b": "2018-01-10 15:12:35+00:00"},
{"a": 5, "b": "2018-01-30 16:06:59+03:00"},
]
cmd = (
"map({id: x.a, date: x.b})," + "sorted(age(.date), reverse=True), map(.id)"
)
expected = "[3, 5, 1, 2]"
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_hide_many(self):
"""Test complex query"""
data = [
{"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}},
{"a": 2, "f": 4, "b": {"c": 632, "d": [1, 2, 3, 4]}},
]
cmd = (
'map({"id": x.a, "c": x.b.c, "data": x.b.d, "x": x["f"]}),'
+ 'hide("data", "c")'
)
expected = '[{"id": 1, "x": null}, {"id": 2, "x": 4}]'
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_first(self):
"""Test fetching first items"""
data = [
{"a": 2, "b": "2018-01-30 16:28:40+00:00"},
{"a": 1, "b": "2018-01-30 15:12:35+00:00"},
{"a": 3, "b": "2018-01-10 15:12:35+00:00"},
{"a": 5, "b": "2018-01-30 16:06:59+03:00"},
]
cmd = (
"map({id: x.a, date: x.b}),"
+ "sorted(age(.date), reverse=True), map(.id), first()"
)
expected = "[3]"
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_age_output(self):
"""
$ cat test.yaml | jf --yamli
'update({id: x.sha, age: age(x.commit.author.date)}),
filter(x.age < age("1 days"))' --indent=2 --yaml
"""
data = [
{"a": 2, "b": "2018-01-30 16:28:40+00:00"},
{"a": 1, "b": "2018-01-30 15:12:35+00:00"},
]
cmd = "map({id: x.a, date: x.b}), sorted(age(.date), reverse=True), first()"
expected = '[{"date": "2018-01-30 15:12:35+00:00", "id": 1}]'
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def test_age(self):
"""
$ cat test.yaml | jf --yamli
'update({id: x.sha, age: age(x.commit.author.date)}),
filter(x.age < age("1 days"))' --indent=2 --yaml
"""
data = [
{"a": 2, "b": "2018-01-30 16:28:40+00:00"},
{"a": 1, "b": "2018-01-30 15:12:35+00:00"},
{"a": 3, "b": "2018-01-10 15:12:35+00:00"},
{"a": 5, "b": "2018-01-30 16:06:59+00:00"},
]
cmd = (
"map({id: x.a, date: x.b}),"
+ "sorted(age(.date), reverse=True),"
+ "map(.id)"
)
expected = "[3, 1, 5, 2]"
result = tolist(list(jf.run_query(cmd, data)))
self.assertEqual(result, expected)
def main(args=None):
"""Main JF execution function"""
parser = argparse.ArgumentParser()
parser.add_argument(
"query",
nargs="?",
default="I",
help="query string for extracting wanted information",
)
parser.add_argument("-d",
"--debug",
action="store_true",
help="print debug messages")
parser.add_argument("-c", "--compact", action="store_true")
parser.add_argument("--indent",
type=int,
default=2,
help="pretty-printed with given indent level")
parser.add_argument("--import_from",
help="add path to search imports from")
parser.add_argument("--import", help="import custom processing module")
parser.add_argument("-l",
"--list",
action="store_true",
help="wrap output to a list")
parser.add_argument("-y",
"--yaml",
action="store_true",
help="output yaml")
parser.add_argument("--yamli", action="store_true", help="input yaml")
parser.add_argument("-j",
"--json",
action="store_true",
help="output json")
parser.add_argument("-s",
"--sort_keys",
action="store_true",
help="sort json output values")
parser.add_argument("--bw",
action="store_true",
default=False,
help="remove colors")
parser.add_argument("--ordered_dict",
action="store_true",
default=False,
help="user ordered dict")
parser.add_argument("-r",
"--raw",
action="store_true",
default=False,
help="raw output")
parser.add_argument("-f", "--cmdfile", help="read command from file")
parser.add_argument(
"-a",
"--ensure_ascii",
action="store_true",
default=False,
help="ensure ascii only characters",
)
parser.add_argument("--ipyfake",
action="store_true",
help=argparse.SUPPRESS)
parser.add_argument("--forcecolor",
action="store_true",
help=argparse.SUPPRESS)
parser.add_argument("-p",
"--ipy",
action="store_true",
help="start IPython shell with data")
parser.add_argument(
"--html_unescape",
action="store_true",
default=False,
help="unescape html entities",
)
parser.add_argument(
"-i",
"--input",
metavar="FILE",
help="files to read. Overrides files argument list",
)
parser.add_argument("-k",
"--kwargs",
help="files to read. Overrides files argument list")
parser.add_argument(
"files",
metavar="FILE",
nargs="*",
default="-",
help="files to read, if empty, stdin is used",
)
args = parser.parse_args(args)
if args.input is not None:
args.files = [args.input]
if args.indent < 0:
args.indent = None
if args.compact:
args.indent = None
set_loggers(args.debug)
kwargs = {}
if args.kwargs:
import re
kwargsre = re.compile(r"([^:,]+)")
kwargs = kwargsre.subn(r'"\1"', args.kwargs.replace("=", ":"))[0]
kwargs = "{%s}" % kwargs
kwargs = json.loads(kwargs)
inq = read_input(args, ordered_dict=args.ordered_dict, **kwargs)
imports = None
if "import" in args.__dict__:
imports = args.__dict__["import"]
if args.cmdfile is None:
query = args.query
else:
query = ", ".join(
filter(
lambda x: len(x) and x[0] != "#",
open(args.cmdfile, "r").read().split("\n"),
))
new_imports = list(
map(
lambda x: x.split()[1],
filter(
lambda x: len(x) and x.startswith("#import "),
open(args.cmdfile, "r").read().split("\n"),
),
))
if len(new_imports):
if imports is None:
imports = []
else:
imports = imports.split(",")
imports = ",".join(imports + new_imports)
if args.query == "":
query = "I"
data = run_query(
query,
inq,
imports=imports,
import_from=args.import_from,
ordered_dict=args.ordered_dict,
)
if args.ipy or args.ipyfake:
banner = ""
if not sys.stdin.isatty():
banner = ("\nNotice: You are inputting data from stdin!\n" +
"This might cause some trouble since jf will try " +
"to get some input from you also.\n" +
"To get the full benefit of jf and IPython, " +
"consider giving the input as a file instead.\n\n" +
"To prevent any unexpected behaviour, jk will " +
"load the full dataset in memory.\n" +
"This might take a while...\n")
data = list(data)
try:
sys.stdin = open("/dev/tty")
except OSError:
pass
ipy(banner, data, args.ipyfake)
return
try:
print_results(data, args)
except FileNotFoundError as ex:
logger.warning("%s", ex)
def test_math_query(self):
data = [{"a": "123"}]
query = 'update({y: math.log(len(.a) * len(.a))}), .y'
self.assertEqual(list(jf.run_query(query, data, "math")), [math.log(3*3)])
def test_simple_query(self):
"""Test simple query"""
data = [{"a": 1}]
self.assertEqual(tolist(jf.run_query("map(x.a)", data)), "[1]")
def test_get_null_item(self):
"""Test simple query"""
data = [{"a": 1}]
self.assertEqual(tolist(jf.run_query('map(x["b"])', data)), "[null]")
