def test_non_json_query(self): """Test complex query""" data = [{"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}}] cmd = "map({id: x.a, data: x.b.d})" expected = '[{"data": [1, 2, 3, 4], "id": 1}]' result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_hide(self): """Test complex query""" data = [{"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}}] cmd = 'map({"id": x.a, "data": x.b.d}), hide("data")' expected = '[{"id": 1}]' result = tolist(list(jf.run_query(cmd, data, "itertools", "modules"))) self.assertEqual(result, expected)
def test_last_two_with_sort(self): """Test fetching last two items using sort and first""" data = [ {"a": 2, "b": "2018-01-30 16:28:40+00:00"}, {"a": 1, "b": "2018-01-30 15:12:35+00:00"}, {"a": 3, "b": "2018-01-10 15:12:35+00:00"}, {"a": 5, "b": "2018-01-30 16:06:59+03:00"}, ] cmd = "map({id: x.a, date: x.b}), sorted(age(.date), reverse=False), map(.id), first(2)" expected = "[2, 1]" result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_sorting_query(self): """Test complex query""" data = [ {"a": 2, "b": {"c": 632, "d": [1, 2, 3, 4]}}, {"a": 1, "b": {"c": 632, "d": [3, 4, 5, 6]}}, {"a": 5, "b": {"c": 632, "d": [5, 6, 7, 8]}}, ] cmd = "map({id: x.a, data: x.b.d[0]}), sorted(.id, reverse=True)" expected = '[{"data": 5, "id": 5}, {"data": 1, "id": 2}, {"data": 3, "id": 1}]' result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_filter_query3(self): """Test complex query""" data = [ {"a": 2, "b": {"c": 632, "d": [1, 2, 3, 4]}}, {"a": 1, "b": {"c": 632, "d": [3, 4, 5, 6]}}, {"a": 5, "b": {"c": 632, "d": [5, 6, 7, 8]}}, ] cmd = "map({id: x.a, data: x.b.d[0]}), filter(.id < .data)" expected = '[{"data": 3, "id": 1}]' result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_age_tz(self): """Test age with items containing differing timezones""" data = [ {"a": 2, "b": "2018-01-30 16:28:40+00:00"}, {"a": 1, "b": "2018-01-30 15:12:35+00:00"}, {"a": 3, "b": "2018-01-10 15:12:35+00:00"}, {"a": 5, "b": "2018-01-30 16:06:59+03:00"}, ] cmd = ( "map({id: x.a, date: x.b})," + "sorted(age(.date), reverse=True), map(.id)" ) expected = "[3, 5, 1, 2]" result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_hide_many(self): """Test complex query""" data = [ {"a": 1, "b": {"c": 632, "d": [1, 2, 3, 4]}}, {"a": 2, "f": 4, "b": {"c": 632, "d": [1, 2, 3, 4]}}, ] cmd = ( 'map({"id": x.a, "c": x.b.c, "data": x.b.d, "x": x["f"]}),' + 'hide("data", "c")' ) expected = '[{"id": 1, "x": null}, {"id": 2, "x": 4}]' result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_first(self): """Test fetching first items""" data = [ {"a": 2, "b": "2018-01-30 16:28:40+00:00"}, {"a": 1, "b": "2018-01-30 15:12:35+00:00"}, {"a": 3, "b": "2018-01-10 15:12:35+00:00"}, {"a": 5, "b": "2018-01-30 16:06:59+03:00"}, ] cmd = ( "map({id: x.a, date: x.b})," + "sorted(age(.date), reverse=True), map(.id), first()" ) expected = "[3]" result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_age_output(self): """ $ cat test.yaml | jf --yamli 'update({id: x.sha, age: age(x.commit.author.date)}), filter(x.age < age("1 days"))' --indent=2 --yaml """ data = [ {"a": 2, "b": "2018-01-30 16:28:40+00:00"}, {"a": 1, "b": "2018-01-30 15:12:35+00:00"}, ] cmd = "map({id: x.a, date: x.b}), sorted(age(.date), reverse=True), first()" expected = '[{"date": "2018-01-30 15:12:35+00:00", "id": 1}]' result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def test_age(self): """ $ cat test.yaml | jf --yamli 'update({id: x.sha, age: age(x.commit.author.date)}), filter(x.age < age("1 days"))' --indent=2 --yaml """ data = [ {"a": 2, "b": "2018-01-30 16:28:40+00:00"}, {"a": 1, "b": "2018-01-30 15:12:35+00:00"}, {"a": 3, "b": "2018-01-10 15:12:35+00:00"}, {"a": 5, "b": "2018-01-30 16:06:59+00:00"}, ] cmd = ( "map({id: x.a, date: x.b})," + "sorted(age(.date), reverse=True)," + "map(.id)" ) expected = "[3, 1, 5, 2]" result = tolist(list(jf.run_query(cmd, data))) self.assertEqual(result, expected)
def main(args=None): """Main JF execution function""" parser = argparse.ArgumentParser() parser.add_argument( "query", nargs="?", default="I", help="query string for extracting wanted information", ) parser.add_argument("-d", "--debug", action="store_true", help="print debug messages") parser.add_argument("-c", "--compact", action="store_true") parser.add_argument("--indent", type=int, default=2, help="pretty-printed with given indent level") parser.add_argument("--import_from", help="add path to search imports from") parser.add_argument("--import", help="import custom processing module") parser.add_argument("-l", "--list", action="store_true", help="wrap output to a list") parser.add_argument("-y", "--yaml", action="store_true", help="output yaml") parser.add_argument("--yamli", action="store_true", help="input yaml") parser.add_argument("-j", "--json", action="store_true", help="output json") parser.add_argument("-s", "--sort_keys", action="store_true", help="sort json output values") parser.add_argument("--bw", action="store_true", default=False, help="remove colors") parser.add_argument("--ordered_dict", action="store_true", default=False, help="user ordered dict") parser.add_argument("-r", "--raw", action="store_true", default=False, help="raw output") parser.add_argument("-f", "--cmdfile", help="read command from file") parser.add_argument( "-a", "--ensure_ascii", action="store_true", default=False, help="ensure ascii only characters", ) parser.add_argument("--ipyfake", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--forcecolor", action="store_true", help=argparse.SUPPRESS) parser.add_argument("-p", "--ipy", action="store_true", help="start IPython shell with data") parser.add_argument( "--html_unescape", action="store_true", default=False, help="unescape html entities", ) parser.add_argument( "-i", "--input", metavar="FILE", help="files to read. Overrides files argument list", ) parser.add_argument("-k", "--kwargs", help="files to read. Overrides files argument list") parser.add_argument( "files", metavar="FILE", nargs="*", default="-", help="files to read, if empty, stdin is used", ) args = parser.parse_args(args) if args.input is not None: args.files = [args.input] if args.indent < 0: args.indent = None if args.compact: args.indent = None set_loggers(args.debug) kwargs = {} if args.kwargs: import re kwargsre = re.compile(r"([^:,]+)") kwargs = kwargsre.subn(r'"\1"', args.kwargs.replace("=", ":"))[0] kwargs = "{%s}" % kwargs kwargs = json.loads(kwargs) inq = read_input(args, ordered_dict=args.ordered_dict, **kwargs) imports = None if "import" in args.__dict__: imports = args.__dict__["import"] if args.cmdfile is None: query = args.query else: query = ", ".join( filter( lambda x: len(x) and x[0] != "#", open(args.cmdfile, "r").read().split("\n"), )) new_imports = list( map( lambda x: x.split()[1], filter( lambda x: len(x) and x.startswith("#import "), open(args.cmdfile, "r").read().split("\n"), ), )) if len(new_imports): if imports is None: imports = [] else: imports = imports.split(",") imports = ",".join(imports + new_imports) if args.query == "": query = "I" data = run_query( query, inq, imports=imports, import_from=args.import_from, ordered_dict=args.ordered_dict, ) if args.ipy or args.ipyfake: banner = "" if not sys.stdin.isatty(): banner = ("\nNotice: You are inputting data from stdin!\n" + "This might cause some trouble since jf will try " + "to get some input from you also.\n" + "To get the full benefit of jf and IPython, " + "consider giving the input as a file instead.\n\n" + "To prevent any unexpected behaviour, jk will " + "load the full dataset in memory.\n" + "This might take a while...\n") data = list(data) try: sys.stdin = open("/dev/tty") except OSError: pass ipy(banner, data, args.ipyfake) return try: print_results(data, args) except FileNotFoundError as ex: logger.warning("%s", ex)
def test_math_query(self): data = [{"a": "123"}] query = 'update({y: math.log(len(.a) * len(.a))}), .y' self.assertEqual(list(jf.run_query(query, data, "math")), [math.log(3*3)])
def test_simple_query(self): """Test simple query""" data = [{"a": 1}] self.assertEqual(tolist(jf.run_query("map(x.a)", data)), "[1]")
def test_get_null_item(self): """Test simple query""" data = [{"a": 1}] self.assertEqual(tolist(jf.run_query('map(x["b"])', data)), "[null]")