def readJSON(filename, default): try: newfile = open(filename, "r") json = JSLON.parse(newfile.read(), {"dict": odict}) newfile.close() return json except OSError: return default
def read(self): """ Read from JSON data file. """ self.base = JSLON.parse(helper.readFrom(self.path), { "dict": JSONDict, "list": JSONList, "string": rpl.String, "int": rpl.Number, "float": JSONFile.invalid, "regex": JSONFile.invalid, "undefined": JSONFile.invalid, })
def writeJSON(filename, json): newfile = open(filename, "w") newfile.write(JSLON.stringify(json, jslonFormat)) newfile.close()
def main(args): parser = argparse.ArgumentParser( description="Manage Megaman PET chip data files.", add_help=False, ) parser.add_argument("-a", "--generate-adv", action="store_true", help="Generate any missing JSON files in Advance PET format using images in CWD.") parser.add_argument("-j", "--translate-json", default="", help="Translate JSON structure to appropriate chip structures. See -h -j for details.") parser.add_argument("-m", "--merge-mode", default="new", help="When translating, specify the mode for merging; either new or overwrite.") parser.add_argument("-q", "--query-files", action="store_true", help="Query a set of JSON files using translation syntax. You may pass any number of files or a regex prefixed with re:") parser.add_argument("-s", "--sort-objects", default="", help="Sort object keys according to given list of given files/regex (see -q).") parser.add_argument("-u", "--updated", default="", help="Write out when the files/regex (see -q) were last updated to the given file.") parser.add_argument("-h", "-?", "--help", action="store_true", help=argparse.SUPPRESS) parser.add_argument("files", nargs="*", help=argparse.SUPPRESS) args = parser.parse_args(args) if args.help: if args.generate_adv: print( "Image names are expected to be of the form #-Name.xxx\n" "The extension is ignored except that JSON files are filtered out." ) elif args.translate_json: print( "Create a file with JSON formatted data contained in an object.\n" "This object must contain a _format key which describes the transformation." "This key must contain an object that has the form " "\"key regex\": [filename transformation, transformation] " "or \"key regex\": { \"_filename\": transformation, transformations... }\n" "where transformation is: an object whose keys are literal and whose " "values are transformations, a literal value (number, boolean, or null), " "or a string that describes the transformation. A literal string can be represented " "as \"'string contents'\"\n" "The syntax allows for indexing, comparisons, and mapping.\n" "Indexing is a sequence of symbols separated by colons.\n" "The symbol 'data' is a reserved word that refers to the data in the matched key.\n" "You may refer to groups in the key with $#. $0 is the key itself.\n" "If the first symbol is a word, it refers to a key in the struct you're transforming.\n" "Comparisons may use: ==, !=, is\n" "Mapping is of the form: symbol { from => to, ... }\n" ) else: parser.print_help() #endif return 0 #endif # Use the provided file(s)/regex or request one. files, regexs = [], [] if args.files: for x in args.files: if x[0:3] != "re:": files.append((x, None)) else: regexs.append(re.compile(x[3:] + "$")) elif args.query_files or args.sort_objects or args.updated: regexs.append(re.compile(input("Enter a file regex: ") + "$")) #endif if regexs: for root, dirs, fns in os.walk("."): root = root[2:] # delete ./ for fn in fns: fn = os.path.join(root, fn) for r in regexs: mo = r.match(fn) if mo: files.append((fn, mo)) break #endif #endfor #endfor #endfor #endif files = sorted(list(set(files))) if args.generate_adv: en = re.compile("^[a-zA-Z0-9 \-+]+$") cp = re.compile("^CP [0-9]+$") blank = re.compile("^\(blank\)$") isIcon = re.compile("^icon$") isBack = re.compile("^(.*) \(back\)$") createable = {} for fn in os.listdir("."): # Make sure this isn't a JSON file. if fn[-5:].lower() == ".json": continue # Now check if it's a chip image. if "." not in fn: continue try: num, name = fn.split("-") int(num) name, ext = name.rsplit(".", 1) if name[-6:] == " (bad)": name = name[:-6] except ValueError: continue if num not in createable: createable[num] = [] createable[num].append((name, fn)) #endfor for num, names in createable.items(): # Okay! Create/update the JSON file. filename, updated = "%s.json" % num, False json = readJSON(filename, odict([ ("type", None), ("class", None), ("cp", None), ("at", None), ("element", None), ("field", None), ("effect", None), ("icon", None), ("pins", None), ("notes", ""), ("releases", {}), ("new", True) ])) if "new" in json: updated = True del json["new"] #endif for name, fn in names: # What language is this name? Can assume "CP", blank, EN, or JP. back = isBack.match(name) if back: name = back.group(1) if cp.match(name): language = "cp" json["cp"] = int(name[3:]) elif blank.match(name): language = "blank" elif isIcon.match(name): if json["icon"] != fn: updated = True json["icon"] = fn continue elif en.match(name): language = "en" else: language = "jp" #endif if language not in json["releases"]: json["releases"][language] = odict([ ("name", name), ("set", None), ("front", { "image": None, "credits": None, }), ("back", { "image": None, "credits": None, }), ("official", True), ("notes", ""), ]) updated = True #endif ref = json["releases"][language]["back" if back else "front"] if ref["image"] != fn: updated = True ref["image"] = fn #endif #endfor if updated: print("Updated: " + filename) writeJSON(filename, json) #endif #endfor elif args.translate_json: json = readJSON(f.read()) form = [] for x, t in json["_format"].items(): try: "_filename" in t except TypeError: if len(t) != 2: return error("Filename not provided for key: " + x, 2) else: t = [t["_filename"], t] del t[1]["_filename"] #endtry t = (list(transformationFormat.finditer(t[0])), t[1]) form.append((re.compile("^" + x + "$"), t)) #endfor del json["_format"] for key, data in json.items(): doBreak = False for r, (fn, t) in form: mo = r.match(key) if mo: try: # Get the filename fn = parseTransformation(fn, json, mo, data, False) except SyntaxError as err: return error("Error processing filename for %s: %s" % (r.pattern[1:-1], err.args[0]), 3) #endtry result = readJSON(fn, odict()) try: # Perform this translation obj = recurseObject(t, json, mo, data) except SyntaxError as err: return error("Error processing data for %s: %s" % (r.pattern[1:-1], err.args[0]), 4) #endtry def recursiveMergeOverwrite(json, obj, path=""): for k, v in obj.items(): if k in json and json[k]: newPath = (path + "." if path else "") + k try: json[k].items except AttributeError: error("Overwriting %s: %s" % (newPath, json[k])) json[k] = v else: recursiveMergeOverwrite(json[k], v, newPath) else: json[k] = v #endif #endfor #enddef def recursiveMergeNew(json, obj): for k, v in obj.items(): if k not in json or json[k] is None: json[k] = v elif json[k] == "" and type(v) is str: json[k] = v else: try: json[k].items except AttributeError: pass else: recursiveMergeNew(json[k], v) #endif #endfor #enddef recursiveMerge = { "new": recursiveMergeNew, "over": recursiveMergeOverwrite, "overwrite": recursiveMergeOverwrite, } try: recursiveMerge[args.merge_mode] except KeyError: return error("Merge mode not found.", 17) else: recursiveMerge[args.merge_mode](result, obj) writeJSON(fn, result) print("Wrote file " + fn) break #endif #endfor #endfor elif args.query_files: if not files: return error("No files found or selected.", 6) full = odict() for x, mo in files: root, ext = os.path.splitext(x) full[root] = (readJSON(x), mo) #endfor wSet, wData, trace = None, None, None while True: try: cmds = input("cmd: ").split("then") for cmdIdx, cmd in enumerate(cmds): if not cmd: continue cmd, *arg = cmd.split(maxsplit=1) cmd = cmd.lower() if arg: arg = arg[0].strip() if cmd in ["help", "h", "?"]: print( "Query system help\n" "Commands:\n" " * exit - Quit querying\n" " * grab - Add list of space-separated filenames to the set." " * find - Enter an expression that returns a boolean, finds all entries (returns filename)\n" " * summ - Enter an expression, returns result for all entires\n" " * info - Enter a filename, displays all info. When linked, no argument is given\n" " * stat - Show current counts: files, matched set, summed data\n" " * clr - Unset current links: all (default), set, or data\n" " * has - Enter an index, finds all files that have this index.\n" " * cmpl - Inverts matched set.\n" " * set - Enter an index and an expression, updates all matched or all files.\n" " * del - Enter an index, deletes from all matched or all files.\n" " * save - Commit all matched or all files to disk.\n" " * load - Reload all matched or all files from disk.\n" "Command Suffixes:\n" " * nothing - Print the result and clear any links.\n" " * * - Links the result with the next command.\n" " * ! - Print the result but don't clear the previous link.\n" " * *! - Don't print or clear anything.\n" "One-line Joiners:\n" " * then - Acts as * on one line." ) elif cmd in ["exit", "quit"]: return 0 elif cmd in ["clear", "clr"]: wSet, wData = None, None elif cmd: cont, carry = False, False if cmd[-1] == "*": cmd, cont = cmd[:-1], True if cmd[-1] == "!": cmd, carry = cmd[:-1], True if cmdIdx < len(cmds) - 1: cont = True doprint = not cont if cmd == "find": nSet = [] if wData is not None: for fn, data in wData.items(): res = parseTransformation(arg, full[fn][0], full[fn][1], data) if type(res) is not bool: error("Result of %s was not bool.") elif res: nSet.append(fn) #endfor if not carry: wData = None elif wSet is not None: for fn in wSet: res = parseTransformation(arg, full[fn][0], full[fn][1], None) if type(res) is not bool: error("Result of %s was not bool.") elif res: nSet.append(fn) #endfor if not carry: wSet = None else: for fn, (json, key) in full.items(): res = parseTransformation(arg, json, key, None) if type(res) is not bool: error("Result of %s was not bool.") elif res: nSet.append(fn) #endfor #endif if cont and not carry: wSet = nSet elif doprint: for fn in nSet: print("%s" % fn) #endif elif cmd == "info": if wSet is not None: for fn in wSet: print("=============== %s ===============" % fn) print(JSLON.stringify(full[fn][0], jslonFormat) + "\n") #endfor if not cont or carry: wSet = None else: if arg not in full: error("Filename does not exist (did you add .json??)") continue print(JSLON.stringify(full[arg][0], jslonFormat) + "\n\n") #endif elif cmd == "summ": nData = odict() if wData is not None: for fn, data in wData.items(): res = parseTransformation(arg, full[fn][0], full[fn][1], data) if res is not None: nData[fn] = res #endfor if not carry: wData = None elif wSet is not None: for fn in wSet: res = parseTransformation(arg, full[fn][0], full[fn][1], None) if res is not None: nData[fn] = res #endfor if not carry: wSet = None else: for fn, (json, key) in full.items(): res = parseTransformation(arg, json, key, None) if res is not None: nData[fn] = res #endfor #endif if cont and not carry: wData = nData elif doprint: for item in nData.items(): print("%s: %s" % item) #endif elif cmd in ["clr", "clear"]: arg = arg.lower() if arg == "data": wData = None elif arg == "set": wSet = None elif arg == "all": wData, wSet = None, None elif cmd == "test": fn, trans = arg.split(maxsplit=1) json, key = full[fn] print(parseTransformation(trans, wSet or json, key, wData)) elif cmd == "stat": print("file count: %i; set count: %s; data count: %s" % ( len(full), "null" if wSet is None else len(wSet), "null" if wData is None else len(wData), )) elif cmd == "has": nSet = [] if wData is not None: for fn, data in wData.items(): if hasIndexing(full[fn][0], full[fn][1], data, arg): nSet.append(fn) #endif #endfor if not carry: wData = None elif wSet is not None: for fn in wSet: if hasIndexing(full[fn][0], full[fn][1], None, arg): nSet.append(fn) #endif #endfor if not carry: wSet = None else: for fn, (json, key) in full.items(): if hasIndexing(json, key, None, arg): nSet.append(fn) #endif #endfor #endif if cont and not carry: wSet = nSet elif doprint: for fn in nSet: print("%s" % fn) #endif elif cmd in ["not", "neg", "cmpl"]: nSet = [] if wSet: for fn in full.keys(): if fn not in wSet: nSet.append(fn) #endfor if not carry: wSet = None else: error("No matched set.") #endif if cont and not carry: wSet = nSet elif doprint: for fn in nSet: print("%s" % fn) #endif elif cmd == "del": if wData is not None: for fn, data in wData.items(): deleteIndexing(full[fn][0], full[fn][1], data, arg) #endfor if not carry: wData = None elif wSet is not None: for fn in wSet: deleteIndexing(full[fn][0], full[fn][1], None, arg) #endfor if not carry: wSet = None else: for fn, (json, key) in full.items(): deleteIndexing(json, key, None, arg) #endfor #endif elif cmd == "set": index, trans = arg.split(maxsplit=1) updata = index == "data" if wData is not None: for fn, data in wData.items(): res = parseTransformation(trans, full[fn][0], full[fn][1], data) if updata: wData[fn] = res else: updateIndexing(full[fn][0], full[fn][1], data, index, res) if doprint: print("%s: %s = %s" % (fn, index, repr(res))) #endfor if not carry and not updata: wData = None elif wSet is not None: if updata: wData = odict() for fn in wSet: res = parseTransformation(trans, full[fn][0], full[fn][1], None) if updata: wData[fn] = res else: updateIndexing(full[fn][0], full[fn][1], None, index, res) if doprint: print("%s: %s = %s" % (fn, index, repr(res))) #endforindex if not carry: wSet = None else: if updata: wData = odict() for fn, (json, key) in full.items(): res = parseTransformation(trans, json, key, None) if updata: wData[fn] = res else: updateIndexing(json, key, None, index, res) if doprint: print("%s: %s = %s" % (fn, index, repr(res))) #endfor #endif elif cmd in ["obj", "+obj"]: updata = arg == "data" if wData: for fn, data in wData.items(): if updata: wData[fn] = {} else: updateIndexing(full[fn][0], full[fn][1], data, arg, {}) if doprint: print("%s: %s = {}" % (fn, arg)) #endfor if not carry and not updata: wData = None elif wSet: if updata: wData = odict() for fn in wSet: if updata: wData[fn] = res else: updateIndexing(full[fn][0], full[fn][1], None, arg, {}) if doprint: print("%s: %s = {}" % (fn, arg)) #endforindex if not carry: wSet = None else: if updata: wData = odict() for fn, (json, key) in full.items(): if updata: wData[fn] = res else: updateIndexing(json, key, None, arg, {}) if doprint: print("%s: %s = {}" % (fn, arg)) #endfor #endif elif cmd == "save": if wData: it = wData.keys() elif wSet: it = wSet else: it = full.keys() for fn in it: filename = full[fn][1].group(0) writeJSON(filename, "w", full[fn][0]) if doprint: print("%s: Saved to %s" % (fn, filename)) #endfor if not carry: wData = None wSet = None #endif elif cmd == "load": # TODO: load new files if wData: it = wData.keys() elif wSet: it = wSet else: it = full.keys() for fn in it: filename = full[fn][1].group(0) full[fn] = (readJSON(filename), full[fn][1]) if doprint: print("%s: Loaded from %s" % (fn, filename)) #endfor elif cmd == "grab": if wSet is None: wSet = [] for fn in arg.split(): if fn in full: wSet.append(fn) else: error("No file %s." % fn) #endfor wSet = list(sorted(set(wSet))) elif cmd == "tb": traceback.print_tb(trace) else: error("Unknown command %s" % cmd) if cmdIdx < len(cmds) - 1: print("Canceling execution.") break #endif #endif #endfor except Exception as err: error("Command raised exception %s: %s" % (err.__class__.__name__, ' '.join(map(str, err.args)))) trace = sys.exc_info()[2] #endtry #endwhile elif args.sort_objects: if not files: return error("No files found or selected.", 6) order = args.sort_objects.split(",") warned = set() def sorter(x): x = x[0] try: return order.index(x) except ValueError: if x not in warned: error("No sort order defined for key " + x) warned.add(x) #endif return len(order) #endtry #enddef def recurse(x): x = sorted(x.items(), key=sorter) for i, (k, v) in enumerate(x): try: v.items except AttributeError: pass else: x[i] = (k, recurse(v)) #endfor return odict(x) #enddef for fn, key in files: json = openJSON(fn, odict()) writeJSON(fn, recurse(json)) #endfor elif args.updated: if not files: return error("No files found or selected.", 6) json = odict() for fn, key in files: stat = os.stat(fn) json[(key.group(1, None) if key and len(key.groups()) > 1 else None) or fn] = int(stat.st_mtime) #endfor writeJSON(args.updated, json) #endif return 0
def main(): f = codecs.open("../test.jslon", encoding="utf-8", mode="r") data = f.read() f.close() now = time.time() root = JSLON.parse(data) print "Parse time: %.3f s" % (time.time() - now) print "====== parse ======" for x in root: if x == "literal": comp(x, u"literal", root[x]) elif x == "double": comp(x, u"double", root[x]) elif x == "single": comp(x, u"single", root[x]) elif x == "line continueheeee": comp(x, u"line continueheeee", root[x]) elif x == "inline hex": comp(x, u"inline hex", root[x]) elif x == "str1": comp(x, u"double quotes", root[x]) elif x == "str2": comp(x, u"single quotes", root[x]) elif x == "str3": comp(x, u"linecontinue", root[x]) elif x == "str4": comp(x, u"escapes\x01\x02\x40 \x200", root[x]) elif x == "num1": comp(x, 123, root[x]) elif x == "num2": comp(x, 1.1, root[x]) elif x == "num3": comp(x, 0, root[x]) elif x == "num4": comp(x, 0.1, root[x]) elif x == "num5": comp(x, 1, root[x]) elif x == "num6": comp(x, 1e10, root[x]) elif x == "num7": comp(x, 1e-5, root[x]) elif x == "num8": comp(x, -15, root[x]) elif x == "hex1": comp(x, 1, root[x]) elif x == "hex2": comp(x, 10, root[x]) elif x == "oct1": comp(x, 7, root[x]) elif x == "oct2": comp(x, 8, root[x]) elif x == "oct3": comp(x, 40, root[x]) elif x == "regex": comp(x, u"123/a\\n%%?", root[x].pattern) elif x == "arr1": comp(x + "[0]", u"hi", root[x][0]) comp(x + "[1]", 1, root[x][1]) comp(x + "[2]", 2, root[x][2]) comp(x + "[3]", 3, root[x][3]) comp(x + "[4]", u"regex", root[x][4].pattern) comp(x + "[5].hi", u"hi", root[x][5]["hi"]) comp(x + "[6][0]", 1, root[x][6][0]) comp(x + "[6][1]", 2, root[x][6][1]) comp(x + "[7]", 5, root[x][7]) elif x == "arr2": comp(x + "[0]", 1, root[x][0]) comp(x + "[1]", 2, root[x][1]) comp(x + "[2]", u"c", root[x][2]) comp(x + "[3]", 4, root[x][3]) elif x == "obj": comp(x + ".abc", u"def", root[x]["abc"]) elif x == "Infinity": comp(x, float, type(root[x])) comp(x, "inf", str(root[x])) elif x == "NaN": comp(x, float, type(root[x])) comp(x, "nan", str(root[x])) elif x == "true": comp(x, True, root[x]) elif x == "false": comp(x, False, root[x]) elif x == "null": comp(x, None, root[x]) elif x == "undefined": comp(x, JSLON.undefined(), root[x]) elif x == "dup": comp(x, 2, root[x]) #endfor print "\n====== stringify ======" comp("Number", u"687123", JSLON.stringify(687123)) comp("Floating point number", u"10.4", JSLON.stringify(10.4)) comp("String", u'"abc"', JSLON.stringify("abc")) comp("String with escapes", u'"abc\\nd\\u0005"', JSLON.stringify("abc\nd\x05")) comp("Array of numbers", u"[1,2,3,4]", JSLON.stringify([1, 2, 3, 4])) comp("Array of arrays", u"[1,[2,[3]],4,[5,[[6,7],8]]]", JSLON.stringify([1, [2, [3]], 4, [5, [[6, 7], 8]]])) comp("Object of strings", u'{"a":"b","c":"d"}', JSLON.stringify({"a": 'b', "c": 'd'})) print "\n====== stringify with options ======" comp("Octal number", u"01", JSLON.stringify(1, {"numBase": 8})) comp("Hexadecimal number", u"0x1", JSLON.stringify(1, {"numBase": 16})) comp("Single quoted string", u"'abc'", JSLON.stringify("abc", {"quotes": "'"})) comp("String (strEscapes: cOu)", u'"abc\\nd\\005a\\u0999"', JSLON.stringify(u"abc\nd\x05a\u0999", {"strEscapes": "cOu"}) ) comp("String (strEscapes: ou)", u'"abc\\12d\\5a\\u0999"', JSLON.stringify(u"abc\nd\x05a\u0999", {"strEscapes": "ou"}) ) comp("String (strEscapes: x)", u'"abc\\x0ad\\x05a"', JSLON.stringify(u"abc\nd\x05a\u0999", {"strEscapes": "x"}) ) comp("Prettyprinted object (keyOwnLine|openOwnLine|endOwnLine, specific)", u'{\n\t"a":\n\t\t[\n\t\t\t0x1,2,{\n\t\t\t\t"b":\n\t\t\t\t\t"abc"\n\t\t\t},\n\t\t\t3\n\t\t]\n}', JSLON.stringify({"a": [1, 2, {"b": "abc"}, 3]},{ "keyOwnLine": True, "openOwnLine": True, "endOwnLine": True, "specific": {"a": {"specific": [{"numBase": 16}]}}, }) ) comp("Prettyprinted object (spaceAfterKey)", u'{"a": "b"}', JSLON.stringify({"a":"b"}, {"spaceAfterKey": True}) ) comp("Prettyprinted array (entriesPerLine, depth)", u'[1,2,\n 3,4,\n 5]', JSLON.stringify([1, 2, 3, 4, 5], {"entriesPerLine": 2, "depth": " "}) )