def main(): usage = "Usage: %prog [options] JSONFILE" parser = OptionParser(usage=usage) parser.add_option("-w", "--write", dest="write", action="store_true", help="write to the Tango DB", metavar="WRITE") parser.add_option("-u", "--update", dest="update", action="store_true", help="don't remove things, only add/update", metavar="UPDATE") parser.add_option("-c", "--case-sensitive", dest="case_sensitive", action="store_true", help=("Don't ignore the case of server, device, " "attribute and property names"), metavar="CASESENSITIVE") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print actions to stderr") parser.add_option("-o", "--output", dest="output", action="store_true", help="Output the relevant DB state as JSON.") parser.add_option("-p", "--input", dest="input", action="store_true", help="Output the input JSON (after filtering).") parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true", help="print out all db calls.") parser.add_option("-v", "--no-validation", dest="validate", default=True, action="store_false", help=("Skip JSON validation")) parser.add_option("-s", "--sleep", dest="sleep", default=0.01, type="float", help=("Number of seconds to sleep between DB calls")) parser.add_option("-n", "--no-colors", action="store_true", dest="no_colors", default=False, help="Don't print colored output") parser.add_option("-i", "--include", dest="include", action="append", help=("Inclusive filter on server configutation")) parser.add_option("-x", "--exclude", dest="exclude", action="append", help=("Exclusive filter on server configutation")) parser.add_option("-I", "--include-classes", dest="include_classes", action="append", help=("Inclusive filter on class configuration")) parser.add_option("-X", "--exclude-classes", dest="exclude_classes", action="append", help=("Exclusive filter on class configuration")) parser.add_option( "-D", "--dbdata", help="Read the given file as DB data instead of using the actual DB", dest="dbdata") options, args = parser.parse_args() if options.no_colors: no_colors() if len(args) == 0: data = load_json(sys.stdin) else: json_file = args[0] with open(json_file) as f: data = load_json(f) # Normalization - making the config conform to standard data = normalize_config(data) # remove any metadata at the top level (should we use this for something?) data = clean_metadata(data) # Optional validation of the JSON file format. if options.validate: validate_json(data) # filtering try: if options.include: data["servers"] = filter_config( data.get("servers", {}), options.include, SERVERS_LEVELS) if options.exclude: data["servers"] = filter_config( data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True) if options.include_classes: data["classes"] = filter_config( data.get("classes", {}), options.include_classes, CLASSES_LEVELS) if options.exclude_classes: data["classes"] = filter_config( data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS, invert=True) except ValueError as e: print >>sys.stderr, red("Filter error:\n%s" % e) sys.exit(ERROR) if not any(k in data for k in ("devices", "servers", "classes")): sys.exit(ERROR) if options.input: print json.dumps(data, indent=4) return # check if there is anything in the DB that will be changed or removed db = PyTango.Database() if options.dbdata: with open(options.dbdata) as f: original = json.loads(f.read()) collisions = {} else: original = get_db_data(db, dservers=True) devices = CaselessDictionary({ dev: (srv, inst, cls) for srv, inst, cls, dev in get_devices_from_dict(data["servers"]) }) orig_devices = CaselessDictionary({ dev: (srv, inst, cls) for srv, inst, cls, dev in get_devices_from_dict(original["servers"]) }) collisions = {} for dev, (srv, inst, cls) in devices.items(): if dev in orig_devices: server = "{}/{}".format(srv, inst) osrv, oinst, ocls = orig_devices[dev] origserver = "{}/{}".format(osrv, oinst) if server.lower() != origserver.lower(): collisions.setdefault(origserver, []).append((ocls, dev)) # get the list of DB calls needed dbcalls = configure(data, original, update=options.update, ignore_case=not options.case_sensitive) # Print out a nice diff if options.verbose: show_actions(original, dbcalls) # perform the db operations (if we're supposed to) if options.write and dbcalls: for i, (method, args, kwargs) in enumerate(dbcalls): if options.sleep: time.sleep(options.sleep) if options.verbose: progressbar(i, len(dbcalls), 20) getattr(db, method)(*args, **kwargs) print # optionally dump some information to stdout if options.output: print json.dumps(original, indent=4) if options.dbcalls: print >>sys.stderr, "Tango database calls:" for method, args, kwargs in dbcalls: print >>sys.stderr, method, args # Check for moved devices and remove empty servers empty = set() for srvname, devs in collisions.items(): if options.verbose: srv, inst = srvname.split("/") for cls, dev in devs: print >>sys.stderr, red("MOVED (because of collision):"), dev print >>sys.stderr, " Server: ", "{}/{}".format(srv, inst) print >>sys.stderr, " Class: ", cls if len(db.get_device_class_list(srvname)) == 2: # just dserver empty.add(srvname) if options.write: db.delete_server(srvname) # finally print out a brief summary of what was done if dbcalls: print print >>sys.stderr, "Summary:" print >>sys.stderr, "\n".join(summarise_calls(dbcalls, original)) if collisions: servers = len(collisions) devices = sum(len(devs) for devs in collisions.values()) print >>sys.stderr, red("Move %d devices from %d servers." % (devices, servers)) if empty and options.verbose: print >>sys.stderr, red("Removed %d empty servers." % len(empty)) if options.write: print >>sys.stderr, red("\n*** Data was written to the Tango DB ***") with NamedTemporaryFile(prefix="dsconfig-", suffix=".json", delete=False) as f: f.write(json.dumps(original, indent=4)) print >>sys.stderr, ("The previous DB data was saved to %s" % f.name) sys.exit(CONFIG_APPLIED) else: print >>sys.stderr, yellow( "\n*** Nothing was written to the Tango DB (use -w) ***") sys.exit(CONFIG_NOT_APPLIED) else: print >>sys.stderr, green("\n*** No changes needed in Tango DB ***") sys.exit(SUCCESS)
def show_actions(data, calls): "Print out a human readable representation of changes" changes = get_changes(data, calls) # Go through all the devices that have been touched # and print out a representation of the changes. # TODO: is there a more reasonable way to sort this? indent = " " * 2 for device in sorted(changes["devices"]): info = changes["devices"][device] if info.get("added"): if info.get("old_server"): print("{} Device: {}".format(yellow("="), device)) else: print("{} Device: {}".format(green("+"), device)) elif info.get("deleted"): print("{} Device: {}".format(red("-"), device)) else: print("{} Device: {}".format(yellow("="), device)) if info.get("server"): if info.get("old_server"): print("{}Server: {} -> {}".format(indent, red(info["old_server"]), green(info["server"]))) else: print("{}Server: {}".format(indent, info["server"])) if info.get("device_class"): if info.get("old_class"): if info["old_class"] != info["device_class"]: print("{}Class: {} -> {}".format(indent, info["old_class"], info["device_class"])) else: print("{}Class: {}".format(indent, info["device_class"])) if info.get("alias"): alias = info.get("alias").get("value") old_alias = info.get("alias").get("old_value") if old_alias: if old_alias != alias: print("{}Alias: {} -> {}".format(indent, red(old_alias), green(alias))) else: print("{}Alias: {}".format(indent, alias)) if info.get("properties"): lines = [] for prop, change in sorted(info.get("properties", {}).items()): if change.get("value"): value = change.get("value") old_value = change.get("old_value") if old_value is not None: if old_value != value: # change property lines.append( yellow("{}= {}".format(indent * 2, prop))) lines.append( property_diff(change["old_value"], change["value"], indent * 3)) else: # new property lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append( green(format_property(change["value"], indent * 3))) else: # delete property lines.append(red("{}- {}".format(indent * 2, prop))) lines.append( red(format_property(change["old_value"], indent * 3))) if lines: print("{}Properties:".format(indent * 1)) print("\n".join(lines)) if info.get("attribute_properties"): lines = [] for attr, props in sorted( info.get("attribute_properties", {}).items()): attr_lines = [] for prop, change in sorted(props.items()): value = change.get("value") old_value = change.get("old_value") if value is not None: if old_value is not None: # change if value != old_value: attr_lines.append( yellow("{}= {}".format(indent * 3, prop))) attr_lines.append( property_diff(change["old_value"], change["value"], indent * 4)) else: # addition attr_lines.append( green("{}+ {}".format(indent * 3, prop))) attr_lines.append( green( format_property(change["value"], indent * 4))) else: # removal attr_lines.append( red("{}- {}".format(indent * 3, prop))) attr_lines.append( red( format_property(change["old_value"], indent * 4))) if attr_lines: lines.append("{}{}".format(indent * 2, attr)) lines.extend(attr_lines) if lines: print("{}Attribute properties:".format(indent)) print("\n".join(lines)) for clss in sorted(changes["classes"]): info = changes["classes"][clss] if info.get("properties"): lines = [] for prop, change in sorted(info.get("properties", {}).items()): if change.get("value"): value = change.get("value") old_value = change.get("old_value") if old_value is not None: if old_value != value: # change property lines.append( yellow("{}= {}".format(indent * 2, prop))) lines.append( property_diff(change["old_value"], change["value"], indent * 3)) else: # new property lines.append(green("{}+ {}".format(indent * 2, prop))) lines.append( green(format_property(change["value"], indent * 3))) else: # delete property lines.append(red("{}- {}".format(indent * 2, prop))) lines.append( red(format_property(change["old_value"], indent * 3))) if lines: print("{} Class Properties:".format(indent * 1)) print("\n".join(lines)) print
def json_to_tango(options, args): if options.no_colors: no_colors() if len(args) == 0: data = load_json(sys.stdin) else: json_file = args[0] with open(json_file) as f: data = load_json(f) # Normalization - making the config conform to standard data = normalize_config(data) # remove any metadata at the top level (should we use this for something?) data = clean_metadata(data) # Optional validation of the JSON file format. if options.validate: validate_json(data) # filtering try: if options.include: data["servers"] = filter_config( data.get("servers", {}), options.include, SERVERS_LEVELS) if options.exclude: data["servers"] = filter_config( data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True) if options.include_classes: data["classes"] = filter_config( data.get("classes", {}), options.include_classes, CLASSES_LEVELS) if options.exclude_classes: data["classes"] = filter_config( data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS, invert=True) except ValueError as e: print(red("Filter error:\n%s" % e), file=sys.stderr) sys.exit(ERROR) if not any(k in data for k in ("devices", "servers", "classes")): sys.exit(ERROR) if options.input: print(json.dumps(data, indent=4)) return # check if there is anything in the DB that will be changed or removed db = tango.Database() if options.dbdata: with open(options.dbdata) as f: original = json.loads(f.read()) collisions = {} else: original = get_db_data(db, dservers=True, class_properties=True) if "servers" in data: devices = CaselessDictionary({ dev: (srv, inst, cls) for srv, inst, cls, dev in get_devices_from_dict(data["servers"]) }) else: devices = CaselessDictionary({}) orig_devices = CaselessDictionary({ dev: (srv, inst, cls) for srv, inst, cls, dev in get_devices_from_dict(original["servers"]) }) collisions = {} for dev, (srv, inst, cls) in list(devices.items()): if dev in orig_devices: server = "{}/{}".format(srv, inst) osrv, oinst, ocls = orig_devices[dev] origserver = "{}/{}".format(osrv, oinst) if server.lower() != origserver.lower(): collisions.setdefault(origserver, []).append((ocls, dev)) # get the list of DB calls needed dbcalls = configure(data, original, update=options.update, ignore_case=not options.case_sensitive, strict_attr_props=not options.nostrictcheck) # Print out a nice diff if options.verbose: show_actions(original, dbcalls) # perform the db operations (if we're supposed to) if options.write and dbcalls: for i, (method, args, kwargs) in enumerate(dbcalls): if options.sleep: time.sleep(options.sleep) if options.verbose: progressbar(i, len(dbcalls), 20) getattr(db, method)(*args, **kwargs) print() # optionally dump some information to stdout if options.output: print(json.dumps(original, indent=4)) if options.dbcalls: print("Tango database calls:", file=sys.stderr) for method, args, kwargs in dbcalls: print(method, args, file=sys.stderr) # Check for moved devices and remove empty servers empty = set() for srvname, devs in list(collisions.items()): if options.verbose: srv, inst = srvname.split("/") for cls, dev in devs: print(red("MOVED (because of collision):"), dev, file=sys.stderr) print(" Server: ", "{}/{}".format(srv, inst), file=sys.stderr) print(" Class: ", cls, file=sys.stderr) if len(db.get_device_class_list(srvname)) == 2: # just dserver empty.add(srvname) if options.write: db.delete_server(srvname) # finally print out a brief summary of what was done if dbcalls: print() print("Summary:", file=sys.stderr) print("\n".join(summarise_calls(dbcalls, original)), file=sys.stderr) if collisions: servers = len(collisions) devices = sum(len(devs) for devs in list(collisions.values())) print(red("Move %d devices from %d servers." % (devices, servers)), file=sys.stderr) if empty and options.verbose: print(red("Removed %d empty servers." % len(empty)), file=sys.stderr) if options.write: print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr) with NamedTemporaryFile(prefix="dsconfig-", suffix=".json", delete=False) as f: f.write(json.dumps(original, indent=4).encode()) print(("The previous DB data was saved to %s" % f.name), file=sys.stderr) sys.exit(CONFIG_APPLIED) else: print(yellow( "\n*** Nothing was written to the Tango DB (use -w) ***"), file=sys.stderr) sys.exit(CONFIG_NOT_APPLIED) else: print(green("\n*** No changes needed in Tango DB ***"), file=sys.stderr) sys.exit(SUCCESS)