def property_diff(old, new, indentation=""):
diff = ndiff(old, new)
lines = []
for line in diff:
if line.startswith("+"):
lines.append(green("{}{}".format(indentation, line)))
elif line.startswith("-"):
lines.append(red("{}{}".format(indentation, line)))
else:
lines.append("{}{}".format(indentation, line))
return "\n".join(lines)
def show_actions(data, calls):
"Print out a human readable representation of changes"
changes = get_changes(data, calls)
# Go through all the devices that have been touched
# and print out a representation of the changes.
# TODO: is there a more reasonable way to sort this?
indent = " " * 2
for device in sorted(changes["devices"]):
info = changes["devices"][device]
if info.get("added"):
if info.get("old_server"):
print("{} Device: {}".format(yellow("="), device))
else:
print("{} Device: {}".format(green("+"), device))
elif info.get("deleted"):
print("{} Device: {}".format(red("-"), device))
else:
print("{} Device: {}".format(yellow("="), device))
if info.get("server"):
if info.get("old_server"):
print("{}Server: {} -> {}".format(indent,
red(info["old_server"]),
green(info["server"])))
else:
print("{}Server: {}".format(indent, info["server"]))
if info.get("device_class"):
if info.get("old_class"):
if info["old_class"] != info["device_class"]:
print("{}Class: {} -> {}".format(indent, info["old_class"],
info["device_class"]))
else:
print("{}Class: {}".format(indent, info["device_class"]))
if info.get("alias"):
alias = info.get("alias").get("value")
old_alias = info.get("alias").get("old_value")
if old_alias:
if old_alias != alias:
print("{}Alias: {} -> {}".format(indent, red(old_alias),
green(alias)))
else:
print("{}Alias: {}".format(indent, alias))
if info.get("properties"):
lines = []
for prop, change in sorted(info.get("properties", {}).items()):
if change.get("value"):
value = change.get("value")
old_value = change.get("old_value")
if old_value is not None:
if old_value != value:
# change property
lines.append(
yellow("{}= {}".format(indent * 2, prop)))
lines.append(
property_diff(change["old_value"],
change["value"], indent * 3))
else:
# new property
lines.append(green("{}+ {}".format(indent * 2, prop)))
lines.append(
green(format_property(change["value"],
indent * 3)))
else:
# delete property
lines.append(red("{}- {}".format(indent * 2, prop)))
lines.append(
red(format_property(change["old_value"], indent * 3)))
if lines:
print("{}Properties:".format(indent * 1))
print("\n".join(lines))
if info.get("attribute_properties"):
lines = []
for attr, props in sorted(
info.get("attribute_properties", {}).items()):
attr_lines = []
for prop, change in sorted(props.items()):
value = change.get("value")
old_value = change.get("old_value")
if value is not None:
if old_value is not None:
# change
if value != old_value:
attr_lines.append(
yellow("{}= {}".format(indent * 3, prop)))
attr_lines.append(
property_diff(change["old_value"],
change["value"], indent * 4))
else:
# addition
attr_lines.append(
green("{}+ {}".format(indent * 3, prop)))
attr_lines.append(
green(
format_property(change["value"],
indent * 4)))
else:
# removal
attr_lines.append(
red("{}- {}".format(indent * 3, prop)))
attr_lines.append(
red(
format_property(change["old_value"],
indent * 4)))
if attr_lines:
lines.append("{}{}".format(indent * 2, attr))
lines.extend(attr_lines)
if lines:
print("{}Attribute properties:".format(indent))
print("\n".join(lines))
for clss in sorted(changes["classes"]):
info = changes["classes"][clss]
if info.get("properties"):
lines = []
for prop, change in sorted(info.get("properties", {}).items()):
if change.get("value"):
value = change.get("value")
old_value = change.get("old_value")
if old_value is not None:
if old_value != value:
# change property
lines.append(
yellow("{}= {}".format(indent * 2, prop)))
lines.append(
property_diff(change["old_value"],
change["value"], indent * 3))
else:
# new property
lines.append(green("{}+ {}".format(indent * 2, prop)))
lines.append(
green(format_property(change["value"],
indent * 3)))
else:
# delete property
lines.append(red("{}- {}".format(indent * 2, prop)))
lines.append(
red(format_property(change["old_value"], indent * 3)))
if lines:
print("{} Class Properties:".format(indent * 1))
print("\n".join(lines))
print
def main():
usage = "Usage: %prog [options] JSONFILE"
parser = OptionParser(usage=usage)
parser.add_option("-w", "--write", dest="write", action="store_true",
help="write to the Tango DB", metavar="WRITE")
parser.add_option("-u", "--update", dest="update", action="store_true",
help="don't remove things, only add/update",
metavar="UPDATE")
parser.add_option("-c", "--case-sensitive", dest="case_sensitive",
action="store_true",
help=("Don't ignore the case of server, device, "
"attribute and property names"),
metavar="CASESENSITIVE")
parser.add_option("-q", "--quiet",
action="store_false", dest="verbose", default=True,
help="don't print actions to stderr")
parser.add_option("-o", "--output", dest="output", action="store_true",
help="Output the relevant DB state as JSON.")
parser.add_option("-p", "--input", dest="input", action="store_true",
help="Output the input JSON (after filtering).")
parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true",
help="print out all db calls.")
parser.add_option("-v", "--no-validation", dest="validate", default=True,
action="store_false", help=("Skip JSON validation"))
parser.add_option("-s", "--sleep", dest="sleep", default=0.01,
type="float",
help=("Number of seconds to sleep between DB calls"))
parser.add_option("-n", "--no-colors",
action="store_true", dest="no_colors", default=False,
help="Don't print colored output")
parser.add_option("-i", "--include", dest="include", action="append",
help=("Inclusive filter on server configutation"))
parser.add_option("-x", "--exclude", dest="exclude", action="append",
help=("Exclusive filter on server configutation"))
parser.add_option("-I", "--include-classes", dest="include_classes",
action="append",
help=("Inclusive filter on class configuration"))
parser.add_option("-X", "--exclude-classes", dest="exclude_classes",
action="append",
help=("Exclusive filter on class configuration"))
parser.add_option(
"-D", "--dbdata",
help="Read the given file as DB data instead of using the actual DB",
dest="dbdata")
options, args = parser.parse_args()
if options.no_colors:
no_colors()
if len(args) == 0:
data = load_json(sys.stdin)
else:
json_file = args[0]
with open(json_file) as f:
data = load_json(f)
# Normalization - making the config conform to standard
data = normalize_config(data)
# remove any metadata at the top level (should we use this for something?)
data = clean_metadata(data)
# Optional validation of the JSON file format.
if options.validate:
validate_json(data)
# filtering
try:
if options.include:
data["servers"] = filter_config(
data.get("servers", {}), options.include, SERVERS_LEVELS)
if options.exclude:
data["servers"] = filter_config(
data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True)
if options.include_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.include_classes, CLASSES_LEVELS)
if options.exclude_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS,
invert=True)
except ValueError as e:
print >>sys.stderr, red("Filter error:\n%s" % e)
sys.exit(ERROR)
if not any(k in data for k in ("devices", "servers", "classes")):
sys.exit(ERROR)
if options.input:
print json.dumps(data, indent=4)
return
# check if there is anything in the DB that will be changed or removed
db = PyTango.Database()
if options.dbdata:
with open(options.dbdata) as f:
original = json.loads(f.read())
collisions = {}
else:
original = get_db_data(db, dservers=True)
devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(data["servers"])
})
orig_devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(original["servers"])
})
collisions = {}
for dev, (srv, inst, cls) in devices.items():
if dev in orig_devices:
server = "{}/{}".format(srv, inst)
osrv, oinst, ocls = orig_devices[dev]
origserver = "{}/{}".format(osrv, oinst)
if server.lower() != origserver.lower():
collisions.setdefault(origserver, []).append((ocls, dev))
# get the list of DB calls needed
dbcalls = configure(data, original,
update=options.update,
ignore_case=not options.case_sensitive)
# Print out a nice diff
if options.verbose:
show_actions(original, dbcalls)
# perform the db operations (if we're supposed to)
if options.write and dbcalls:
for i, (method, args, kwargs) in enumerate(dbcalls):
if options.sleep:
time.sleep(options.sleep)
if options.verbose:
progressbar(i, len(dbcalls), 20)
getattr(db, method)(*args, **kwargs)
print
# optionally dump some information to stdout
if options.output:
print json.dumps(original, indent=4)
if options.dbcalls:
print >>sys.stderr, "Tango database calls:"
for method, args, kwargs in dbcalls:
print >>sys.stderr, method, args
# Check for moved devices and remove empty servers
empty = set()
for srvname, devs in collisions.items():
if options.verbose:
srv, inst = srvname.split("/")
for cls, dev in devs:
print >>sys.stderr, red("MOVED (because of collision):"), dev
print >>sys.stderr, " Server: ", "{}/{}".format(srv, inst)
print >>sys.stderr, " Class: ", cls
if len(db.get_device_class_list(srvname)) == 2: # just dserver
empty.add(srvname)
if options.write:
db.delete_server(srvname)
# finally print out a brief summary of what was done
if dbcalls:
print
print >>sys.stderr, "Summary:"
print >>sys.stderr, "\n".join(summarise_calls(dbcalls, original))
if collisions:
servers = len(collisions)
devices = sum(len(devs) for devs in collisions.values())
print >>sys.stderr, red("Move %d devices from %d servers." %
(devices, servers))
if empty and options.verbose:
print >>sys.stderr, red("Removed %d empty servers." % len(empty))
if options.write:
print >>sys.stderr, red("\n*** Data was written to the Tango DB ***")
with NamedTemporaryFile(prefix="dsconfig-", suffix=".json",
delete=False) as f:
f.write(json.dumps(original, indent=4))
print >>sys.stderr, ("The previous DB data was saved to %s" %
f.name)
sys.exit(CONFIG_APPLIED)
else:
print >>sys.stderr, yellow(
"\n*** Nothing was written to the Tango DB (use -w) ***")
sys.exit(CONFIG_NOT_APPLIED)
else:
print >>sys.stderr, green("\n*** No changes needed in Tango DB ***")
sys.exit(SUCCESS)
def summarise_calls(dbcalls, dbdata):
"""
A brief summary of the operations performed by a list of DB calls
"""
methods = [
"add_device",
"delete_device",
"put_device_property",
"delete_device_property",
"put_device_attribute_property",
"delete_device_attribute_property",
"put_class_property",
"delete_class_property",
"put_class_attribute_property",
"delete_class_attribute_property",
"put_device_alias",
"delete_device_alias"
]
old_servers = get_servers_from_dict(dbdata)
new_servers = set()
servers = defaultdict(set)
devices = defaultdict(set)
counts = defaultdict(int)
for method, args, kwargs in dbcalls:
if method == "add_device":
info = args[0]
servers[method].add(info.server)
if not info.server.lower() in old_servers:
new_servers.add(info.server)
n = 1
elif "device_property" in method:
n = len(args[1])
devices[method].add(args[0].upper())
elif "attribute_property" in method:
n = sum(len(ps) for attr, ps in args[1].items())
devices[method].add(args[0].upper())
elif "property" in method:
n = len(args[1])
else:
n = 1
counts[method] += n
messages = {
"add_device": (green, "Add %%d devices to %d servers." %
len(servers["add_device"])),
"delete_device": (red, "Delete %d devices."),
"put_device_property": (
yellow, "Add/change %%d device properties in %d devices." %
len(devices["put_device_property"])),
"delete_device_property": (
red, "Delete %%d device properties from %d devices." %
len(devices["delete_device_property"])),
"put_device_attribute_property": (
yellow, "Add/change %%d device attribute properties in %d devices." %
len(devices["put_device_attribute_property"])),
"delete_device_attribute_property": (
red, "Delete %%d device attribute properties from %d devices." %
len(devices["delete_device_attribute_property"])),
"put_class_property": (yellow, "Add/change %d class properties."),
"delete_class_property": (red, "Delete %d class properties."),
"put_class_attribute_property": (
yellow, "Add/change %d class attribute properties"),
"delete_class_attribute_property": (
red, "Delete %d class attribute properties."),
"put_device_alias": (green, "Add/change %d device aliases"),
"delete_device_alias": (red, "Delete %d device aliases")
}
summary = []
if new_servers:
summary.append(green("Add %d servers." % len(new_servers)))
for method in methods:
if method in counts:
color, message = messages[method]
summary.append(color(message % counts[method]))
return summary
def json_to_tango(options, args):
if options.no_colors:
no_colors()
if len(args) == 0:
data = load_json(sys.stdin)
else:
json_file = args[0]
with open(json_file) as f:
data = load_json(f)
# Normalization - making the config conform to standard
data = normalize_config(data)
# remove any metadata at the top level (should we use this for something?)
data = clean_metadata(data)
# Optional validation of the JSON file format.
if options.validate:
validate_json(data)
# filtering
try:
if options.include:
data["servers"] = filter_config(
data.get("servers", {}), options.include, SERVERS_LEVELS)
if options.exclude:
data["servers"] = filter_config(
data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True)
if options.include_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.include_classes, CLASSES_LEVELS)
if options.exclude_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS,
invert=True)
except ValueError as e:
print(red("Filter error:\n%s" % e), file=sys.stderr)
sys.exit(ERROR)
if not any(k in data for k in ("devices", "servers", "classes")):
sys.exit(ERROR)
if options.input:
print(json.dumps(data, indent=4))
return
# check if there is anything in the DB that will be changed or removed
db = tango.Database()
if options.dbdata:
with open(options.dbdata) as f:
original = json.loads(f.read())
collisions = {}
else:
original = get_db_data(db, dservers=True, class_properties=True)
if "servers" in data:
devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(data["servers"])
})
else:
devices = CaselessDictionary({})
orig_devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(original["servers"])
})
collisions = {}
for dev, (srv, inst, cls) in list(devices.items()):
if dev in orig_devices:
server = "{}/{}".format(srv, inst)
osrv, oinst, ocls = orig_devices[dev]
origserver = "{}/{}".format(osrv, oinst)
if server.lower() != origserver.lower():
collisions.setdefault(origserver, []).append((ocls, dev))
# get the list of DB calls needed
dbcalls = configure(data, original,
update=options.update,
ignore_case=not options.case_sensitive,
strict_attr_props=not options.nostrictcheck)
# Print out a nice diff
if options.verbose:
show_actions(original, dbcalls)
# perform the db operations (if we're supposed to)
if options.write and dbcalls:
for i, (method, args, kwargs) in enumerate(dbcalls):
if options.sleep:
time.sleep(options.sleep)
if options.verbose:
progressbar(i, len(dbcalls), 20)
getattr(db, method)(*args, **kwargs)
print()
# optionally dump some information to stdout
if options.output:
print(json.dumps(original, indent=4))
if options.dbcalls:
print("Tango database calls:", file=sys.stderr)
for method, args, kwargs in dbcalls:
print(method, args, file=sys.stderr)
# Check for moved devices and remove empty servers
empty = set()
for srvname, devs in list(collisions.items()):
if options.verbose:
srv, inst = srvname.split("/")
for cls, dev in devs:
print(red("MOVED (because of collision):"), dev, file=sys.stderr)
print(" Server: ", "{}/{}".format(srv, inst), file=sys.stderr)
print(" Class: ", cls, file=sys.stderr)
if len(db.get_device_class_list(srvname)) == 2: # just dserver
empty.add(srvname)
if options.write:
db.delete_server(srvname)
# finally print out a brief summary of what was done
if dbcalls:
print()
print("Summary:", file=sys.stderr)
print("\n".join(summarise_calls(dbcalls, original)), file=sys.stderr)
if collisions:
servers = len(collisions)
devices = sum(len(devs) for devs in list(collisions.values()))
print(red("Move %d devices from %d servers." %
(devices, servers)), file=sys.stderr)
if empty and options.verbose:
print(red("Removed %d empty servers." % len(empty)), file=sys.stderr)
if options.write:
print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr)
with NamedTemporaryFile(prefix="dsconfig-", suffix=".json",
delete=False) as f:
f.write(json.dumps(original, indent=4).encode())
print(("The previous DB data was saved to %s" %
f.name), file=sys.stderr)
sys.exit(CONFIG_APPLIED)
else:
print(yellow(
"\n*** Nothing was written to the Tango DB (use -w) ***"), file=sys.stderr)
sys.exit(CONFIG_NOT_APPLIED)
else:
print(green("\n*** No changes needed in Tango DB ***"), file=sys.stderr)
sys.exit(SUCCESS)
