def test_modify_class_attribute_property():
"Check that the attribute property is changed"
config = fake.tango_database()
if "classes" in config:
clss = choice(config["classes"].keys())
props = config["classes"][clss]
attr = "test_attribute"
propname, value = fake.tango_attribute_property()
if "attribute_properties" not in props:
props["attribute_properties"] = {}
props["attribute_properties"][attr] = {propname: value}
orig_config = deepcopy(config)
props["attribute_properties"][attr][propname] = "abc"
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_class_attribute_property", (clss, {
attr: {
propname: "abc"
}
}), {})
assert calls[0] == expected
def test_remove_device_property():
"""Check that the property is removed"""
config = fake.tango_database()
orig_config = deepcopy(config)
srv, inst, clss, dev, name, value = pick_random_property(config)
del config["servers"][srv][inst][clss][dev]["properties"][name]
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("delete_device_property", (dev, {name: value}), {})
assert calls[0] == expected
def test_modify_device_property_line():
"""Check that a property is updated when a line is changed"""
config = fake.tango_database()
orig_config = deepcopy(config)
srv, inst, clss, dev, name, value = pick_random_property(config)
value[randint(0, len(value) - 1)] = fake.word()
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_device_property", (dev, {name: value}), {})
assert calls[0] == expected
def test_remove_device():
config = fake.tango_database()
orig_config = deepcopy(config)
_, _, classname, devices = pick_random_class(config)
device = choice(devices.keys())
del devices[device]
calls = configure(config, orig_config)
assert len(calls) == 1
assert calls[0] == ("delete_device", (device, ), {})
def test_remove_class_property():
"""Check that the property is removed"""
config = fake.tango_database(classes=(3, 5))
orig_config = deepcopy(config)
clss, name, value = pick_random_class_property(config)
del config["classes"][clss]["properties"][name]
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("delete_class_property", (clss, {name: value}), {})
assert calls[0] == expected
def test_add_device_property():
"Check that the property is added"
config = fake.tango_database()
orig_config = deepcopy(config)
name, value = fake.tango_property()
_, _, _, dev, props = pick_random_device(config)
props["properties"][name] = value
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_device_property", (dev, {name: value}), {})
assert calls[0] == expected
def test_add_class_property():
"Check that the property is added"
config = fake.tango_database(classes=(3, 5))
orig_config = deepcopy(config)
name, value = fake.tango_property()
clss = choice(config["classes"].keys())
props = config["classes"][clss]
props["properties"][name] = value
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_class_property", (clss, {name: value}), {})
assert calls[0] == expected
def test_remove_device_property_line():
"""Check that the property is overwritten with the new value when
removing a line from a property"""
config = fake.tango_database()
orig_config = deepcopy(config)
_, _, _, dev, name, value = pick_random_property(config)
old_value = deepcopy(value)
del value[randint(0, len(value) - 1)]
calls = configure(config, orig_config)
assert len(calls) == 1
if len(value):
expected = ("put_device_property", (dev, {name: value}), {})
else:
expected = ("delete_device_property", (dev, {name: old_value}), {})
assert calls[0] == expected
def test_add_device_to_existing_class():
config = fake.tango_database()
orig_config = deepcopy(config)
srv, inst, cls, devices = pick_random_class(config)
devname, device = fake.tango_device()
devices[devname] = {}
calls = configure(config, orig_config)
assert len(calls) == 1
method, [info], kwargs = calls[0]
assert method == "add_device"
assert type(info) == PyTango.DbDevInfo
assert info.server == "%s/%s" % (srv, inst)
assert info.klass == cls
assert info.name == devname
def test_cant_remove_protected_attribute_property():
"Check that the attribute property is *not* removed"
config = fake.tango_database()
_, _, _, dev, props = pick_random_device(config)
attr = "test_attribute"
propname, value = fake.tango_attribute_property()
if "attribute_properties" not in props:
props["attribute_properties"] = {}
props["attribute_properties"][attr] = {propname: value}
orig_config = deepcopy(config)
del props["attribute_properties"][attr][propname]
calls = configure(config, orig_config)
assert len(calls) == 0
def test_cant_remove_protected_class_attribute_property():
"Check that the attribute property is *not* removed"
config = fake.tango_database()
clss = choice(config["classes"].keys())
props = config["classes"][clss]
attr = "test_attribute"
propname, value = fake.tango_attribute_property()
if "attribute_properties" not in props:
props["attribute_properties"] = {}
props["attribute_properties"][attr] = {propname: value}
orig_config = deepcopy(config)
del props["attribute_properties"][attr][propname]
calls = configure(config, orig_config)
assert len(calls) == 0
def test_add_server():
config = fake.tango_database()
orig_config = deepcopy(config)
config["servers"]["test_server"] = {
"test_instance": {
"test_class": {
"my/test/device": {}
}
}
}
calls = configure(config, orig_config)
assert len(calls) == 1
method, [info], kwargs = calls[0]
assert method == "add_device"
assert type(info) == PyTango.DbDevInfo
assert info.server == "test_server/test_instance"
assert info.klass == "test_class"
assert info.name == "my/test/device"
def test_add_attribute_property():
"Check that the attribute property is added"
config = fake.tango_database()
orig_config = deepcopy(config)
_, _, _, dev, props = pick_random_device(config)
attr = "test_attribute"
propname, value = fake.tango_attribute_property()
if "attribute_properties" not in props:
props["attribute_properties"] = {}
props["attribute_properties"][attr] = {propname: value}
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_device_attribute_property", (dev, {
attr: {
propname: value
}
}), {})
assert calls[0] == expected
def test_add_class_attribute_property():
"Check that the attribute property is added"
config = fake.tango_database(classes=(3, 5))
orig_config = deepcopy(config)
clss = choice(list(config["classes"].keys()))
props = config["classes"][clss]
attr = "test_attribute"
propname, value = fake.tango_attribute_property()
if "attribute_properties" not in props:
props["attribute_properties"] = {}
props["attribute_properties"][attr] = {propname: value}
calls = configure(config, orig_config)
assert len(calls) == 1
expected = ("put_class_attribute_property", (clss, {
attr: {
propname: value
}
}), {})
assert calls[0] == expected
def main():
usage = "Usage: %prog [options] JSONFILE"
parser = OptionParser(usage=usage)
parser.add_option("-w", "--write", dest="write", action="store_true",
help="write to the Tango DB", metavar="WRITE")
parser.add_option("-u", "--update", dest="update", action="store_true",
help="don't remove things, only add/update",
metavar="UPDATE")
parser.add_option("-c", "--case-sensitive", dest="case_sensitive",
action="store_true",
help=("Don't ignore the case of server, device, "
"attribute and property names"),
metavar="CASESENSITIVE")
parser.add_option("-q", "--quiet",
action="store_false", dest="verbose", default=True,
help="don't print actions to stderr")
parser.add_option("-o", "--output", dest="output", action="store_true",
help="Output the relevant DB state as JSON.")
parser.add_option("-p", "--input", dest="input", action="store_true",
help="Output the input JSON (after filtering).")
parser.add_option("-d", "--dbcalls", dest="dbcalls", action="store_true",
help="print out all db calls.")
parser.add_option("-v", "--no-validation", dest="validate", default=True,
action="store_false", help=("Skip JSON validation"))
parser.add_option("-s", "--sleep", dest="sleep", default=0.01,
type="float",
help=("Number of seconds to sleep between DB calls"))
parser.add_option("-n", "--no-colors",
action="store_true", dest="no_colors", default=False,
help="Don't print colored output")
parser.add_option("-i", "--include", dest="include", action="append",
help=("Inclusive filter on server configutation"))
parser.add_option("-x", "--exclude", dest="exclude", action="append",
help=("Exclusive filter on server configutation"))
parser.add_option("-I", "--include-classes", dest="include_classes",
action="append",
help=("Inclusive filter on class configuration"))
parser.add_option("-X", "--exclude-classes", dest="exclude_classes",
action="append",
help=("Exclusive filter on class configuration"))
parser.add_option(
"-D", "--dbdata",
help="Read the given file as DB data instead of using the actual DB",
dest="dbdata")
options, args = parser.parse_args()
if options.no_colors:
no_colors()
if len(args) == 0:
data = load_json(sys.stdin)
else:
json_file = args[0]
with open(json_file) as f:
data = load_json(f)
# Normalization - making the config conform to standard
data = normalize_config(data)
# remove any metadata at the top level (should we use this for something?)
data = clean_metadata(data)
# Optional validation of the JSON file format.
if options.validate:
validate_json(data)
# filtering
try:
if options.include:
data["servers"] = filter_config(
data.get("servers", {}), options.include, SERVERS_LEVELS)
if options.exclude:
data["servers"] = filter_config(
data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True)
if options.include_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.include_classes, CLASSES_LEVELS)
if options.exclude_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS,
invert=True)
except ValueError as e:
print >>sys.stderr, red("Filter error:\n%s" % e)
sys.exit(ERROR)
if not any(k in data for k in ("devices", "servers", "classes")):
sys.exit(ERROR)
if options.input:
print json.dumps(data, indent=4)
return
# check if there is anything in the DB that will be changed or removed
db = PyTango.Database()
if options.dbdata:
with open(options.dbdata) as f:
original = json.loads(f.read())
collisions = {}
else:
original = get_db_data(db, dservers=True)
devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(data["servers"])
})
orig_devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(original["servers"])
})
collisions = {}
for dev, (srv, inst, cls) in devices.items():
if dev in orig_devices:
server = "{}/{}".format(srv, inst)
osrv, oinst, ocls = orig_devices[dev]
origserver = "{}/{}".format(osrv, oinst)
if server.lower() != origserver.lower():
collisions.setdefault(origserver, []).append((ocls, dev))
# get the list of DB calls needed
dbcalls = configure(data, original,
update=options.update,
ignore_case=not options.case_sensitive)
# Print out a nice diff
if options.verbose:
show_actions(original, dbcalls)
# perform the db operations (if we're supposed to)
if options.write and dbcalls:
for i, (method, args, kwargs) in enumerate(dbcalls):
if options.sleep:
time.sleep(options.sleep)
if options.verbose:
progressbar(i, len(dbcalls), 20)
getattr(db, method)(*args, **kwargs)
print
# optionally dump some information to stdout
if options.output:
print json.dumps(original, indent=4)
if options.dbcalls:
print >>sys.stderr, "Tango database calls:"
for method, args, kwargs in dbcalls:
print >>sys.stderr, method, args
# Check for moved devices and remove empty servers
empty = set()
for srvname, devs in collisions.items():
if options.verbose:
srv, inst = srvname.split("/")
for cls, dev in devs:
print >>sys.stderr, red("MOVED (because of collision):"), dev
print >>sys.stderr, " Server: ", "{}/{}".format(srv, inst)
print >>sys.stderr, " Class: ", cls
if len(db.get_device_class_list(srvname)) == 2: # just dserver
empty.add(srvname)
if options.write:
db.delete_server(srvname)
# finally print out a brief summary of what was done
if dbcalls:
print
print >>sys.stderr, "Summary:"
print >>sys.stderr, "\n".join(summarise_calls(dbcalls, original))
if collisions:
servers = len(collisions)
devices = sum(len(devs) for devs in collisions.values())
print >>sys.stderr, red("Move %d devices from %d servers." %
(devices, servers))
if empty and options.verbose:
print >>sys.stderr, red("Removed %d empty servers." % len(empty))
if options.write:
print >>sys.stderr, red("\n*** Data was written to the Tango DB ***")
with NamedTemporaryFile(prefix="dsconfig-", suffix=".json",
delete=False) as f:
f.write(json.dumps(original, indent=4))
print >>sys.stderr, ("The previous DB data was saved to %s" %
f.name)
sys.exit(CONFIG_APPLIED)
else:
print >>sys.stderr, yellow(
"\n*** Nothing was written to the Tango DB (use -w) ***")
sys.exit(CONFIG_NOT_APPLIED)
else:
print >>sys.stderr, green("\n*** No changes needed in Tango DB ***")
sys.exit(SUCCESS)
def json_to_tango(options, args):
if options.no_colors:
no_colors()
if len(args) == 0:
data = load_json(sys.stdin)
else:
json_file = args[0]
with open(json_file) as f:
data = load_json(f)
# Normalization - making the config conform to standard
data = normalize_config(data)
# remove any metadata at the top level (should we use this for something?)
data = clean_metadata(data)
# Optional validation of the JSON file format.
if options.validate:
validate_json(data)
# filtering
try:
if options.include:
data["servers"] = filter_config(
data.get("servers", {}), options.include, SERVERS_LEVELS)
if options.exclude:
data["servers"] = filter_config(
data.get("servers", {}), options.exclude, SERVERS_LEVELS, invert=True)
if options.include_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.include_classes, CLASSES_LEVELS)
if options.exclude_classes:
data["classes"] = filter_config(
data.get("classes", {}), options.exclude_classes, CLASSES_LEVELS,
invert=True)
except ValueError as e:
print(red("Filter error:\n%s" % e), file=sys.stderr)
sys.exit(ERROR)
if not any(k in data for k in ("devices", "servers", "classes")):
sys.exit(ERROR)
if options.input:
print(json.dumps(data, indent=4))
return
# check if there is anything in the DB that will be changed or removed
db = tango.Database()
if options.dbdata:
with open(options.dbdata) as f:
original = json.loads(f.read())
collisions = {}
else:
original = get_db_data(db, dservers=True, class_properties=True)
if "servers" in data:
devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(data["servers"])
})
else:
devices = CaselessDictionary({})
orig_devices = CaselessDictionary({
dev: (srv, inst, cls)
for srv, inst, cls, dev
in get_devices_from_dict(original["servers"])
})
collisions = {}
for dev, (srv, inst, cls) in list(devices.items()):
if dev in orig_devices:
server = "{}/{}".format(srv, inst)
osrv, oinst, ocls = orig_devices[dev]
origserver = "{}/{}".format(osrv, oinst)
if server.lower() != origserver.lower():
collisions.setdefault(origserver, []).append((ocls, dev))
# get the list of DB calls needed
dbcalls = configure(data, original,
update=options.update,
ignore_case=not options.case_sensitive,
strict_attr_props=not options.nostrictcheck)
# Print out a nice diff
if options.verbose:
show_actions(original, dbcalls)
# perform the db operations (if we're supposed to)
if options.write and dbcalls:
for i, (method, args, kwargs) in enumerate(dbcalls):
if options.sleep:
time.sleep(options.sleep)
if options.verbose:
progressbar(i, len(dbcalls), 20)
getattr(db, method)(*args, **kwargs)
print()
# optionally dump some information to stdout
if options.output:
print(json.dumps(original, indent=4))
if options.dbcalls:
print("Tango database calls:", file=sys.stderr)
for method, args, kwargs in dbcalls:
print(method, args, file=sys.stderr)
# Check for moved devices and remove empty servers
empty = set()
for srvname, devs in list(collisions.items()):
if options.verbose:
srv, inst = srvname.split("/")
for cls, dev in devs:
print(red("MOVED (because of collision):"), dev, file=sys.stderr)
print(" Server: ", "{}/{}".format(srv, inst), file=sys.stderr)
print(" Class: ", cls, file=sys.stderr)
if len(db.get_device_class_list(srvname)) == 2: # just dserver
empty.add(srvname)
if options.write:
db.delete_server(srvname)
# finally print out a brief summary of what was done
if dbcalls:
print()
print("Summary:", file=sys.stderr)
print("\n".join(summarise_calls(dbcalls, original)), file=sys.stderr)
if collisions:
servers = len(collisions)
devices = sum(len(devs) for devs in list(collisions.values()))
print(red("Move %d devices from %d servers." %
(devices, servers)), file=sys.stderr)
if empty and options.verbose:
print(red("Removed %d empty servers." % len(empty)), file=sys.stderr)
if options.write:
print(red("\n*** Data was written to the Tango DB ***"), file=sys.stderr)
with NamedTemporaryFile(prefix="dsconfig-", suffix=".json",
delete=False) as f:
f.write(json.dumps(original, indent=4).encode())
print(("The previous DB data was saved to %s" %
f.name), file=sys.stderr)
sys.exit(CONFIG_APPLIED)
else:
print(yellow(
"\n*** Nothing was written to the Tango DB (use -w) ***"), file=sys.stderr)
sys.exit(CONFIG_NOT_APPLIED)
else:
print(green("\n*** No changes needed in Tango DB ***"), file=sys.stderr)
sys.exit(SUCCESS)
