def normalize_config(config):
"""Take a 'loose' config and return a new config that conforms to the
DSConfig format.
Current transforms:
- server instances (e.g. 'TangoTest/1') are split into a server
level and an instance level (e.g. 'TangoTest' -> '1'). This is to
convert "v1" format files to the "v2" format.
- "devices" toplevel; allows to change *existing* devices by just
adding them directly to a "devices" key in the config, instead
of having to list out the server, instance and class (since this
information can be gotten from the DB.)
"""
old_config = expand_config(config)
new_config = SetterDict()
if "servers" in old_config:
new_config.servers = old_config["servers"]
if "classes" in old_config:
new_config.classes = old_config["classes"]
if "devices" in old_config:
db = PyTango.Database()
for device, props in old_config["devices"].items():
try:
info = db.get_device_info(device)
except PyTango.DevFailed as e:
sys.exit("Can't reconfigure device %s: %s" %
(device, str(e[0].desc)))
srv, inst = info.ds_full_name.split("/")
new_config.servers[srv][inst][info.class_name][device] = props
return new_config.to_dict()
def get_db_data(db, patterns=None, class_properties=False, **options):
# dump TANGO database into JSON. Optionally filter which things to include
# (currently only "positive" filters are possible; you can say which
# servers/classes/devices to include, but you can't exclude selectively)
# By default, dserver devices aren't included!
dbproxy = PyTango.DeviceProxy(db.dev_name())
data = SetterDict()
if not patterns:
# the user did not specify a pattern, so we will dump *everything*
servers = get_servers_with_filters(
dbproxy, **options)
data.servers.update(servers)
if class_properties:
classes = get_classes_properties(dbproxy)
data.classes.update(classes)
else:
# go through all patterns and fill in the data
for pattern in patterns:
prefix, pattern = pattern.split(":")
kwargs = {prefix: pattern}
kwargs.update(options)
servers = get_servers_with_filters(dbproxy, **kwargs)
data.servers.update(servers)
if class_properties:
classes = get_classes_properties(dbproxy, server=pattern)
data.classes.update(classes)
return data.to_dict()
def get_dict_from_db(db, data, narrow=False, skip_protected=True):
"""Takes a data dict, checks if any if the definitions are already
in the DB and returns a dict describing them.
By default it includes all devices for each server+class, use the
'narrow' flag to limit to the devices present in the input data.
"""
# This is where we'll collect all the relevant data
dbdict = SetterDict()
moved_devices = defaultdict(list)
# Devices that are already defined somewhere else
for server, inst, clss, device in get_devices_from_dict(
data.get("servers", {})):
try:
devinfo = db.get_device_info(device)
srvname = "%s/%s" % (server, inst)
if devinfo.ds_full_name.lower() != srvname.lower():
moved_devices[devinfo.ds_full_name].append((clss, device))
except PyTango.DevFailed:
pass
# Servers
for srvr, insts in data.get("servers", {}).items():
for inst, classes in insts.items():
for clss, devs in classes.items():
devs = CaselessDictionary(devs)
if narrow:
devices = devs.keys()
else:
srv_full_name = "%s/%s" % (srvr, inst)
devices = db.get_device_name(srv_full_name, clss)
for device in devices:
new_props = devs.get(device, {})
db_props = get_device(db, device, new_props, skip_protected)
dbdict.servers[srvr][inst][clss][device] = db_props
# Classes
for class_name, cls in data.get("classes", {}).items():
props = cls.get("properties", {}).keys()
for prop, value in db.get_class_property(class_name, props).items():
if value:
value = [str(v) for v in value]
dbdict.classes[class_name].properties[prop] = value
attr_props = cls.get("attribute_properties")
if attr_props:
dbprops = db.get_class_attribute_property(class_name,
attr_props.keys())
for attr, props in dbprops.items():
props = dict((prop, [str(v) for v in values])
for prop, values in props.items())
dbdict.classes[class_name].attribute_properties[attr] = props
return dbdict.to_dict(), moved_devices
def get_changes(data, calls):
"""Combine a list of database calls into "changes" that can
be more easily turned into a readable representation"""
devices = get_devices_from_dict(data["servers"])
device_mapping = CaselessDict(
(device, (server, instance, clss))
for server, instance, clss, device in devices)
classes = data.get("classes", {})
# The idea is to first go through all database calls and collect
# the changes per device. Then we can print it all out in a more
# readable format since it will all be collected per device.
#
# Example of resulting dict:
# {
# "sys/tg_test/1": {
# "server": "TangoTest",
# "instance": "test",
# "device_class": "TangoTest",
# "properties": {
# # added property has no old_value
# "hej": {
# "value": ["a", "b", "c"]
# },
# # removed property has no value
# "svej": {
# "old_value": ["dsaodkasokd"]
# },
# # changed property has both
# "flepp": {
# "old_value": ["1", "3"],
# "value": ["1", "2", "3"]
# }
# }
# }
# }
changes = {"devices": SetterDict(), "classes": SetterDict()}
# Go through all database calls and store the changes
for method, args, kwargs in calls:
if method == "put_device_alias":
device, alias = args
if device in device_mapping:
old_alias = get_device_data(device, device_mapping,
data).get("alias")
else:
old_alias = None
changes["devices"][device].update(
{"alias": {
"old_value": old_alias,
"value": alias
}})
elif method == "add_device":
info, = args
changes["devices"][info.name].update(added=True,
server=info.server,
device_class=info._class)
if info.name in device_mapping:
old_server, old_instance, old_class = device_mapping[info.name]
changes["devices"][info.name]["old_server"] = "{}/{}".format(
old_server, old_instance)
changes["devices"][info.name]["old_class"] = old_class
elif method == "delete_device":
device, = args
server, instance, clss = device_mapping[device]
device_data = data["servers"][server][instance][clss]
properties = device_data.get("properties", {})
changes["devices"][device].update(deleted=True,
server="{}/{}".format(
server, instance),
instance=instance,
device_class=clss,
properties=properties)
elif method == "put_device_property":
device, properties = args
old_data = get_device_data(device, device_mapping, data)
caseless_props = CaselessDict(old_data.get("properties", {}))
if "properties" not in changes["devices"][device]:
changes["devices"][device]["properties"] = {}
for name, value in properties.items():
old_value = caseless_props.get(name)
if value != old_value:
changes["devices"][device]["properties"].update(
{name: {
"value": value,
"old_value": old_value
}})
elif method == "delete_device_property":
device, properties = args
old_data = get_device_data(device, device_mapping, data)
caseless_props = CaselessDict(old_data.get("properties", {}))
prop_changes = changes["devices"][device].setdefault(
"properties", {})
for prop in properties:
old_value = caseless_props[prop]
prop_changes[prop] = {"old_value": old_value}
elif method == "put_device_attribute_property":
device, properties = args
attr_props = changes["devices"][device].setdefault(
"attribute_properties", {})
old_data = get_device_data(device, device_mapping, data)
caseless_attrs = CaselessDict(
old_data.get("attribute_properties", {}))
for attr, props in properties.items():
caseless_props = CaselessDict(caseless_attrs.get(attr, {}))
for name, value in props.items():
old_value = caseless_props.get(name)
if value != old_value:
attr_props[attr] = {
name: {
"old_value": old_value,
"value": value
}
}
elif method == "delete_device_attribute_property":
device, attributes = args
prop_changes = attr_props = changes["devices"][device].setdefault(
"attribute_properties", {})
old_data = get_device_data(device, device_mapping, data)
caseless_attrs = CaselessDict(
old_data.get("attribute_properties", {}))
for attr, props in attributes.items():
caseless_props = CaselessDict(caseless_attrs[attr])
for prop in props:
old_value = caseless_props.get(prop)
attr_props[attr] = {prop: {"old_value": old_value}}
elif method == "put_class_property":
clss, properties = args
old_data = classes.get(clss, {})
caseless_props = CaselessDict(old_data.get("properties", {}))
prop_changes = changes["classes"][clss].setdefault(
"properties", {})
for name, value in properties.items():
old_value = caseless_props.get(name)
if value != old_value:
prop_changes.update(
{name: {
"value": value,
"old_value": old_value
}})
elif method == "delete_class_property":
clss, properties = args
old_data = classes.get(clss, {})
caseless_props = CaselessDict(old_data.get("properties", {}))
prop_changes = changes["classes"][clss].setdefault(
"properties", {})
for prop in properties:
old_value = caseless_props.get(prop)
prop_changes[prop] = {"old_value": old_value}
elif method == "put_class_attribute_property":
clss, properties = args
attr_props = changes["classes"][clss].setdefault(
"attribute_properties", {})
old_data = classes.get(clss, {})
caseless_attrs = CaselessDict(
old_data.get("attribute_properties", {}))
for attr, props in properties.items():
caseless_props = CaselessDict(caseless_attrs.get(attr, {}))
for name, value in props.items():
old_value = caseless_props.get(name)
if value != old_value:
attr_props[attr] = {
name: {
"old_value": old_value,
"value": value
}
}
elif method == "delete_class_attribute_property":
clss, attributes = args
attr_props = changes["classes"][clss].setdefault(
"attribute_properties", {})
old_data = classes.get(clss, {})
caseless_attrs = CaselessDict(old_data.get("properties", {}))
for attr, props in attributes.items():
caseless_props = CaselessDict(caseless_attrs.get(attr, {}))
for prop in props:
old_value = caseless_props.get(prop)
attr_props[attr] = {prop: {"old_value": old_value}}
return {
"devices": changes["devices"].to_dict(),
"classes": changes["classes"].to_dict(),
}
def get_servers_with_filters(dbproxy, server="*", clss="*", device="*",
properties=True, attribute_properties=True,
aliases=True, dservers=False,
subdevices=False, uppercase_devices=False,
timeout=10):
"""
A performant way to get servers and devices in bulk from the DB
by direct SQL statements and joins, instead of e.g. using one
query to get the properties of each device.
TODO: are there any length restrictions on the query results? In
that case, use limit and offset to get page by page.
"""
server = server.replace("*", "%") # mysql wildcards
clss = clss.replace("*", "%")
device = device.replace("*", "%")
devices = AppendingDict()
# Queries can sometimes take more than de default 3 s, so it's
# good to increase the timeout a bit.
# TODO: maybe instead use automatic retry and increase timeout
# each time?
dbproxy.set_timeout_millis(timeout*1000)
if properties:
# Get all relevant device properties
query = (
"SELECT device, property_device.name, property_device.value"
" FROM property_device"
" INNER JOIN device ON property_device.device = device.name"
" WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
if not dservers:
query += " AND class != 'DServer'"
if not subdevices:
query += " AND property_device.name != '__SubDevices'"
_, result = dbproxy.command_inout("DbMySqlSelect",
query % (server, clss, device))
for d, p, v in nwise(result, 3):
# the properties are encoded in latin-1; we want utf-8
decoded_value = v.decode('iso-8859-1').encode('utf8')
devices[maybe_upper(d, uppercase_devices)].properties[p] = decoded_value
if attribute_properties:
# Get all relevant attribute properties
query = (
"SELECT device, attribute, property_attribute_device.name,"
" property_attribute_device.value"
" FROM property_attribute_device"
" INNER JOIN device ON property_attribute_device.device ="
" device.name"
" WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
if not dservers:
query += " AND class != 'DServer'"
_, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
for d, a, p, v in nwise(result, 4):
dev = devices[maybe_upper(d, uppercase_devices)]
# the properties are encoded in latin-1; we want utf-8
decoded_value = v.decode('iso-8859-1').encode('utf8')
dev.attribute_properties[a][p] = decoded_value
devices = devices.to_dict()
# dump relevant servers
query = (
"SELECT server, class, name, alias FROM device"
" WHERE server LIKE '%s' AND class LIKE '%s' AND name LIKE '%s'")
if not dservers:
query += " AND class != 'DServer'"
_, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
# combine all the information we have
servers = SetterDict()
for s, c, d, a in nwise(result, 4):
try:
srv, inst = s.split("/")
except ValueError:
# Malformed server name? It can happen!
continue
devname = maybe_upper(d, uppercase_devices)
device = devices.get(devname, {})
if a and aliases:
device["alias"] = a
servers[srv][inst][c][devname] = device
return servers