class IppmDNSBridge(object):
def __init__(self, logger=None):
self.logger = Logger("mdnsbridge", logger)
self.services = {}
self.config = {"priority": 0}
self.config.update(_config)
def _checkLocalQueryServiceExists(self):
url = "http://127.0.0.1/x-nmos/query/v1.0/";
try:
# Request to localhost:18870/ - if it succeeds, the service exists AND is running AND is accessible
r = requests.get(url, timeout=0.5)
if r is not None and r.status_code == 200:
# If any results, put them in self.services
return url
except Exception as e:
self.logger.writeWarning("No local query service running {}".format(e))
return ""
def getHref(self, srv_type, priority=None):
if priority == None:
priority = self.config["priority"]
if self.logger != None:
self.logger.writeDebug("IppmDNSBridge priority = {}".format(priority))
# Check if type is in services. If not add it
if srv_type not in self.services:
self.services[srv_type] = []
# Check if there are any of that type of service, if not do a request
no_results = True
for service in self.services[srv_type]:
if priority >= 100:
if service["priority"] == priority:
no_results = False
elif service["priority"] < 100:
no_results = False
if no_results:
self._updateServices(srv_type)
# Re-check if there are any and return "" if not.
current_priority = 99
valid_services = []
for service in self.services[srv_type]:
if priority >= 100:
if service["priority"] == priority:
return self._createHref(service["address"], service["port"])
else:
if service["priority"] < current_priority:
current_priority = service["priority"]
valid_services = []
if service["priority"] == current_priority:
valid_services.append(service)
if len(valid_services) == 0:
self.logger.writeWarning("No services found: {}".format(srv_type))
if srv_type == "nmos-query":
return self._checkLocalQueryServiceExists()
return ""
# Randomise selection. Delete entry from the services list and return it
random.seed()
index = random.randint(0, len(valid_services)-1)
service = valid_services[index]
href = self._createHref(service["address"], service["port"])
self.services[srv_type].remove(service)
return href
def _createHref(self, address, port):
formatted_address = address
if ":" in formatted_address:
formatted_address = "[" + formatted_address + "]"
return "http://" + formatted_address + ":" + str(port)
def _updateServices(self, srv_type):
req_url = "http://127.0.0.1/x-nmos-opensourceprivatenamespace/mdnsbridge/v1.0/" + srv_type + "/";
try:
# Request to localhost/x-nmos-opensourceprivatenamespace/mdnsbridge/v1.0/<type>/
r = requests.get(req_url, timeout=0.5, proxies={'http': ''})
if r is not None and r.status_code == 200:
# If any results, put them in self.services
self.services[srv_type] = r.json()["representation"]
except Exception as e:
self.logger.writeWarning("Exception updating services: {}".format(e))
class NodeFacadeService:
def __init__(self, interactive=False):
self.logger = Logger("facade", None)
if HOST == "":
self.logger.writeFatal(
"Unable to start facade due to lack of connectivity")
sys.exit(1)
self.running = False
self.httpServer = None
self.interface = None
self.interactive = interactive
self.registry = None
self.registry_cleaner = None
self.node_id = None
self.mdns = MDNSEngine()
self.mdnsname_suffix = '_' + str(HOSTNAME) + "_" + str(getpid())
self.mappings = {
"device": "ver_dvc",
"flow": "ver_flw",
"source": "ver_src",
"sender": "ver_snd",
"receiver": "ver_rcv",
"self": "ver_slf"
}
self.mdns_updater = MDNSUpdater(self.mdns,
"_nmos-node._tcp",
"node" + self.mdnsname_suffix,
self.mappings,
PORT,
self.logger,
txt_recs={
"api_ver": "v1.0,v1.1,v1.2",
"api_proto": "http"
})
self.aggregator = Aggregator(self.logger, self.mdns_updater)
def sig_handler(self):
print 'Pressed ctrl+c'
self.stop()
def sig_hup_handler(self):
if getLocalIP() != "":
global HOST
HOST = updateHost()
self.registry.modify_node(href=self.generate_href(),
host=HOST,
api={
"versions": NODE_APIVERSIONS,
"endpoints":
self.generate_endpoints()
},
interfaces=self.list_interfaces())
def generate_endpoints(self):
endpoints = []
if HTTPS_MODE != "enabled":
endpoints.append({
"host": HOST,
"port": 80, #Everything should go via apache proxy
"protocol": "http"
})
if HTTPS_MODE != "disabled":
endpoints.append({
"host": HOST,
"port": 443, #Everything should go via apache proxy
"protocol": "https"
})
return endpoints
def generate_href(self):
if HTTPS_MODE == "enabled":
return "https://{}/".format(HOST)
else:
return "http://{}/".format(HOST)
def list_interfaces(self):
interfaces = {}
# Initially populate interfaces from known-good location
net_path = "/sys/class/net/"
if os.path.exists(net_path):
for interface_name in os.listdir(net_path):
if interface_name != "lo":
address_path = net_path + interface_name + "/address"
if os.path.exists(address_path):
address = open(address_path, "r").readline()
interfaces[interface_name] = {
"name":
interface_name,
"chassis_id":
None,
"port_id":
address.lower().strip("\n").replace(":", "-")
}
# Attempt to source proper LLDP data for interfaces
if os.path.exists("/usr/sbin/lldpcli"):
try:
chassis_data = json.loads(
check_output(
["/usr/sbin/lldpcli", "show", "chassis", "-f",
"json"]))
chassis_id = chassis_data["local-chassis"]['chassis'].values(
)[0]["id"]["value"]
if chassis_data["local-chassis"]['chassis'].values(
)[0]["id"]["type"] == "mac":
chassis_id = chassis_id.lower().replace(":", "-")
interface_data = json.loads(
check_output([
"/usr/sbin/lldpcli", "show", "statistics", "-f", "json"
]))
if isinstance(interface_data["lldp"]["interface"], dict):
for interface_name in interface_data["lldp"][
"interface"].keys():
if interface_name in interfaces:
# Only correct the Chassis ID. Port ID MUST be a MAC address
interfaces[interface_name][
"chassis_id"] = chassis_id
else:
for interface_block in interface_data["lldp"]["interface"]:
interface_name = interface_block.keys()[0]
if interface_name in interfaces:
# Only correct the Chassis ID. Port ID MUST be a MAC address
interfaces[interface_name][
"chassis_id"] = chassis_id
except Exception:
pass
return interfaces.values()
def start(self):
if self.running:
gevent.signal(signal.SIGINT, self.sig_handler)
gevent.signal(signal.SIGTERM, self.sig_handler)
gevent.signal(signal.SIGHUP, self.sig_hup_handler)
self.mdns.start()
self.node_id = get_node_id()
node_version = str(ptptime.ptp_detail()[0]) + ":" + str(
ptptime.ptp_detail()[1])
node_data = {
"id":
self.node_id,
"label":
nmoscommonconfig.config.get('node_label', FQDN),
"description":
nmoscommonconfig.config.get('node_description',
"Node on {}".format(FQDN)),
"tags":
nmoscommonconfig.config.get('node_tags', {}),
"href":
self.generate_href(),
"host":
HOST,
"services": [],
"hostname":
HOSTNAME,
"caps": {},
"version":
node_version,
"api": {
"versions": NODE_APIVERSIONS,
"endpoints": self.generate_endpoints(),
},
"clocks": [
{
"name": "clk0",
"ref_type": "internal",
},
{
"name": "clk1",
"ref_type": "ptp",
"version": "IEEE1588-2008",
"traceable": False,
"gmid": "00-00-00-00-00-00-00-00",
"locked": False,
},
# Extra values will be filled in as needed at point of checking
],
"interfaces":
self.list_interfaces()
}
self.registry = FacadeRegistry(self.mappings.keys(), self.aggregator,
self.mdns_updater, self.node_id,
node_data, self.logger)
self.registry_cleaner = FacadeRegistryCleaner(self.registry)
self.registry_cleaner.start()
self.httpServer = HttpServer(FacadeAPI,
PORT,
'0.0.0.0',
api_args=[self.registry])
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeInfo('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeInfo("Running on port: {}".format(
self.httpServer.port))
try:
self.logger.writeInfo("Registering as {}...".format(self.node_id))
self.aggregator.register(
'node', self.node_id,
**legalise_resource(node_data, "node", NODE_REGVERSION))
except Exception as e:
self.logger.writeWarning("Could not register: {}".format(
e.__repr__()))
self.interface = FacadeInterface(self.registry, self.logger)
self.interface.start()
def run(self):
self.running = True
pidfile = "/tmp/ips-nodefacade.pid"
file(pidfile, 'w').write(str(getpid()))
self.start()
daemon.notify("READY=1")
while self.running:
self.registry.update_ptp()
time.sleep(1)
os.unlink(pidfile)
self._cleanup()
def _cleanup(self):
try:
self.logger.writeDebug("cleanup: unregister facade " +
self.node_id)
self.aggregator.unregister('node', self.node_id)
except Exception as e:
self.logger.writeWarning("Could not unregister: {}".format(e))
if self.mdns:
try:
self.mdns.stop()
except Exception as e:
self.logger.writeWarning("Could not stop mdns: {}".format(e))
self.registry_cleaner.stop()
self.interface.stop()
self.httpServer.stop()
self.aggregator.stop()
self.logger.writeInfo("Stopped main()")
def stop(self):
self.running = False
class FacadeRegistry(object):
def __init__(self, resources, aggregator, mdns_updater, node_id, node_data, logger=None):
# `node_data` must be correctly structured
self.permitted_resources = resources
self.services = {}
self.clocks = {"clk0": {"name": "clk0", "ref_type": "internal"}}
self.aggregator = aggregator
self.mdns_updater = mdns_updater
self.node_id = node_id
assert "interfaces" in node_data # Check data conforms to latest supported API version
self.node_data = node_data
self.logger = Logger("facade_registry", logger)
def modify_node(self, **kwargs):
for key in kwargs.keys():
if key in self.node_data:
self.node_data[key] = kwargs[key]
self.update_node()
def update_node(self):
self.node_data["services"] = []
for service_name in self.services:
href = None
if self.services[service_name]["href"]:
if self.services[service_name]["proxy_path"]:
href = self.node_data["href"] + self.services[service_name]["proxy_path"]
self.node_data["services"].append({
"href": href,
"type": self.services[service_name]["type"],
"authorization": self.services[service_name]["authorization"]
})
self.node_data["clocks"] = list(itervalues(self.clocks))
self.node_data["version"] = str(ptptime.ptp_detail()[0]) + ":" + str(ptptime.ptp_detail()[1])
try:
self.aggregator.register("node", self.node_id, **self.preprocess_resource("node", self.node_data["id"],
self.node_data, NODE_REGVERSION))
except Exception as e:
self.logger.writeError("Exception re-registering node: {}".format(e))
def register_service(self, name, srv_type, pid, href=None, proxy_path=None, authorization=False):
if name in self.services:
return RES_EXISTS
self.services[name] = {
"heartbeat": time.time(),
"resource": {}, # Registered resources live under here
"control": {}, # Registered device controls live under here
"pid": pid,
"href": href,
"proxy_path": proxy_path,
"type": srv_type,
"authorization": authorization
}
for resource_name in self.permitted_resources:
self.services[name]["resource"][resource_name] = {}
self.update_node()
return RES_SUCCESS
def update_service(self, name, pid, href=None, proxy_path=None):
if name not in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
self.services[name]["href"] = href
self.services[name]["proxy_path"] = proxy_path
self.update_node()
return RES_SUCCESS
def unregister_service(self, name, pid):
if name not in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
for namespace in ["resource", "control"]:
for type in self.services[name][namespace].keys():
for key in self.services[name][namespace][type].keys():
if namespace == "control":
self._register(name, "control", pid, type, "remove", self.services[name][namespace][type][key])
else:
self._unregister(name, namespace, pid, type, key)
self.services.pop(name, None)
self.update_node()
return RES_SUCCESS
def heartbeat_service(self, name, pid):
if name not in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
return RES_SUCCESS
def cleanup_services(self):
timed_out = time.time() - HEARTBEAT_TIMEOUT
for name in list(self.services.keys()):
if self.services[name]["heartbeat"] < timed_out:
self.unregister_service(name, self.services[name]["pid"])
def register_resource(self, service_name, pid, type, key, value):
if type not in self.permitted_resources:
return RES_UNSUPPORTED
return self._register(service_name, "resource", pid, type, key, value)
def register_control(self, service_name, pid, device_id, control_data):
return self._register(
service_name=service_name,
namespace="control",
pid=pid,
type=device_id,
key="add",
value=control_data
)
def _register(self, service_name, namespace, pid, type, key, value):
if namespace != "control":
if "max_api_version" not in value:
self.logger.writeWarning(
"Service {}: Registration without valid api version specified".format(service_name)
)
value["max_api_version"] = "v1.0"
elif api_ver_compare(value["max_api_version"], NODE_REGVERSION) < 0:
self.logger.writeWarning(
"Trying to register resource with api version too low: '{}' : {}".format(key, json.dumps(value))
)
if service_name not in self.services:
return RES_NOEXISTS
if not self.services[service_name]["pid"] == pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
# Add a node_id to those resources which need one
if type == 'device':
value['node_id'] = self.node_id
if namespace == "control":
if type not in self.services[service_name][namespace]:
# 'type' is the Device ID in this case
self.services[service_name][namespace][type] = {}
if key == "add":
# Register
self.services[service_name][namespace][type][value["href"]] = value
else:
# Unregister
self.services[service_name][namespace][type].pop(value["href"], None)
# Reset the parameters below to force re-registration of the corresponding Device
namespace = "resource"
key = type # Device ID
type = "device"
value = None
for name in self.services: # Find the service which registered the Device in question
if key in self.services[name]["resource"][type]:
value = self.services[name]["resource"][type][key]
break
if not value: # Device isn't actually registered at present
return RES_SUCCESS
else:
self.services[service_name][namespace][type][key] = value
# Don't pass non-registration exceptions to clients
try:
if namespace == "resource":
self._update_mdns(type)
except Exception as e:
self.logger.writeError("Exception registering with mDNS: {}".format(e))
try:
self.aggregator.register_into(namespace, type, key, **self.preprocess_resource(type, key, value,
NODE_REGVERSION))
self.logger.writeDebug("registering {} {}".format(type, key))
except Exception as e:
self.logger.writeError("Exception registering {}: {}".format(namespace, e))
return RES_OTHERERROR
return RES_SUCCESS
def update_resource(self, service_name, pid, type, key, value):
return self.register_resource(service_name, pid, type, key, value)
def find_service(self, type, key):
for service_name in self.services.keys():
if key in self.services[service_name]["resource"][type]:
return service_name
return None
def unregister_resource(self, service_name, pid, type, key):
if type not in self.permitted_resources:
return RES_UNSUPPORTED
return self._unregister(service_name, "resource", pid, type, key)
def unregister_control(self, service_name, pid, device_id, control_data):
# Note use of register here, as we're updating an existing Device
return self._register(service_name, "control", pid, device_id, "remove", control_data)
def _unregister(self, service_name, namespace, pid, type, key):
if service_name not in self.services:
return RES_NOEXISTS
if self.services[service_name]["pid"] != pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
self.services[service_name][namespace][type].pop(key, None)
# Don't pass non-registration exceptions to clients
try:
self.aggregator.unregister_from(namespace, type, key)
except Exception as e:
self.logger.writeError("Exception unregistering {}: {}".format(namespace, e))
return RES_OTHERERROR
try:
if namespace == "resource":
self._update_mdns(type)
except ServiceAlreadyExistsException as e:
# We can't do anything about this, so just return success
self.logger.writeError("Exception unregistering from mDNS: {}".format(e))
except Exception as e:
self.logger.writeError("Exception unregistering from mDNS: {}".format(e))
return RES_OTHERERROR
return RES_SUCCESS
def list_services(self, api_version="v1.0"):
return list(self.services.keys())
def get_service_href(self, name, api_version="v1.0"):
if name not in self.services:
return RES_NOEXISTS
href = self.services[name]["href"]
if self.services[name]["proxy_path"]:
href += "/" + self.services[name]["proxy_path"]
return href
def get_service_type(self, name, api_version="v1.0"):
if name not in self.services:
return RES_NOEXISTS
return self.services[name]["type"]
def preprocess_url(self, url):
parsed_url = urlparse(url)
scheme = parsed_url.scheme
if PROTOCOL == "https":
if scheme == "http":
scheme = "https"
elif scheme == "ws":
scheme = "wss"
netloc = self.node_data["host"]
if parsed_url.port:
netloc += ":{}".format(parsed_url.port)
parsed_url = parsed_url._replace(netloc=netloc, scheme=scheme)
return urlunparse(parsed_url)
def preprocess_resource(self, type, key, value, api_version="v1.0"):
if type == "device":
value_copy = copy.deepcopy(value)
for name in self.services:
if key in self.services[name]["control"] and "controls" in value_copy:
value_copy["controls"] = value_copy["controls"] + list(self.services[name]["control"][key].values())
if "controls" in value_copy:
for control in value_copy["controls"]:
control["href"] = self.preprocess_url(control["href"])
return translate_api_version(value_copy, type, api_version)
elif type == "sender":
value_copy = copy.deepcopy(value)
if "manifest_href" in value_copy:
value_copy["manifest_href"] = self.preprocess_url(value_copy["manifest_href"])
return translate_api_version(value_copy, type, api_version)
else:
return translate_api_version(value, type, api_version)
def list_resource(self, type, api_version="v1.0"):
if type not in self.permitted_resources:
return RES_UNSUPPORTED
response = {}
for name in self.services:
response = (dict(list(response.items()) + [
(k, self.preprocess_resource(type, k, x, api_version))
for (k, x) in self.services[name]["resource"][type].items()
if (api_version == "v1.0" or (
"max_api_version" in x and api_ver_compare(x["max_api_version"], api_version) >= 0
))
]))
return response
def _len_resource(self, type):
response = 0
for name in self.services:
response += len(self.services[name]["resource"][type])
return response
def _update_mdns(self, type):
if type not in self.permitted_resources:
return RES_UNSUPPORTED
if not self.mdns_updater:
return
num_items = self._len_resource(type)
if num_items == 1:
try:
self.mdns_updater.update_mdns(type, "register")
except Exception:
self.mdns_updater.update_mdns(type, "update")
elif num_items == 0:
self.mdns_updater.update_mdns(type, "unregister")
else:
self.mdns_updater.update_mdns(type, "update")
def list_self(self, api_version="v1.0"):
return self.preprocess_resource("node", self.node_data["id"], self.node_data, api_version)
def _ptp_clock(self):
clk = {
"name": "clk1",
"ref_type": "ptp",
"version": "IEEE1588-2008",
"traceable": False,
"gmid": "00-00-00-00-00-00-00-00",
"locked": False,
}
sts = IppClock().PTPStatus()
if len(sts.keys()) > 0:
clk['traceable'] = sts['timeTraceable']
clk['gmid'] = sts['grandmasterClockIdentity'].lower()
clk['locked'] = (sts['ofm'][0] == 0)
return clk
def update_ptp(self):
if IPP_UTILS_CLOCK_AVAILABLE:
old_clk = None
if "clk1" in self.clocks:
old_clk = copy.copy(self.clocks["clk1"])
clk = self._ptp_clock()
if old_clk is None:
self.register_clock(clk)
elif clk != old_clk:
self.update_clock(clk)
def register_clock(self, clk_data):
if "name" not in clk_data:
return RES_OTHERERROR
if clk_data["name"] in self.clocks:
return RES_EXISTS
self.clocks[clk_data["name"]] = clk_data
self.update_node()
return RES_SUCCESS
def update_clock(self, clk_data):
if "name" not in clk_data:
return RES_OTHERERROR
if clk_data["name"] in self.clocks:
self.clocks[clk_data["name"]] = clk_data
self.update_node()
return RES_SUCCESS
return RES_NOEXISTS
def unregister_clock(self, clk_name):
if clk_name in self.clocks:
del self.clocks[clk_name]
self.update_node()
return RES_SUCCESS
return RES_NOEXISTS
class Aggregator(object):
"""This class serves as a proxy for the distant aggregation service running elsewhere on the network.
It will search out aggregators and locate them, falling back to other ones if the one it is connected to
disappears, and resending data as needed."""
def __init__(self, logger=None, mdns_updater=None):
self.logger = Logger("aggregator_proxy", logger)
self.mdnsbridge = IppmDNSBridge(logger=self.logger)
self.aggregator = ""
self.registration_order = [
"device", "source", "flow", "sender", "receiver"
]
self._mdns_updater = mdns_updater
# 'registered' is a local mirror of aggregated items. There are helper methods
# for manipulating this below.
self._registered = {
'node': None,
'registered': False,
'entities': {
'resource': {}
}
}
self._running = True
self._reg_queue = gevent.queue.Queue()
self.heartbeat_thread = gevent.spawn(self._heartbeat)
self.queue_thread = gevent.spawn(self._process_queue)
# The heartbeat thread runs in the background every five seconds.
# If when it runs the Node is believed to be registered it will perform a heartbeat
def _heartbeat(self):
self.logger.writeDebug("Starting heartbeat thread")
while self._running:
heartbeat_wait = 5
if not self._registered["registered"]:
self._process_reregister()
elif self._registered["node"]:
# Do heartbeat
try:
self.logger.writeDebug(
"Sending heartbeat for Node {}".format(
self._registered["node"]["data"]["id"]))
self._SEND(
"POST", "/health/nodes/" +
self._registered["node"]["data"]["id"])
except InvalidRequest as e:
if e.status_code == 404:
# Re-register
self.logger.writeWarning(
"404 error on heartbeat. Marking Node for re-registration"
)
self._registered["registered"] = False
if (self._mdns_updater is not None):
self._mdns_updater.inc_P2P_enable_count()
else:
# Client side error. Report this upwards via exception, but don't resend
self.logger.writeError(
"Unrecoverable error code {} received from Registration API on heartbeat"
.format(e.status_code))
self._running = False
except:
# Re-register
self.logger.writeWarning(
"Unexpected error on heartbeat. Marking Node for re-registration"
)
self._registered["registered"] = False
else:
self._registered["registered"] = False
if (self._mdns_updater is not None):
self._mdns_updater.inc_P2P_enable_count()
while heartbeat_wait > 0 and self._running:
gevent.sleep(1)
heartbeat_wait -= 1
self.logger.writeDebug("Stopping heartbeat thread")
# Provided the Node is believed to be correctly registered, hand off a single request to the SEND method
# On client error, clear the resource from the local mirror
# On other error, mark Node as unregistered and trigger re-registration
def _process_queue(self):
self.logger.writeDebug("Starting HTTP queue processing thread")
while self._running or (
self._registered["registered"] and not self._reg_queue.empty()
): # Checks queue not empty before quitting to make sure unregister node gets done
if not self._registered["registered"] or self._reg_queue.empty():
gevent.sleep(1)
else:
try:
queue_item = self._reg_queue.get()
namespace = queue_item["namespace"]
res_type = queue_item["res_type"]
res_key = queue_item["key"]
if queue_item["method"] == "POST":
if res_type == "node":
data = self._registered["node"]
try:
self.logger.writeInfo(
"Attempting registration for Node {}".
format(self._registered["node"]["data"]
["id"]))
self._SEND("POST", "/{}".format(namespace),
data)
self._SEND(
"POST", "/health/nodes/" +
self._registered["node"]["data"]["id"])
self._registered["registered"] = True
if self._mdns_updater is not None:
self._mdns_updater.P2P_disable()
except Exception as ex:
self.logger.writeWarning(
"Error registering Node: %r" %
(traceback.format_exc(), ))
elif res_key in self._registered["entities"][
namespace][res_type]:
data = self._registered["entities"][namespace][
res_type][res_key]
try:
self._SEND("POST", "/{}".format(namespace),
data)
except InvalidRequest as e:
self.logger.writeWarning(
"Error registering {} {}: {}".format(
res_type, res_key, e))
self.logger.writeWarning(
"Request data: {}".format(
self._registered["entities"][namespace]
[res_type][res_key]))
del self._registered["entities"][namespace][
res_type][res_key]
elif queue_item["method"] == "DELETE":
translated_type = res_type + 's'
try:
self._SEND(
"DELETE",
"/{}/{}/{}".format(namespace, translated_type,
res_key))
except InvalidRequest as e:
self.logger.writeWarning(
"Error deleting resource {} {}: {}".format(
translated_type, res_key, e))
else:
self.logger.writeWarning(
"Method {} not supported for Registration API interactions"
.format(queue_item["method"]))
except Exception as e:
self._registered["registered"] = False
if (self._mdns_updater is not None):
self._mdns_updater.P2P_disable()
self.logger.writeDebug("Stopping HTTP queue processing thread")
# Queue a request to be processed. Handles all requests except initial Node POST which is done in _process_reregister
def _queue_request(self, method, namespace, res_type, key):
self._reg_queue.put({
"method": method,
"namespace": namespace,
"res_type": res_type,
"key": key
})
# Register 'resource' type data including the Node
# NB: Node registration is managed by heartbeat thread so may take up to 5 seconds!
def register(self, res_type, key, **kwargs):
self.register_into("resource", res_type, key, **kwargs)
# Unregister 'resource' type data including the Node
def unregister(self, res_type, key):
self.unregister_from("resource", res_type, key)
# General register method for 'resource' types
def register_into(self, namespace, res_type, key, **kwargs):
data = kwargs
send_obj = {"type": res_type, "data": data}
if 'id' not in send_obj["data"]:
self.logger.writeWarning(
"No 'id' present in data, using key='{}': {}".format(
key, data))
send_obj["data"]["id"] = key
if namespace == "resource" and res_type == "node":
# Handle special Node type
self._registered["node"] = send_obj
else:
self._add_mirror_keys(namespace, res_type)
self._registered["entities"][namespace][res_type][key] = send_obj
self._queue_request("POST", namespace, res_type, key)
# General unregister method for 'resource' types
def unregister_from(self, namespace, res_type, key):
if namespace == "resource" and res_type == "node":
# Handle special Node type
self._registered["node"] = None
elif res_type in self._registered["entities"][namespace]:
self._add_mirror_keys(namespace, res_type)
if key in self._registered["entities"][namespace][res_type]:
del self._registered["entities"][namespace][res_type][key]
self._queue_request("DELETE", namespace, res_type, key)
# Deal with missing keys in local mirror
def _add_mirror_keys(self, namespace, res_type):
if namespace not in self._registered["entities"]:
self._registered["entities"][namespace] = {}
if res_type not in self._registered["entities"][namespace]:
self._registered["entities"][namespace][res_type] = {}
# Re-register just the Node, and queue requests in order for other resources
def _process_reregister(self):
if self._registered.get("node", None) is None:
self.logger.writeDebug("No node registered, re-register returning")
return
try:
self.logger.writeDebug(
"Clearing old Node from API prior to re-registration")
self._SEND(
"DELETE",
"/resource/nodes/" + self._registered["node"]["data"]["id"])
except InvalidRequest as e:
# 404 etc is ok
self.logger.writeInfo(
"Invalid request when deleting Node prior to registration: {}".
format(e))
except Exception as ex:
# Server error is bad, no point continuing
self.logger.writeError("Aborting Node re-register! {}".format(ex))
return
self._registered["registered"] = False
if (self._mdns_updater is not None):
self._mdns_updater.inc_P2P_enable_count()
# Drain the queue
while not self._reg_queue.empty():
try:
self._reg_queue.get(block=False)
except gevent.queue.Queue.Empty:
break
try:
# Register the node, and immediately heartbeat if successful to avoid race with garbage collect.
self.logger.writeInfo(
"Attempting re-registration for Node {}".format(
self._registered["node"]["data"]["id"]))
self._SEND("POST", "/resource", self._registered["node"])
self._SEND(
"POST",
"/health/nodes/" + self._registered["node"]["data"]["id"])
self._registered["registered"] = True
if self._mdns_updater is not None:
self._mdns_updater.P2P_disable()
except Exception as e:
self.logger.writeWarning("Error re-registering Node: {}".format(e))
self.aggregator == "" # Fallback to prevent us getting stuck if the Reg API issues a 4XX error incorrectly
return
# Re-register items that must be ordered
# Re-register things we have in the local cache.
# "namespace" is e.g. "resource"
# "entities" are the things associated under that namespace.
for res_type in self.registration_order:
for namespace, entities in self._registered["entities"].items():
if res_type in entities:
self.logger.writeInfo(
"Ordered re-registration for type: '{}' in namespace '{}'"
.format(res_type, namespace))
for key in entities[res_type]:
self._queue_request("POST", namespace, res_type, key)
# Re-register everything else
# Re-register things we have in the local cache.
# "namespace" is e.g. "resource"
# "entities" are the things associated under that namespace.
for namespace, entities in self._registered["entities"].items():
for res_type in entities:
if res_type not in self.registration_order:
self.logger.writeInfo(
"Unordered re-registration for type: '{}' in namespace '{}'"
.format(res_type, namespace))
for key in entities[res_type]:
self._queue_request("POST", namespace, res_type, key)
# Stop the Aggregator object running
def stop(self):
self.logger.writeDebug("Stopping aggregator proxy")
self._running = False
self.heartbeat_thread.join()
self.queue_thread.join()
# Handle sending all requests to the Registration API, and searching for a new 'aggregator' if one fails
def _SEND(self, method, url, data=None):
if self.aggregator == "":
self.aggregator = self.mdnsbridge.getHref(REGISTRATION_MDNSTYPE)
if data is not None:
data = json.dumps(data)
url = AGGREGATOR_APINAMESPACE + "/" + AGGREGATOR_APINAME + "/" + AGGREGATOR_APIVERSION + url
for i in range(0, 3):
if self.aggregator == "":
self.logger.writeWarning(
"No aggregator available on the network or mdnsbridge unavailable"
)
raise NoAggregator(self._mdns_updater)
self.logger.writeDebug("{} {}".format(
method, urljoin(self.aggregator, url)))
# We give a long(ish) timeout below, as the async request may succeed after the timeout period
# has expired, causing the node to be registered twice (potentially at different aggregators).
# Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic
# to web clients - so, sacrifice a little timeliness for things working as designed the
# majority of the time...
try:
if nmoscommonconfig.config.get('prefer_ipv6', False) == False:
R = requests.request(method,
urljoin(self.aggregator, url),
data=data,
timeout=1.0)
else:
R = requests.request(method,
urljoin(self.aggregator, url),
data=data,
timeout=1.0,
proxies={'http': ''})
if R is None:
# Try another aggregator
self.logger.writeWarning(
"No response from aggregator {}".format(
self.aggregator))
elif R.status_code in [200, 201]:
if R.headers.get(
"content-type",
"text/plain").startswith("application/json"):
return R.json()
else:
return R.content
elif R.status_code == 204:
return
elif (R.status_code / 100) == 4:
self.logger.writeWarning(
"{} response from aggregator: {} {}".format(
R.status_code, method,
urljoin(self.aggregator, url)))
raise InvalidRequest(R.status_code, self._mdns_updater)
else:
self.logger.writeWarning(
"Unexpected status from aggregator {}: {}, {}".format(
self.aggregator, R.status_code, R.content))
except requests.exceptions.RequestException as ex:
# Log a warning, then let another aggregator be chosen
self.logger.writeWarning("{} from aggregator {}".format(
ex, self.aggregator))
# This aggregator is non-functional
self.aggregator = self.mdnsbridge.getHref(REGISTRATION_MDNSTYPE)
self.logger.writeInfo("Updated aggregator to {} (try {})".format(
self.aggregator, i))
raise TooManyRetries(self._mdns_updater)
class FacadeInterface(object):
def __init__(self, registry, logger):
self.host = Host(ADDRESS)
self.registry = registry
self.logger = Logger("facade_interface", logger)
def getbases(cl):
bases = list(cl.__bases__)
for x in cl.__bases__:
bases += getbases(x)
return bases
for cl in [
self.__class__,
] + getbases(self.__class__):
for name in cl.__dict__.keys():
value = getattr(self, name)
if callable(value):
if hasattr(value, "ipc_method"):
self.host.ipcmethod(name)(value)
def start(self):
self.host.start()
def stop(self):
self.host.stop()
@ipcmethod
def srv_register(self, name, srv_type, pid, href, proxy_path):
self.logger.writeInfo("Service Register {}, {}, {}, {}, {}".format(
name, srv_type, pid, href, proxy_path))
return self.registry.register_service(name, srv_type, pid, href,
proxy_path)
#TODO: =None should be removed once proxying removed from node facade
@ipcmethod
def srv_update(self, name, pid, href, proxy_path):
self.logger.writeInfo("Service Update {}, {}, {}, {}".format(
name, pid, href, proxy_path))
return self.registry.update_service(name, pid, href, proxy_path)
@ipcmethod
def srv_unregister(self, name, pid):
self.logger.writeInfo("Service Unregister {}, {}".format(name, pid))
return self.registry.unregister_service(name, pid)
@ipcmethod
def srv_heartbeat(self, name, pid):
self.logger.writeDebug("Service Heartbeat {}, {}".format(name, pid))
return self.registry.heartbeat_service(name, pid)
@ipcmethod
def res_register(self, name, pid, type, key, value):
self.logger.writeInfo("Resource Register {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.register_resource(name, pid, type, key, value)
@ipcmethod
def res_update(self, name, pid, type, key, value):
self.logger.writeInfo("Resource Update {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.update_resource(name, pid, type, key, value)
@ipcmethod
def res_unregister(self, name, pid, type, key):
self.logger.writeInfo("Resource Unregister {} {} {} {}".format(
name, pid, type, key))
return self.registry.unregister_resource(name, pid, type, key)
@ipcmethod
def control_register(self, name, pid, device_id, control_data):
self.logger.writeInfo("Control Register {} {} {} {}".format(
name, pid, device_id, control_data))
return self.registry.register_control(name, pid, device_id,
control_data)
@ipcmethod
def control_unregister(self, name, pid, device_id, control_data):
self.logger.writeInfo("Control Unregister {} {} {} {}".format(
name, pid, device_id, control_data))
return self.registry.unregister_control(name, pid, device_id,
control_data)
@ipcmethod
def self_get(self, name, pid, api_version):
return self.registry.list_self(api_version)
@ipcmethod
def status_get(self, name, pid):
return self.registry.aggregator.status()
@ipcmethod
def clock_register(self, name, pid, clk_data):
self.logger.writeInfo("Clock Register {} {}".format(name, pid))
return self.registry.register_clock(clk_data)
@ipcmethod
def clock_update(self, name, pid, clk_data):
self.logger.writeInfo("Clock Update {} {}".format(name, pid))
return self.registry.update_clock(clk_data)
@ipcmethod
def clock_unregister(self, name, pid, clk_name):
self.logger.writeInfo("Clock Unregister {} {}".format(name, pid))
return self.registry.unregister_clock(clk_name)
class Aggregator(object):
"""This class serves as a proxy for the distant aggregation service running elsewhere on the network.
It will search out aggregators and locate them, falling back to other ones if the one it is connected to
disappears, and resending data as needed."""
def __init__(self, logger=None, mdns_updater=None, auth_registry=None):
self.logger = Logger("aggregator_proxy", logger)
self.mdnsbridge = IppmDNSBridge(logger=self.logger)
self.aggregator_apiversion = None
self.service_type = None
self._set_api_version_and_srv_type(
_config.get('nodefacade').get('NODE_REGVERSION'))
self.aggregator = None
self.registration_order = [
"device", "source", "flow", "sender", "receiver"
]
self._mdns_updater = mdns_updater
# '_node_data' is a local mirror of aggregated items.
self._node_data = {
'node': None,
'registered': False,
'entities': {
'resource': {}
}
}
self._running = True
self._aggregator_list_stale = True
self._aggregator_failure = False # Variable to flag when aggregator has returned and unexpected error
self._backoff_active = False
self._backoff_period = 0
self.auth_registrar = None # Class responsible for registering with Auth Server
self.auth_registry = auth_registry # Top level class that tracks locally registered OAuth clients
self.auth_client = None # Instance of Oauth client responsible for performing token requests
self._reg_queue = gevent.queue.Queue()
self.main_thread = gevent.spawn(self._main_thread)
self.queue_thread = gevent.spawn(self._process_queue)
def _set_api_version_and_srv_type(self, api_ver):
"""Set the aggregator api version equal to parameter and DNS-SD service type based on api version"""
self.aggregator_apiversion = api_ver
self._set_service_type(api_ver)
def _set_service_type(self, api_ver):
"""Set DNS-SD service type based on current api version in use"""
if api_ver in ['v1.0', 'v1.1', 'v1.2']:
self.service_type = LEGACY_REG_MDNSTYPE
else:
self.service_type = REGISTRATION_MDNSTYPE
def _main_thread(self):
"""The main thread runs in the background.
If, when it runs, the Node is believed to be registered it will perform a heartbeat every 5 seconds.
If the Node is not registered it will try to register the Node"""
self.logger.writeDebug("Starting main thread")
while self._running:
if self._node_data["node"] and self.aggregator is None:
self._discovery_operation()
elif self._node_data["node"] and self._node_data["registered"]:
self._registered_operation()
else:
self._node_data["registered"] = False
self.aggregator = None
gevent.sleep(0.2)
self.logger.writeDebug("Stopping heartbeat thread")
def _discovery_operation(self):
"""In Discovery operation the Node will wait a backoff period if defined to allow aggregators to recover when in
a state of error. Selecting the most appropriate aggregator and try to register with it.
If a registration fails then another aggregator will be tried."""
self.logger.writeDebug("Entering Discovery Mode")
# Wait backoff period
# Do not wait backoff period if aggregator failed, a new aggregator should be tried immediately
if not self._aggregator_failure:
self._back_off_timer()
self._aggregator_failure = False
# Update cached list of aggregators
if self._aggregator_list_stale:
self._flush_cached_aggregators()
while True:
self.aggregator = self._get_aggregator()
if self.aggregator is None:
self.logger.writeDebug("Failed to find registration API")
break
self.logger.writeDebug("Aggregator set to: {}".format(
self.aggregator))
# Perform initial heartbeat, which will attempt to register Node if not already registered
if self._heartbeat():
# Successfully registered Node with aggregator andproceed to registered operation
# Else will try next aggregator
break
def _registered_operation(self):
"""In Registered operation, the Node is registered so a heartbeat will be performed,
if the heartbeat is successful the Node will wait 5 seconds before attempting another heartbeat.
Else another aggregator will be selected"""
if not self._heartbeat():
# Heartbeat failed
# Flag to update cached list of aggregators and immediately try new aggregator
self.aggregator = None
self._aggregator_failure = True
def _heartbeat(self):
"""Performs a heartbeat to registered aggregator
If heartbeat fails it will take actions to correct the error, by re-registering the Node
If successfull will return True, else will return False"""
if not self.aggregator:
return False
try:
R = self._send(
"POST", self.aggregator, self.aggregator_apiversion,
"health/nodes/{}".format(
self._node_data["node"]["data"]["id"]))
if R.status_code == 200 and self._node_data["registered"]:
# Continue to registered operation
self.logger.writeDebug(
"Successful heartbeat for Node {}".format(
self._node_data["node"]["data"]["id"]))
self._registered()
heartbeat_wait = 5
while heartbeat_wait > 0 and self._running:
gevent.sleep(1)
heartbeat_wait -= 1
return True
elif R.status_code in [200, 409]:
# Delete node from registry
if self._unregister_node(R.headers.get('Location')):
return self._register_node(self._node_data["node"])
else:
# Try next registry
return False
except InvalidRequest as e:
if e.status_code == 404:
# Re-register
self.logger.writeWarning(
"404 error on heartbeat. Marking Node for re-registration")
self._node_data["registered"] = False
return self._register_node(self._node_data["node"])
else:
# Other error, try next registry
return False
except ServerSideError:
self.logger.writeWarning(
"Server Side Error on heartbeat. Trying another registry")
return False
except Exception as e:
# Re-register
self.logger.writeWarning(
"Unexpected error on heartbeat: {}. Marking Node for re-registration"
.format(e))
self._node_data["registered"] = False
return False
def _register_auth(self, client_name, client_uri):
"""Register OAuth client with Authorization Server"""
self.logger.writeInfo(
"Attempting to register dynamically with Auth Server")
auth_registrar = AuthRegistrar(client_name=client_name,
redirect_uri=PROTOCOL + '://' + FQDN +
NODE_APIROOT + 'authorize',
client_uri=client_uri,
allowed_scope=ALLOWED_SCOPE,
allowed_grant=ALLOWED_GRANTS)
if auth_registrar.registered is True:
return auth_registrar
else:
self.logger.writeWarning(
"Unable to successfully register with Authorization Server")
def _register_node(self, node_obj):
"""Attempt to register Node with aggregator
Returns True is node was successfully registered with aggregator
Returns False if registration failed
If registration failed with 200 or 409, will attempt to delete and re-register"""
if node_obj is None:
return False
# Drain the queue
while not self._reg_queue.empty():
try:
self._reg_queue.get(block=False)
except gevent.queue.Queue.Empty:
break
try:
# Try register the Node 3 times with aggregator before failing back to next aggregator
for i in range(0, 3):
R = self._send("POST", self.aggregator,
self.aggregator_apiversion, "resource",
node_obj)
if R.status_code == 201:
# Continue to registered operation
self.logger.writeInfo(
"Node Registered with {} at version {}".format(
self.aggregator, self.aggregator_apiversion))
self._registered()
# Trigger registration of Nodes resources
self._register_node_resources()
return True
elif R.status_code in [200, 409]:
# Delete node from aggregator & re-register
if self._unregister_node(R.headers.get('Location')):
continue
else:
# Try next aggregator
return False
except Exception as e:
self.logger.writeError("Failed to register node {}".format(e))
return False
def _register_node_resources(self):
# Re-register items that must be ordered
# Re-register things we have in the local cache.
# "namespace" is e.g. "resource"
# "entities" are the things associated under that namespace.
for res_type in self.registration_order:
for namespace, entities in self._node_data["entities"].items():
if res_type in entities:
self.logger.writeInfo(
"Ordered re-registration for type: '{}' in namespace '{}'"
.format(res_type, namespace))
for key in entities[res_type]:
self._queue_request("POST", namespace, res_type, key)
# Re-register everything else
# Re-register things we have in the local cache.
# "namespace" is e.g. "resource"
# "entities" are the things associated under that namespace.
for namespace, entities in self._node_data["entities"].items():
for res_type in entities:
if res_type not in self.registration_order:
self.logger.writeInfo(
"Unordered re-registration for type: '{}' in namespace '{}'"
.format(res_type, namespace))
for key in entities[res_type]:
self._queue_request("POST", namespace, res_type, key)
def _registered(self):
"""Mark Node as registered and reset counters"""
if (self._mdns_updater is not None):
self._mdns_updater.P2P_disable()
self._node_data['registered'] = True
self._aggregator_list_stale = True
self._reset_backoff_period()
def _reset_backoff_period(self):
self.logger.writeDebug("Resetting backoff period")
self._backoff_period = 0
def _increase_backoff_period(self):
"""Exponentially increase the backoff period, until set maximum reached"""
self.logger.writeDebug("Increasing backoff period")
self._aggregator_list_stale = True
if self._backoff_period == 0:
self._backoff_period = BACKOFF_INITIAL_TIMOUT_SECONDS
return
self._backoff_period *= 2
if self._backoff_period > BACKOFF_MAX_TIMEOUT_SECONDS:
self._backoff_period = BACKOFF_MAX_TIMEOUT_SECONDS
def _back_off_timer(self):
"""Sleep for defined backoff period"""
self.logger.writeDebug("Backoff timer enabled for {} seconds".format(
self._backoff_period))
self._backoff_active = True
gevent.sleep(self._backoff_period)
self._backoff_active = False
def _flush_cached_aggregators(self):
"""Flush the list of cached aggregators in the mdns bridge client,
preventing the use of out of date aggregators"""
self.logger.writeDebug("Flushing cached list of aggregators")
self._aggregator_list_stale = False
self.mdnsbridge.updateServices(self.service_type)
def _get_aggregator(self):
"""Get the most appropriate aggregator from the mdns bridge client.
If no aggregator found increment P2P counter, update cache and increase backoff
If reached the end of available aggregators update cache and increase backoff"""
try:
return self.mdnsbridge.getHrefWithException(
self.service_type, None, self.aggregator_apiversion, PROTOCOL,
OAUTH_MODE)
except NoService:
self.logger.writeDebug(
"No Registration services found: {} {} {}".format(
self.service_type, self.aggregator_apiversion, PROTOCOL))
if self._mdns_updater is not None:
self._mdns_updater.inc_P2P_enable_count()
self._increase_backoff_period()
return None
except EndOfServiceList:
self.logger.writeDebug(
"End of Registration services list: {} {} {}".format(
self.service_type, self.aggregator_apiversion, PROTOCOL))
self._increase_backoff_period()
return None
def _unregister_node(self, url_path=None):
"""Delete node from registry, using url_path if specified"""
if self.aggregator is None:
self.logger.writeWarning(
'Could not un-register as no aggregator set')
return False
try:
self._node_data['registered'] = False
if url_path is None:
R = self._send(
'DELETE', self.aggregator, self.aggregator_apiversion,
'resource/nodes/{}'.format(
self._node_data['node']["data"]["id"]))
else:
parsed_url = urlparse(url_path)
R = self._send_request('DELETE', self.aggregator,
parsed_url.path)
if R.status_code == 204:
# Successfully deleted node from Registry
self.logger.writeInfo(
"Node unregistered from {} at version {}".format(
self.aggregator, self.aggregator_apiversion))
return True
else:
return False
except Exception as e:
self.logger.writeDebug(
'Exception raised while un-registering {}'.format(e))
return False
def _process_queue(self):
"""Provided the Node is believed to be correctly registered, hand off a single request to the SEND method
On client error, clear the resource from the local mirror
On other error, mark Node as unregistered and trigger re-registration"""
self.logger.writeDebug("Starting HTTP queue processing thread")
# Checks queue not empty before quitting to make sure unregister node gets done
while self._running:
if (not self._node_data["registered"] or self._reg_queue.empty()
or self._backoff_active or not self.aggregator):
gevent.sleep(1)
else:
try:
queue_item = self._reg_queue.get()
namespace = queue_item["namespace"]
res_type = queue_item["res_type"]
res_key = queue_item["key"]
if queue_item["method"] == "POST":
if res_type == "node":
send_obj = self._node_data.get("node")
else:
send_obj = self._node_data["entities"][namespace][
res_type].get(res_key)
if send_obj is None:
self.logger.writeError(
"No data to send for resource {}".format(
res_type))
continue
try:
self._send("POST", self.aggregator,
self.aggregator_apiversion,
"{}".format(namespace), send_obj)
self.logger.writeInfo("Registered {} {} {}".format(
namespace, res_type, res_key))
except InvalidRequest as e:
self.logger.writeWarning(
"Error registering {} {}: {}".format(
res_type, res_key, e))
self.logger.writeWarning(
"Request data: {}".format(send_obj))
del self._node_data["entities"][namespace][
res_type][res_key]
elif queue_item["method"] == "DELETE":
translated_type = res_type + 's'
if namespace == "resource" and res_type == "node":
# Handle special Node type
self._node_data["node"] = None
self._node_data["registered"] = False
try:
self._send(
"DELETE", self.aggregator,
self.aggregator_apiversion,
"{}/{}/{}".format(namespace, translated_type,
res_key))
self.logger.writeInfo(
"Un-registered {} {} {}".format(
namespace, translated_type, res_key))
except InvalidRequest as e:
self.logger.writeWarning(
"Error deleting resource {} {}: {}".format(
translated_type, res_key, e))
else:
self.logger.writeWarning(
"Method {} not supported for Registration API interactions"
.format(queue_item["method"]))
except ServerSideError:
self.aggregator = None
self._aggregator_failure = True
self._add_request_to_front_of_queue(queue_item)
except Exception as e:
self.logger.writeError(
"Unexpected Error while processing queue, marking Node for re-registration\n"
"{}".format(e))
self._node_data["registered"] = False
self.aggregator = None
if (self._mdns_updater is not None):
self._mdns_updater.P2P_disable()
self.logger.writeDebug("Stopping HTTP queue processing thread")
def _queue_request(self, method, namespace, res_type, key):
"""Queue a request to be processed.
Handles all requests except initial Node POST which is done in _process_reregister"""
self._reg_queue.put({
"method": method,
"namespace": namespace,
"res_type": res_type,
"key": key
})
def _add_request_to_front_of_queue(self, request):
"""Adds item to the front of the queue"""
new_queue = deque()
new_queue.append(request)
# Drain the queue
while not self._reg_queue.empty():
try:
new_queue.append(self._reg_queue.get())
except gevent.queue.Queue.Empty:
break
# Add items back to the queue
while True:
try:
self._reg_queue.put(new_queue.popleft())
except IndexError:
break
def register_auth_client(self, client_name, client_uri):
"""Function for Registering OAuth client with Auth Server and instantiating OAuth Client class"""
if OAUTH_MODE is True:
if self.auth_registrar is None:
self.auth_registrar = self._register_auth(
client_name=client_name, client_uri=client_uri)
if self.auth_registrar and self.auth_client is None:
try:
# Register Node Client
self.auth_registry.register_client(
client_name=client_name,
client_uri=client_uri,
**self.auth_registrar.server_metadata)
self.logger.writeInfo(
"Successfully registered Auth Client")
except (OSError, IOError):
self.logger.writeError(
"Exception accessing OAuth credentials. This may be a file permissions issue."
)
return
# Extract the 'RemoteApp' class created when registering
self.auth_client = getattr(self.auth_registry, client_name)
# Fetch Token
self.get_auth_token()
def get_auth_token(self):
"""Fetch Access Token either using redirection grant flow or using auth_client"""
if self.auth_client is not None and self.auth_registrar is not None:
try:
if "authorization_code" in self.auth_registrar.client_metadata.get(
"grant_types", {}):
self.logger.writeInfo(
"Endpoint '/oauth' on Node API will provide redirect to authorization endpoint on Auth Server."
)
return
elif "client_credentials" in self.auth_registrar.client_metadata.get(
"grant_types", {}):
# Fetch Token
token = self.auth_client.fetch_access_token()
# Store token in member variable to be extracted using `fetch_local_token` function
self.auth_registry.bearer_token = token
else:
raise OAuth2Error(
"Client not registered with supported Grant Type")
except OAuth2Error as e:
self.logger.writeError(
"Failure fetching access token. {}".format(e))
def register(self, res_type, key, **kwargs):
"""Register 'resource' type data including the Node
NB: Node registration is managed by heartbeat thread so may take up to 5 seconds! """
self.register_into("resource", res_type, key, **kwargs)
def unregister(self, res_type, key):
"""Unregister 'resource' type data including the Node"""
self.unregister_from("resource", res_type, key)
def register_into(self, namespace, res_type, key, **kwargs):
"""General register method for 'resource' types"""
data = kwargs
send_obj = {"type": res_type, "data": data}
if 'id' not in send_obj["data"]:
self.logger.writeWarning(
"No 'id' present in data, using key='{}': {}".format(
key, data))
send_obj["data"]["id"] = key
if namespace == "resource" and res_type == "node":
# Ensure Registered with Auth Server (is there a better place for this)
if OAUTH_MODE is True:
self.register_auth_client("nmos-node-{}".format(data["id"]),
FQDN)
# Handle special Node type when Node is not registered, by immediately registering
if self._node_data["node"] is None:
# Will trigger registration in main thread
self._node_data["node"] = send_obj
return
# Update Node Data
self._node_data["node"] = send_obj
else:
self._add_mirror_keys(namespace, res_type)
self._node_data["entities"][namespace][res_type][key] = send_obj
self._queue_request("POST", namespace, res_type, key)
def unregister_from(self, namespace, res_type, key):
"""General unregister method for 'resource' types"""
if namespace == "resource" and res_type == "node":
# Handle special Node type
self._unregister_node()
self._node_data["node"] = None
return
elif res_type in self._node_data["entities"][namespace]:
self._add_mirror_keys(namespace, res_type)
if key in self._node_data["entities"][namespace][res_type]:
del self._node_data["entities"][namespace][res_type][key]
self._queue_request("DELETE", namespace, res_type, key)
def _add_mirror_keys(self, namespace, res_type):
"""Deal with missing keys in local mirror"""
if namespace not in self._node_data["entities"]:
self._node_data["entities"][namespace] = {}
if res_type not in self._node_data["entities"][namespace]:
self._node_data["entities"][namespace][res_type] = {}
def stop(self):
"""Stop the Aggregator object running"""
self.logger.writeDebug("Stopping aggregator proxy")
self._running = False
self.main_thread.join()
self.queue_thread.join()
def status(self):
"""Return the current status of node in the aggregator"""
return {
"api_href": self.aggregator,
"api_version": self.aggregator_apiversion,
"registered": self._node_data["registered"]
}
def _send(self, method, aggregator, api_ver, url, data=None):
"""Handle sending request to the registration API, with error handling
HTTP 200, 201, 204, 409 - Success, return response
Timeout, HTTP 5xx, Connection Error - Raise ServerSideError Exception
HTTP 4xx - Raise InvalidRequest Exception"""
url = "{}/{}/{}".format(AGGREGATOR_APIROOT, api_ver, url)
try:
resp = self._send_request(method, aggregator, url, data)
if resp is None:
self.logger.writeWarning(
"No response from aggregator {}".format(aggregator))
raise ServerSideError
elif resp.status_code in [200, 201, 204, 409]:
return resp
elif (resp.status_code // 100) == 4:
self.logger.writeWarning(
"{} response from aggregator: {} {}".format(
resp.status_code, method, urljoin(aggregator, url)))
self.logger.writeDebug("\nResponse: {}".format(resp.content))
raise InvalidRequest(resp.status_code)
else:
self.logger.writeWarning(
"Unexpected status from aggregator {}: {}, {}".format(
aggregator, resp.status_code, resp.content))
raise ServerSideError
except requests.exceptions.RequestException as e:
# Log a warning, then let another aggregator be chosen
self.logger.writeWarning("{} from aggregator {}".format(
e, aggregator))
raise ServerSideError
def _send_request(self, method, aggregator, url_path, data=None):
"""Low level method to send a HTTP request"""
url = urljoin(aggregator, url_path)
self.logger.writeDebug("{} {}".format(method, url))
# We give a long(ish) timeout below, as the async request may succeed after the timeout period
# has expired, causing the node to be registered twice (potentially at different aggregators).
# Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic
# to web clients - so, sacrifice a little timeliness for things working as designed the
# majority of the time...
kwargs = {"method": method, "url": url, "json": data, "timeout": 1.0}
if _config.get('prefer_ipv6') is True:
kwargs["proxies"] = {'http': ''}
# If not in OAuth mode, perform standard request
if OAUTH_MODE is False or self.auth_client is None:
return requests.request(**kwargs)
else:
# If in OAuth Mode, use OAuth client to automatically fetch token / refresh token if expired
with self.auth_registry.app.app_context():
try:
return self.auth_client.request(**kwargs)
# General OAuth Error (e.g. incorrect request details, invalid client, etc.)
except OAuth2Error as e:
self.logger.writeError(
"Failed to fetch token before making API call to {}. {}"
.format(url, e))
self.auth_registrar = self.auth_client = None
class QueryService:
def __init__(self, logger=None):
self.running = False
self.logger = Logger("regquery")
self.logger.writeDebug('Running QueryService')
self.config = {"priority": 0}
self._load_config()
self.mdns = MDNSEngine()
self.httpServer = HttpServer(QueryServiceAPI,
WS_PORT,
'0.0.0.0',
api_args=[self.logger])
def start(self):
if self.running:
gevent.signal(signal.SIGINT, self.sig_handler)
gevent.signal(signal.SIGTERM, self.sig_handler)
self.running = True
self.mdns.start()
HOST = getLocalIP()
self.logger.writeDebug('Running web socket server on %i' % WS_PORT)
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeDebug('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeDebug("Running on port: {}".format(
self.httpServer.port))
priority = self.config["priority"]
if not str(priority).isdigit() or priority < 100:
priority = 0
self.mdns.register('query_' + str(HOST), '_nmos-query._tcp',
DNS_SD_PORT, {
"pri": priority,
"api_ver": "v1.0,v1.1,v1.2",
"api_proto": "http"
})
def run(self):
self.running = True
self.start()
while self.running:
time.sleep(1)
self._cleanup()
def _cleanup(self):
self.mdns.stop()
self.mdns.close()
self.httpServer.stop()
def sig_handler(self):
self.stop()
def stop(self):
self.running = False
def _load_config(self):
try:
config_file = "/etc/nmos-query/config.json"
if os.path.isfile(config_file):
f = open(config_file, 'r')
extra_config = json.loads(f.read())
self.config.update(extra_config)
except Exception as e:
self.logger.writeDebug("Exception loading config: {}".format(e))
class FacadeInterface(object):
def __init__(self, registry, logger):
self.host = Host(ADDRESS)
self.registry = registry
self.logger = Logger("facade_interface", logger)
def getbases(cl):
bases = list(cl.__bases__)
for x in cl.__bases__:
bases += getbases(x)
return bases
for cl in [
self.__class__,
] + getbases(self.__class__):
for name in cl.__dict__.keys():
value = getattr(self, name)
if callable(value):
if hasattr(value, "ipc_method"):
self.host.ipcmethod(name)(value)
def start(self):
self.host.start()
def stop(self):
self.host.stop()
@ipcmethod
def srv_register(self, name, srv_type, pid, href, proxy_path):
self.logger.writeInfo("Service Register {}, {}, {}, {}, {}".format(
name, srv_type, pid, href, proxy_path))
return self.registry.register_service(name, srv_type, pid, href,
proxy_path)
@ipcmethod
def srv_update(self, name, pid, href, proxy_path):
self.logger.writeInfo("Service Update {}, {}, {}, {}".format(
name, pid, href, proxy_path))
return self.registry.update_service(name, pid, href, proxy_path)
@ipcmethod
def srv_unregister(self, name, pid):
self.logger.writeInfo("Service Unregister {}, {}".format(name, pid))
return self.registry.unregister_service(name, pid)
@ipcmethod
def srv_heartbeat(self, name, pid):
self.logger.writeDebug("Service Heartbeat {}, {}".format(name, pid))
return self.registry.heartbeat_service(name, pid)
@ipcmethod
def res_register(self, name, pid, type, key, value):
self.logger.writeInfo("Resource Register {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.register_resource(name, pid, type, key, value)
@ipcmethod
def res_update(self, name, pid, type, key, value):
self.logger.writeInfo("Resource Update {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.update_resource(name, pid, type, key, value)
@ipcmethod
def res_unregister(self, name, pid, type, key):
self.logger.writeInfo("Resource Unregister {} {} {} {}".format(
name, pid, type, key))
return self.registry.unregister_resource(name, pid, type, key)
@ipcmethod
def timeline_register(self, name, pid, type, key, value):
self.logger.writeInfo("Timeline Register {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.register_to_timeline(name, pid, type, key, value)
@ipcmethod
def timeline_update(self, name, pid, type, key, value):
self.logger.writeInfo("Timeline Update {} {} {} {} {}".format(
name, pid, type, key, value))
return self.registry.update_timeline(name, pid, type, key, value)
@ipcmethod
def timeline_unregister(self, name, pid, type, key):
self.logger.writeInfo("Timeline Unregister {} {} {} {}".format(
name, pid, type, key))
return self.registry.unregister_from_timeline(name, pid, type, key)
@ipcmethod
def self_get(self, name, pid, api_version):
return self.registry.list_self(api_version)
class QueryCommon(object):
def __init__(self, logger=None, api_version="v1.0"):
self.logger = Logger("regquery", _parent=logger)
self.query_sockets = QuerySocketsCommon(WS_PORT, logger=self.logger)
# there is a choice here: watch at specific top levels (if flat), or watch all data.
# initially, watch all data - this may be less than ideal.
self.watcher = ChangeWatcher(reg['host'],
reg['port'],
handler=self,
logger=self.logger)
self.watcher.start()
self.api_version = api_version
def _cleanup(self):
self.watcher.stop()
self.watcher.join(timeout=5)
# generates a predictable UID for this process
def gen_source_id(self):
seed = '{}{}'.format(os.getpid(), socket.gethostname())
uid = str(uuid.uuid3(uuid.NAMESPACE_DNS, seed))
return uid
# parse services and render as a dictionary
def parse_services_dict(self, obj, url, args, verbose):
res_type_pattern = None
if url is not None and url != '/' and url != '':
res_type_pattern = util.translate_resourcetypes(url)
if 'node' in obj:
unpacked = etcd_unpack(obj)
nodes = self._match_nodes(unpacked, res_type_pattern, args,
verbose)
return nodes
return []
# extract objects of given types that also match supplied url and args
def _match_nodes(self, obj, pattern, args, verbose):
retval = []
for k, v in obj.items():
if any(rtype in k for rtype in VALID_TYPES) and type(v) is unicode:
if self._matches_path(k, pattern):
downgrade_ver = None
if args and "query.downgrade" in args:
downgrade_ver = args["query.downgrade"]
# Downgrade / convert any mis-versioned objects as required
resource_type = util.get_resourcetypes(k).replace("/", "")
json_repr = None
if resource_type != "":
json_repr = json.loads(v)
json_repr = version_transforms.convert(
json_repr, resource_type, self.api_version,
downgrade_ver)
# If nothing could be downgraded, skip over the object
if not json_repr:
continue
node = self._summarise(json_repr)
if self._matches_args(node, args):
if verbose:
retval.append(node)
else:
retval.append(node['id'])
elif type(v) is dict:
# explore more
retval = retval + self._match_nodes(v, pattern, args, verbose)
return retval
# see if href matches supplied regex
def _matches_path(self, href, pattern):
return pattern is None or pattern in href
# see if object matches supplied arguments
def _matches_args(self, obj, args):
arg_checker = QueryFilterCommon()
return arg_checker.check_args(args, obj)
# summarise service in a presentable way
def _summarise(self, json_repr):
if not json_repr:
return {}
removals = (x for x in json_repr.keys() if x.startswith("@_"))
for key in removals:
del json_repr[key]
return json_repr
def _process_response(self, response):
"""
Process a response from a GET long-poll on etcd (watch).
`response' is a dict, decoded from JSON.
"""
self.logger.writeDebug('process response {}'.format(response))
if response['action'] == 'set' or response['action'] == 'delete':
unpacked = etcd_unpack(response)
for k, v in unpacked.items():
restype = util.get_resourcetypes(k)
if restype in VALID_TYPES:
n_obj = {}
pn_obj = {}
if v:
n_obj = json.loads(v.get('node', '{}'))
pn_obj = json.loads(v.get('prevNode', '{}'))
if response['action'] == 'set' and pn_obj != n_obj:
self.do_sup(k, pn_obj, n_obj)
elif response['action'] == 'delete':
self.do_sdown(k, pn_obj, n_obj)
else:
self.logger.writeError(
"Invalid type '{}' in response.".format(restype))
# Queries
# get data for supplied path
def get_data_for_path(self, path, args):
# Set verbosity
verbose = not string.lower(args.get('verbose', '')) == 'false'
url = 'http://%s:%i/v2/keys/resource/?recursive=true' % (reg['host'],
reg['port'])
response = requests.request('GET', url, proxies={'http': ''})
if response.status_code != 200:
self.logger.writeError('bad status_code %i' % response.status_code)
return None
else:
return self.parse_services_dict(json.loads(response.text), path,
args, verbose)
def get_ws_subscribers(self, socket_id=None):
obj = None
if socket_id:
obj = self.query_sockets.get_socket(socket_id)
else:
obj = self.query_sockets.get_socketlist()
return obj
def post_ws_subscribers(self, json):
obj, created = self.query_sockets.post_socket(json)
return obj, created
def delete_ws_subscribers(self, socket_id):
res = self.query_sockets.delete_socket(socket_id)
return res
def do_sync(self, ws, socket):
# HTTP GET on etcd registry at top level
path = util.translate_resourcetypes(socket.resource_path)
url = 'http://{}:{}/v2/keys/resource/{}?recursive=true'.format(
reg['host'], reg['port'], path)
event = GrainEvent()
event.source_id = self.gen_source_id()
event.topic = socket.resource_path
event.flow_id = socket.uuid
# TODO: could get expensive with lots of flows...
try:
r = requests.request('GET', url, proxies={'http': ''})
if r.status_code not in [200, 404]:
err = {
"type":
"error",
"data":
"{} getting resources of topic {}".format(
r.status_code, path)
}
ws.send(json.dumps(err))
return err
obj = json.loads(r.text)
unpacked = etcd_unpack(obj)
nodes = self._match_nodes(unpacked,
path,
socket.params,
verbose=True)
for node in nodes:
event.addGrainFromObj(pre_obj=node, post_obj=node)
ws.send(json.dumps(event.obj()))
except Exception as err:
self.logger.writeError('Exception in do_sync: {}'.format(err))
def do_sup(self, path, pn_obj, n_obj):
self.logger.writeDebug('do_sup {}'.format(path))
if cmp(n_obj, pn_obj) == 0:
return
ws = self.query_sockets.find_socks(path=path, obj=n_obj, p_obj=pn_obj)
event = GrainEvent()
event.source_id = self.gen_source_id()
event.topic = util.get_resourcetypes(path)
for s in ws:
self.logger.writeDebug('next ws ' + s.ws_href)
downgrade_ver = None
if s.params and "query.downgrade" in s.params:
downgrade_ver = s.params["query.downgrade"]
s_n_obj = version_transforms.convert(n_obj,
event.topic.replace("/", ""),
self.api_version,
downgrade_ver)
s_pn_obj = version_transforms.convert(pn_obj,
event.topic.replace("/", ""),
self.api_version,
downgrade_ver)
if not s_n_obj and not s_pn_obj:
continue
s_n_obj = self._summarise(s_n_obj)
s_pn_obj = self._summarise(s_pn_obj)
event.flow_id = s.uuid
event.clearGrains()
if not self._matches_args(s_pn_obj, s.params):
# Didn't previously match filter, so should be returned
event.addGrainFromObj(pre_obj={}, post_obj=n_obj)
elif not self._matches_args(s_n_obj, s.params):
# Doesn't match filter any longer, so shouldn't be returned
event.addGrainFromObj(pre_obj=s_pn_obj, post_obj={})
else:
event.addGrainFromObj(pre_obj=s_pn_obj, post_obj=s_n_obj)
s.notify_subscribers(event.obj())
def do_sdown(self, path, pn_obj, n_obj):
self.logger.writeDebug('do_sdown {}'.format(path))
ws = self.query_sockets.find_socks(path=path, obj=n_obj, p_obj=pn_obj)
event = GrainEvent()
event.source_id = self.gen_source_id()
event.topic = util.get_resourcetypes(path)
for s in ws:
self.logger.writeDebug('next ws' + s.ws_href)
downgrade_ver = None
if s.params and "query.downgrade" in s.params:
downgrade_ver = s.params["query.downgrade"]
s_pn_obj = version_transforms.convert(pn_obj,
event.topic.replace("/", ""),
self.api_version,
downgrade_ver)
if not s_pn_obj:
continue
s_pn_obj = self._summarise(s_pn_obj)
event.flow_id = s.uuid
event.clearGrains()
event.addGrainFromObj(pre_obj=s_pn_obj, post_obj={})
s.notify_subscribers(event.obj())
class NodeFacadeService:
def __init__(self, interactive=False):
self.logger = Logger("facade", None)
if HOST == "":
self.logger.writeFatal(
"Unable to start facade due to lack of connectivity")
sys.exit(1)
self.running = False
self.httpServer = None
self.interface = None
self.interactive = interactive
self.registry = None
self.registry_cleaner = None
self.node_id = None
self.mdns = MDNSEngine()
self.mappings = {
"device": "ver_dvc",
"flow": "ver_flw",
"source": "ver_src",
"sender": "ver_snd",
"receiver": "ver_rcv",
"self": "ver_slf"
}
self.mdns_updater = None
self.auth_registry = AuthRegistry(app=None, scope=ALLOWED_SCOPE)
self.protocol = PROTOCOL
if PROTOCOL == "https":
self.dns_sd_port = DNS_SD_HTTPS_PORT
else:
self.dns_sd_port = DNS_SD_HTTP_PORT
if ENABLE_P2P:
self.mdns_updater = MDNSUpdater(self.mdns,
DNS_SD_TYPE,
DNS_SD_NAME,
self.mappings,
self.dns_sd_port,
self.logger,
txt_recs=self._mdns_txt(
NODE_APIVERSIONS,
self.protocol, OAUTH_MODE))
self.aggregator = Aggregator(self.logger, self.mdns_updater,
self.auth_registry)
def _mdns_txt(self, versions, protocol, oauth_mode):
return {
"api_ver": ",".join(versions),
"api_proto": protocol,
"api_auth": str(oauth_mode).lower()
}
def sig_handler(self):
print('Pressed ctrl+c')
self.stop()
def sig_hup_handler(self):
if getLocalIP() != "":
global HOST
HOST = updateHost()
self.registry.modify_node(href=self.generate_href(),
host=HOST,
api={
"versions": NODE_APIVERSIONS,
"endpoints":
self.generate_endpoints()
},
interfaces=self.list_interfaces())
def generate_endpoints(self):
endpoints = []
endpoints.append({
"host": HOST,
"port": self.dns_sd_port, # Everything should go via apache proxy
"protocol": self.protocol,
"authorization": OAUTH_MODE
})
return endpoints
def generate_href(self):
return "{}://{}/".format(PROTOCOL, HOST)
def list_interfaces(self):
interfaces = {}
# Initially populate interfaces from known-good location
net_path = "/sys/class/net/"
if os.path.exists(net_path):
for interface_name in os.listdir(net_path):
address_path = net_path + interface_name + "/address"
if os.path.exists(address_path) and interface_name != 'lo':
with open(address_path, 'r') as address_file:
address = address_file.readline().strip('\n')
if address:
interfaces[interface_name] = {
"name": interface_name,
"chassis_id": None,
"port_id": address.lower().replace(":", "-")
}
# Attempt to source proper LLDP data for interfaces
if os.path.exists("/usr/sbin/lldpcli"):
try:
chassis_data = json.loads(
check_output(
["/usr/sbin/lldpcli", "show", "chassis", "-f",
"json"]))
chassis_id = chassis_data["local-chassis"]['chassis'].values(
)[0]["id"]["value"]
if chassis_data["local-chassis"]['chassis'].values(
)[0]["id"]["type"] == "mac":
chassis_id = chassis_id.lower().replace(":", "-")
interface_data = json.loads(
check_output([
"/usr/sbin/lldpcli", "show", "statistics", "-f", "json"
]))
if isinstance(interface_data["lldp"]["interface"], dict):
for interface_name in interface_data["lldp"][
"interface"].keys():
if interface_name in interfaces:
# Only correct the Chassis ID. Port ID MUST be a MAC address
interfaces[interface_name][
"chassis_id"] = chassis_id
else:
for interface_block in interface_data["lldp"]["interface"]:
interface_name = interface_block.keys()[0]
if interface_name in interfaces:
# Only correct the Chassis ID. Port ID MUST be a MAC address
interfaces[interface_name][
"chassis_id"] = chassis_id
except Exception:
pass
return list(itervalues(interfaces))
def start(self):
if self.running:
gevent.signal_handler(signal.SIGINT, self.sig_handler)
gevent.signal_handler(signal.SIGTERM, self.sig_handler)
gevent.signal_handler(signal.SIGHUP, self.sig_hup_handler)
self.mdns.start()
self.node_id = get_node_id()
node_version = str(ptptime.ptp_detail()[0]) + ":" + str(
ptptime.ptp_detail()[1])
node_data = {
"id":
self.node_id,
"label":
_config.get('node_label', FQDN),
"description":
_config.get('node_description', "Node on {}".format(FQDN)),
"tags":
_config.get('node_tags', {}),
"href":
self.generate_href(),
"host":
HOST,
"services": [],
"hostname":
HOSTNAME,
"caps": {},
"version":
node_version,
"api": {
"versions": NODE_APIVERSIONS,
"endpoints": self.generate_endpoints(),
},
"clocks": [],
"interfaces":
self.list_interfaces()
}
self.registry = FacadeRegistry(self.mappings.keys(), self.aggregator,
self.mdns_updater, self.node_id,
node_data, self.logger)
self.registry_cleaner = FacadeRegistryCleaner(self.registry)
self.registry_cleaner.start()
self.httpServer = HttpServer(
FacadeAPI,
PORT,
'0.0.0.0',
api_args=[self.registry, self.auth_registry])
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeInfo('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeInfo("Running on port: {}".format(
self.httpServer.port))
try:
self.logger.writeInfo("Registering as {}...".format(self.node_id))
self.aggregator.register(
'node', self.node_id,
**translate_api_version(node_data, "node", NODE_REGVERSION))
except Exception as e:
self.logger.writeWarning("Could not register: {}".format(
e.__repr__()))
self.interface = FacadeInterface(self.registry, self.logger)
self.interface.start()
def run(self):
self.running = True
pidfile = "/tmp/ips-nodefacade.pid"
with open(pidfile, 'w') as f:
f.write(str(getpid()))
self.start()
daemon.notify(SYSTEMD_READY)
while self.running:
self.registry.update_ptp()
time.sleep(1)
os.unlink(pidfile)
def _cleanup(self):
try:
self.logger.writeDebug("cleanup: unregister facade " +
self.node_id)
self.aggregator.unregister('node', self.node_id)
except Exception as e:
self.logger.writeWarning("Could not unregister: {}".format(e))
if self.mdns:
try:
self.mdns.stop()
except Exception as e:
self.logger.writeWarning("Could not stop mdns: {}".format(e))
self.registry_cleaner.stop()
self.interface.stop()
self.httpServer.stop()
self.aggregator.stop()
self.mdns_updater.stop()
self.logger.writeInfo("Stopped main()")
def stop(self):
self._cleanup()
self.running = False
class EtcdEventQueue(object):
"""
Attempt to overcome the "missed etcd event" issue, which can be caused when
processing of a return from a http long-poll takes too long, and an event is
missed in etcd.
This uses etcd's "waitIndex" functionality, which has the caveat that only
the last 1000 events are stored. So, whilst this scheme should not miss events
for fast updates, the case where 1000 updates occur within the space of a
single event being processed will still be missed. This is unlikely, but still
possible, so a "sentinel" message with action=index_skip will be sent to
the output queue when this happens.
To use this, the `queue' member of EtcdEventQueue is iterable:
q = EtcdEventQueue()
for message in q.queue:
# process
This uses http://www.gevent.org/gevent.queue.html as an underlying data
structure, so can be consumed from multiple greenlets if necessary.
"""
def __init__(self, host, port, logger=None):
self.queue = gevent.queue.Queue()
self._base_url = "http://{}:{}/v2/keys/resource/".format(host, port)
self._long_poll_url = self._base_url + "?recursive=true&wait=true"
self._greenlet = gevent.spawn(self._wait_event, 0)
self._alive = True
self._logger = Logger("etcd_watch", logger)
def _get_index(self, current_index):
index = current_index
try:
response = requests.get(self._base_url, proxies={'http': ''}, timeout=1)
if response is not None:
if response.status_code == 200:
index = _get_etcd_index(response, self._logger)
self._logger.writeDebug("waitIndex now = {}".format(index))
# Always want to know if the index we were waiting on was greater
# than current index, as this indicates something that needs further
# investigation...
if index < current_index:
self._logger.writeWarning("Index decreased! {} -> {}".format(current_index, index))
elif response.status_code in [400, 404]:
# '/resource' not found in etcd yet, back off for a second and set waitIndex to value of the x-etcd-index header
index = int(response.headers.get('x-etcd-index', 0))
self._logger.writeInfo("{} not found, wait... waitIndex={}".format(self._base_url, index))
gevent.sleep(1)
else:
# response was None...
self._logger.writeWarning("Could not GET {} after timeout; waitIndex now=0".format(self._base_url))
index = 0
except Exception as ex:
# Getting the new index failed, so reset to 0.
self._logger.writeWarning("Reset waitIndex to 0, error: {}".format(ex))
index = 0
return index
def _wait_event(self, since):
current_index = since
while self._alive:
req = None
try:
# Make the long-poll request to etcd using the current
# "waitIndex". A timeout is used as situations have been
# observed where the etcd modification index decreases (caused
# by network partition or by a node having it's data reset?),
# and the query service is not restarted, hence the code below
# is left waiting for a much higher modification index than it
# should. To mitigate this simply, when a timeout occurs,
# assume that the modified index is "wrong", and forcibly try
# to fetch the next index. This may "miss" updates, which is of
# limited consequence. An enhancement (and therefore
# complication...) could use the fact that the timeout is
# small, and set waitIndex to the x-etcd-index result minus
# some heuristically determined number of updates, to try and
# catch the "back-in-time" updates stored in etcd's log, but
# this feels brittle and overcomplicated for something that
# could be solved by a browser refresh/handling of the "skip"
# event to request a full set of resources.
# https://github.com/coreos/etcd/blob/master/Documentation/api.md#waiting-for-a-change
next_index_param = "&waitIndex={}".format(current_index + 1)
req = requests.get(self._long_poll_url + next_index_param, proxies={'http': ''}, timeout=20)
except socket.timeout:
# Get a new wait index to watch from by querying /resource
self._logger.writeDebug("Timeout waiting on long-poll. Refreshing waitIndex...")
current_index = self._get_index(current_index)
continue
except Exception as ex:
self._logger.writeWarning("Could not contact etcd: {}".format(ex))
gevent.sleep(5)
continue
if req is not None:
# Decode payload, which should be json...
try:
json = req.json()
except Exception:
self._logger.writeError("Error decoding payload: {}".format(req.text))
continue
if req.status_code == 200:
# Return from request was OK, so put the payload on the queue.
# NOTE: we use the "modifiedIndex" of the _node_ we receive, NOT the header.
# This follows the etcd docs linked above.
self.queue.put(json)
current_index = json.get('node', {}).get('modifiedIndex', current_index)
else:
# Error codes documented here:
# https://github.com/coreos/etcd/blob/master/Documentation/errorcode.md
self._logger.writeInfo("error: http:{}, etcd:{}".format(req.status_code, json.get('errorCode', 0)))
if json.get('errorCode', 0) == 401:
# Index has been cleared. This may cause missed events, so send an (invented) sentinel message to queue.
new_index = self._get_index(current_index)
self._logger.writeWarning("etcd history not available; skipping {} -> {}".format(current_index, new_index))
self.queue.put({'action': 'index_skip', 'from': current_index, 'to': new_index})
current_index = new_index
def stop(self):
self._logger.writeInfo("Stopping service")
print "stopping"
self._alive = False
self._greenlet.kill(timeout=5)
self.queue.put(StopIteration)
class FacadeRegistry(object):
def __init__(self,
resources,
aggregator,
mdns_updater,
node_id,
node_data,
logger=None):
# `node_data` must be correctly structured
self.permitted_resources = resources
self.services = {}
self.aggregator = aggregator
self.mdns_updater = mdns_updater
self.node_id = node_id
assert ("interfaces" in node_data
) # Check data conforms to latest supported API version
self.node_data = node_data
self.logger = Logger("facade_registry", logger)
def modify_node(self, **kwargs):
for key in kwargs.keys():
if key in self.node_data:
self.node_data[key] = kwargs[key]
self.update_node()
def update_node(self):
self.node_data["services"] = []
for service_name in self.services:
href = None
if self.services[service_name]["href"]:
if self.services[service_name]["proxy_path"]:
href = "http://{}/{}".format(
self.node_data["host"],
self.services[service_name]["proxy_path"])
self.node_data["services"].append({
"href":
href,
"type":
self.services[service_name]["type"]
})
self.node_data["version"] = str(ptptime.ptp_detail()[0]) + ":" + str(
ptptime.ptp_detail()[1])
try:
self.aggregator.register(
"node", self.node_id,
**legalise_resource(self.node_data, "node", NODE_REGVERSION))
except Exception as e:
self.logger.writeError(
"Exception re-registering node: {}".format(e))
def register_service(self,
name,
srv_type,
pid,
href=None,
proxy_path=None):
if name in self.services:
return RES_EXISTS
self.services[name] = {
"heartbeat": time.time(),
"resource": {}, # Registered resources live under here
"timeline": {
"flowsegment": {}
}, # Registered "timeline" items live under here
"pid": pid,
"href": href,
"proxy_path": proxy_path,
"type": srv_type
}
for resource_name in self.permitted_resources:
self.services[name]["resource"][resource_name] = {}
self.update_node()
return RES_SUCCESS
def update_service(self, name, pid, href=None, proxy_path=None):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
self.services[name]["href"] = href
self.services[name]["proxy_path"] = proxy_path
self.update_node()
return RES_SUCCESS
def unregister_service(self, name, pid):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
for namespace in ["resource", "timeline"]:
for type in self.services[name][namespace].keys():
for key in self.services[name][namespace][type].keys():
self._unregister(name, namespace, pid, type, key)
self.services.pop(name, None)
self.update_node()
return RES_SUCCESS
def heartbeat_service(self, name, pid):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
return RES_SUCCESS
def cleanup_services(self):
timed_out = time.time() - HEARTBEAT_TIMEOUT
for name in self.services.keys():
if self.services[name]["heartbeat"] < timed_out:
self.unregister_service(name, self.services[name]["pid"])
def register_resource(self, service_name, pid, type, key, value):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
return self._register(service_name, "resource", pid, type, key, value)
def register_to_timeline(self, service_name, pid, type, key, value):
return self._register(service_name, "timeline", pid, type, key, value)
def _register(self, service_name, namespace, pid, type, key, value):
if "max_api_version" not in value:
self.logger.writeWarning(
"Service {}: Registration without valid api version specified".
format(service_name))
value["max_api_version"] = "v1.0"
elif api_version_less_than(value["max_api_version"], NODE_REGVERSION):
self.logger.writeWarning(
"Trying to register resource with api version too low: \"%s\" : %s",
key, json.dumps(value))
if not service_name in self.services:
return RES_NOEXISTS
if not self.services[service_name]["pid"] == pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
# Add a node_id to those resources which need one
if type == 'device':
value['node_id'] = self.node_id
self.services[service_name][namespace][type][key] = value
# Don't pass non-registration exceptions to clients
try:
self._update_mdns(type)
except Exception as e:
self.logger.writeError(
"Exception registering with mDNS: {}".format(e))
try:
self.aggregator.register_into(
namespace, type, key,
**legalise_resource(value, type, NODE_REGVERSION))
self.logger.writeDebug("registering {} {}".format(type, key))
except Exception as e:
self.logger.writeError("Exception registering {}: {}".format(
namespace, e))
return RES_OTHERERROR
return RES_SUCCESS
def update_resource(self, service_name, pid, type, key, value):
return self.register_resource(service_name, pid, type, key, value)
def find_service(self, type, key):
for service_name in self.services.keys():
if key in self.services[service_name]["resource"][type]:
return service_name
return None
def update_timeline(self, service_name, pid, type, key, value):
return self._register(service_name, "timeline", pid, type, key, value)
def unregister_resource(self, service_name, pid, type, key):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
return self._unregister(service_name, "resource", pid, type, key)
def unregister_from_timeline(self, service_name, pid, type, key):
return self._unregister(service_name, "timeline", pid, type, key)
def _unregister(self, service_name, namespace, pid, type, key):
if not service_name in self.services:
return RES_NOEXISTS
if not self.services[service_name]["pid"] == pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
self.services[service_name][namespace][type].pop(key, None)
# Don't pass non-registration exceptions to clients
try:
self.aggregator.unregister_from(namespace, type, key)
except Exception as e:
self.logger.writeError("Exception unregistering {}: {}".format(
namespace, e))
return RES_OTHERERROR
try:
self._update_mdns(type)
except Exception as e:
extype, exmsg = e
self.logger.writeError(
"Exception unregistering from mDNS: {}".format(e))
if extype != -65548: # Name conflict
return RES_OTHERERROR
return RES_SUCCESS
def list_services(self, api_version="v1.0"):
return self.services.keys()
def get_service_href(self, name, api_version="v1.0"):
if not name in self.services:
return RES_NOEXISTS
href = self.services[name]["href"]
if self.services[name]["proxy_path"]:
href += "/" + self.services[name]["proxy_path"]
return href
def get_service_type(self, name, api_version="v1.0"):
if not name in self.services:
return RES_NOEXISTS
return self.services[name]["type"]
def list_resource(self, type, api_version="v1.0"):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
response = {}
for name in self.services:
response = (dict(
response.items() +
[(k, legalise_resource(x, type, api_version))
for (k, x) in self.services[name]["resource"][type].items()
if ((api_version == "v1.0") or
("max_api_version" in x and not api_version_less_than(
x["max_api_version"], api_version)))]))
return response
def _update_mdns(self, type):
items = self.list_resource(type)
if not isinstance(items, dict):
return
if len(items) == 1:
try:
self.mdns_updater.update_mdns(type, "register")
except Exception:
self.mdns_updater.update_mdns(type, "update")
elif len(items) == 0:
self.mdns_updater.update_mdns(type, "unregister")
else:
self.mdns_updater.update_mdns(type, "update")
def list_self(self, api_version="v1.0"):
return legalise_resource(self.node_data, "node", api_version)
def update_ptp(self):
do_update = False
for clk in self.node_data['clocks']:
if "ref_type" in clk and clk["ref_type"] == "ptp":
old_clk = copy.copy(clk)
clk['traceable'] = False
clk['gmid'] = '00-00-00-00-00-00-00-00'
clk['locked'] = False
if clk != old_clk:
do_update = True
if do_update:
self.update_node()
class FacadeRegistry(object):
def __init__(self, resources, aggregator, mdns_updater, node_id, node_data, logger=None):
# `node_data` must be correctly structured
self.permitted_resources = resources
self.services = {}
self.aggregator = aggregator
self.mdns_updater = mdns_updater
self.node_id = node_id
assert("interfaces" in node_data) # Check data conforms to latest supported API version
self.node_data = node_data
self.logger = Logger("facade_registry", logger)
def modify_node(self, **kwargs):
for key in kwargs.keys():
if key in self.node_data:
self.node_data[key] = kwargs[key]
self.update_node()
def update_node(self):
self.node_data["services"] = []
for service_name in self.services:
href = None
if self.services[service_name]["href"]:
if self.services[service_name]["proxy_path"]:
href = self.node_data["href"] + self.services[service_name]["proxy_path"]
self.node_data["services"].append({"href": href, "type": self.services[service_name]["type"]})
self.node_data["version"] = str(ptptime.ptp_detail()[0]) + ":" + str(ptptime.ptp_detail()[1])
try:
self.aggregator.register("node", self.node_id, **self.preprocess_resource("node", self.node_data["id"], self.node_data, NODE_REGVERSION))
except Exception as e:
self.logger.writeError("Exception re-registering node: {}".format(e))
def register_service(self, name, srv_type, pid, href=None, proxy_path=None):
if name in self.services:
return RES_EXISTS
self.services[name] = {
"heartbeat": time.time(),
"resource": {}, # Registered resources live under here
"control": {}, # Registered device controls live under here
"pid": pid,
"href": href,
"proxy_path": proxy_path,
"type": srv_type
}
for resource_name in self.permitted_resources:
self.services[name]["resource"][resource_name] = {}
self.update_node()
return RES_SUCCESS
def update_service(self, name, pid, href=None, proxy_path=None):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
self.services[name]["href"] = href
self.services[name]["proxy_path"] = proxy_path
self.update_node()
return RES_SUCCESS
def unregister_service(self, name, pid):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
for namespace in ["resource", "control"]:
for type in self.services[name][namespace].keys():
for key in self.services[name][namespace][type].keys():
if namespace == "control":
self._register(name, "control", pid, type, "remove", self.services[name][namespace][type][key])
else:
self._unregister(name, namespace, pid, type, key)
self.services.pop(name, None)
self.update_node()
return RES_SUCCESS
def heartbeat_service(self, name, pid):
if not name in self.services:
return RES_NOEXISTS
if self.services[name]["pid"] != pid:
return RES_UNAUTHORISED
self.services[name]["heartbeat"] = time.time()
return RES_SUCCESS
def cleanup_services(self):
timed_out = time.time() - HEARTBEAT_TIMEOUT
for name in self.services.keys():
if self.services[name]["heartbeat"] < timed_out:
self.unregister_service(name, self.services[name]["pid"])
def register_resource(self, service_name, pid, type, key, value):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
return self._register(service_name, "resource", pid, type, key, value)
def register_control(self, service_name, pid, device_id, control_data):
return self._register(service_name, "control", pid, device_id, "add", control_data)
def _register(self, service_name, namespace, pid, type, key, value):
if namespace != "control":
if "max_api_version" not in value:
self.logger.writeWarning("Service {}: Registration without valid api version specified".format(service_name))
value["max_api_version"] = "v1.0"
elif api_version_less_than(value["max_api_version"], NODE_REGVERSION):
self.logger.writeWarning("Trying to register resource with api version too low: '{}' : {}".format(key, json.dumps(value)))
if not service_name in self.services:
return RES_NOEXISTS
if not self.services[service_name]["pid"] == pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
# Add a node_id to those resources which need one
if type == 'device':
value['node_id'] = self.node_id
if namespace == "control":
if type not in self.services[service_name][namespace]:
# 'type' is the Device ID in this case
self.services[service_name][namespace][type] = {}
if key == "add":
# Register
self.services[service_name][namespace][type][value["href"]] = value
else:
# Unregister
self.services[service_name][namespace][type].pop(value["href"], None)
# Reset the parameters below to force re-registration of the corresponding Device
namespace = "resource"
key = type # Device ID
type = "device"
value = None
for name in self.services: # Find the service which registered the Device in question
if key in self.services[name]["resource"][type]:
value = self.services[name]["resource"][type][key]
break
if not value: # Device isn't actually registered at present
return RES_SUCCESS
else:
self.services[service_name][namespace][type][key] = value
# Don't pass non-registration exceptions to clients
try:
if namespace == "resource":
self._update_mdns(type)
except Exception as e:
self.logger.writeError("Exception registering with mDNS: {}".format(e))
try:
self.aggregator.register_into(namespace, type, key, **self.preprocess_resource(type, key, value, NODE_REGVERSION))
self.logger.writeDebug("registering {} {}".format(type, key))
except Exception as e:
self.logger.writeError("Exception registering {}: {}".format(namespace, e))
return RES_OTHERERROR
return RES_SUCCESS
def update_resource(self, service_name, pid, type, key, value):
return self.register_resource(service_name, pid, type, key, value)
def find_service(self, type, key):
for service_name in self.services.keys():
if key in self.services[service_name]["resource"][type]:
return service_name
return None
def unregister_resource(self, service_name, pid, type, key):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
return self._unregister(service_name, "resource", pid, type, key)
def unregister_control(self, service_name, pid, device_id, control_data):
# Note use of register here, as we're updating an existing Device
return self._register(service_name, "control", pid, device_id, "remove", control_data)
def _unregister(self, service_name, namespace, pid, type, key):
if not service_name in self.services:
return RES_NOEXISTS
if not self.services[service_name]["pid"] == pid:
return RES_UNAUTHORISED
if key == "00000000-0000-0000-0000-000000000000":
return RES_OTHERERROR
self.services[service_name][namespace][type].pop(key, None)
# Don't pass non-registration exceptions to clients
try:
self.aggregator.unregister_from(namespace, type, key)
except Exception as e:
self.logger.writeError("Exception unregistering {}: {}".format(namespace, e))
return RES_OTHERERROR
try:
if namespace == "resource":
self._update_mdns(type)
except Exception as e:
extype, exmsg = e
self.logger.writeError("Exception unregistering from mDNS: {}".format(e))
if extype != -65548: # Name conflict
return RES_OTHERERROR
return RES_SUCCESS
def list_services(self, api_version="v1.0"):
return self.services.keys()
def get_service_href(self, name, api_version="v1.0"):
if not name in self.services:
return RES_NOEXISTS
href = self.services[name]["href"]
if self.services[name]["proxy_path"]:
href += "/" + self.services[name]["proxy_path"]
return href
def get_service_type(self, name, api_version="v1.0"):
if not name in self.services:
return RES_NOEXISTS
return self.services[name]["type"]
def preprocess_resource(self, type, key, value, api_version="v1.0"):
if type == "device":
value_copy = copy.deepcopy(value)
for name in self.services:
if key in self.services[name]["control"] and "controls" in value_copy:
value_copy["controls"] = value_copy["controls"] + self.services[name]["control"][key].values()
return legalise_resource(value_copy, type, api_version)
else:
return legalise_resource(value, type, api_version)
def list_resource(self, type, api_version="v1.0"):
if not type in self.permitted_resources:
return RES_UNSUPPORTED
response = {}
for name in self.services:
response = (dict(response.items() + [ (k, self.preprocess_resource(type, k, x, api_version))
for (k,x) in self.services[name]["resource"][type].items()
if ((api_version == "v1.0") or
("max_api_version" in x and
not api_version_less_than(x["max_api_version"], api_version))) ]))
return response
def _update_mdns(self, type):
items = self.list_resource(type)
if not isinstance(items, dict):
return
if len(items) == 1:
try:
self.mdns_updater.update_mdns(type, "register")
except Exception:
self.mdns_updater.update_mdns(type, "update")
elif len(items) == 0:
self.mdns_updater.update_mdns(type, "unregister")
else:
self.mdns_updater.update_mdns(type, "update")
def list_self(self, api_version="v1.0"):
return self.preprocess_resource("node", self.node_data["id"], self.node_data, api_version)
def update_ptp(self):
if IPP_UTILS_CLOCK_AVAILABLE:
sts = IppClock().PTPStatus()
do_update = False
for clk in self.node_data['clocks']:
old_clk = copy.copy(clk)
if "ref_type" in clk and clk["ref_type"] == "ptp":
clk['traceable'] = False
clk['gmid'] = '00-00-00-00-00-00-00-00'
clk['locked'] = False
if IPP_UTILS_CLOCK_AVAILABLE:
if len(sts.keys()) > 0:
clk['traceable'] = sts['timeTraceable']
clk['gmid'] = sts['grandmasterClockIdentity'].lower()
clk['locked'] = (sts['ofm'][0] == 0)
if clk != old_clk:
do_update = True
if do_update:
self.update_node()
class QueryService:
def __init__(self, logger=None):
self.running = False
self.logger = Logger("regquery")
self.logger.writeDebug('Running QueryService')
# HTTPS under test only at present
# enabled = Use HTTPS only in all URLs and mDNS adverts
# disabled = Use HTTP only in all URLs and mDNS adverts
# mixed = Use HTTP in all URLs, but additionally advertise an HTTPS endpoint for discovery of this API only
self.config = {"priority": 100, "https_mode": "disabled"}
self._load_config()
self.mdns = MDNSEngine()
self.httpServer = HttpServer(QueryServiceAPI, WS_PORT, '0.0.0.0', api_args=[self.logger, self.config])
def start(self):
if self.running:
gevent.signal(signal.SIGINT, self.sig_handler)
gevent.signal(signal.SIGTERM, self.sig_handler)
self.running = True
self.mdns.start()
self.logger.writeDebug('Running web socket server on %i' % WS_PORT)
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeDebug('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeDebug("Running on port: {}".format(self.httpServer.port))
priority = self.config["priority"]
if not str(priority).isdigit() or priority < 100:
priority = 0
if self.config["https_mode"] != "enabled":
self.mdns.register(DNS_SD_NAME + "_http", DNS_SD_TYPE, DNS_SD_HTTP_PORT,
{"pri": priority,
"api_ver": ",".join(API_VERSIONS),
"api_proto": "http"})
if self.config["https_mode"] != "disabled":
self.mdns.register(DNS_SD_NAME + "_https", DNS_SD_TYPE, DNS_SD_HTTPS_PORT,
{"pri": priority,
"api_ver": ",".join(API_VERSIONS),
"api_proto": "https"})
def run(self):
self.running = True
self.start()
while self.running:
time.sleep(1)
self._cleanup()
def _cleanup(self):
self.mdns.stop()
self.mdns.close()
self.httpServer.stop()
def sig_handler(self):
self.stop()
def stop(self):
self.running = False
def _load_config(self):
try:
config_file = "/etc/ips-regquery/config.json"
if os.path.isfile(config_file):
f = open(config_file, 'r')
extra_config = json.loads(f.read())
self.config.update(extra_config)
except Exception as e:
self.logger.writeDebug("Exception loading config: {}".format(e))
class ConnectionManagementService:
def __init__(self, logger=None):
self.running = False
from nmoscommon.logger import Logger
from nmosnode.facade import Facade
self.logger = Logger("conmanage")
self.logger.writeWarning("Could not find ipppython facade")
self.facade = Facade("{}/{}".format(CONN_APINAME, CONN_APIVERSIONS[-1]),
address="ipc:///tmp/ips-nodefacade", logger=self.logger)
self.logger.writeDebug("Running Connection Management Service")
self.httpServer = HttpServer(ConnectionManagementAPI, WS_PORT,
'0.0.0.0', api_args=[self.logger])
def start(self):
'''Call this to run the API without blocking'''
if self.running:
gevent.signal_handler(signal.SIGINT, self.sig_handler)
gevent.signal_handler(signal.SIGTERM, self.sig_handler)
self.running = True
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeDebug('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeDebug("Running on port: {}"
.format(self.httpServer.port))
self.facade.register_service("http://127.0.0.1:{}".format(self.httpServer.port),
"{}{}/".format(CONN_ROOT[1:], CONN_APIVERSIONS[-1]))
try:
from nmosconnectiondriver.httpIpstudioDriver import httpIpstudioDriver
self.logger.writeInfo("Using ipstudio driver")
# Start the IPStudio driver
self.driver = httpIpstudioDriver(
self.httpServer.api,
self.logger,
self.facade
)
except ImportError:
# Start the mock driver
self.driver = NmosDriver(
self.httpServer.api,
self.logger,
self.facade
)
def run(self):
'''Call this to run the API in keep-alive (blocking) mode'''
self.running = True
self.start()
itercount = 0
while self.running:
gevent.sleep(1)
itercount += 1
if itercount == 5:
self.facade.heartbeat_service()
itercount = 0
self._cleanup()
def _cleanup(self):
self.httpServer.stop()
self.facade.unregister_service()
def sig_handler(self):
self.stop()
def stop(self):
'''Gracefully shut down the API'''
self._cleanup()
self.running = False
class GarbageCollect(object):
parent_tab = {
'devices': [('nodes', 'node_id')],
'senders': [('devices', 'device_id')],
'receivers': [('devices', 'device_id')],
'sources': [('devices', 'device_id')],
'flows': [('devices', 'device_id'), ('sources', 'source_id')]
}
def __init__(self, registry, identifier, logger=None, interval=INTERVAL):
"""
interval
Number of seconds between checks / collections. An interval of '0'
means 'never check'.
"""
self.registry = registry
self.logger = Logger("garbage_collect", logger)
self.identifier = identifier
if interval > 0:
gevent.spawn_later(interval, self.garbage_collect)
def garbage_collect(self):
# Check to see if garbage collection hasn't been done recently (by another aggregator)
# Uses ETCD's prevExist=false function
# See https://github.com/coreos/etcd/blob/master/Documentation/api.md#atomic-compare-and-swap
try:
flag = self.registry.put_garbage_collection_flag(
host=self.identifier, ttl=LOCK_TIMEOUT)
if flag.status_code != 201:
self.logger.writeDebug(
"Not collecting - another collector has recently collected"
)
return
# Kick off a collection with a specified timeout.
try:
with gevent.Timeout(TIMEOUT, TooLong):
self._collect()
finally:
self.logger.writeDebug("remove flag")
self._remove_flag()
except Exception as e:
self.logger.writeError(
"Could not write garbage collect flag: {}".format(e))
finally:
# Always schedule another
gevent.spawn_later(INTERVAL, self.garbage_collect)
self.logger.writeDebug("scheduled...")
def _collect(self):
try:
self.logger.writeDebug("Collecting: {}".format(self.identifier))
# create list of nodes still alive
alive_nodes = []
health_dict = self.registry.get_healths()
for h in health_dict.get('/health', {}).keys():
node_name = h.split('/')[-1]
alive_nodes.append(node_name)
# TODO: GETs... maybe getting the whole response in one go is better?
# Maybe doing these async is a good idea? For now, this suffices.
all_types = [
"nodes", "devices", "senders", "receivers", "sources", "flows"
]
resources = {
rtype: self.registry.get_all(rtype)
for rtype in all_types
}
# Get a flat list of (type, resource) pairs for existing resources
# TODO: combine with above
all_resources = []
for res_type, res in resources.items():
all_resources += [(res_type, x) for x in res]
# Initialise the removal queue with any dead nodes
nodes = [x.strip('/') for x in self.registry.getresources("nodes")]
# TODO: already have this above...
kill_q = [('nodes', node_id) for node_id in nodes
if node_id not in alive_nodes]
# Create a list of (type, id) pairs of resources that should be removed.
to_kill = []
# Find orphaned resources
kill_q += self.__find_dead_resources(all_resources, to_kill)
# Process the removal queue.
while kill_q:
gevent.sleep(0.0)
# Add these resources to the list of removals
to_kill += kill_q
# Reduce search space; this resource can never parent another
# This proves to be faster in the long run.
all_resources = [
x for x in all_resources
if (x[0], x[1]['id']) not in to_kill
]
# Look through remaining resources and get a new kill_q
kill_q = self.__find_dead_resources(all_resources, to_kill)
for resource_type, resource_id in to_kill:
self.logger.writeInfo("removing resource: {}/{}".format(
resource_type, resource_id))
self.registry.delete(resource_type, resource_id)
except self.registry.RegistryUnavailable:
self.logger.writeWarning("registry unavailable")
except TooLong:
self.logger.writeWarning("took too long")
except Exception as e:
self.logger.writeError("unhandled exception: {}".format(e))
def __find_dead_resources(self, all_resources, to_kill):
def is_alive(parent_def):
if parent_def in to_kill:
return False
parent_type, parent_id = parent_def
found_parent = next(
(x for x in all_resources
if x[0] == parent_type and x[1]['id'] == parent_id), None)
return found_parent is not None
# Build a list of resource to remove
kill_q = []
# Look through all remaining resources
for child_type, child in all_resources:
# We need never consider nodes; they should have already been marked.
if child_type == "nodes":
continue
child_id = child['id']
# Get parent for child. There is only ever one; anything with multiple
# parent entries in the parent table has multiple entries for backward
# compatibility, in order strongest->weakest.
parents = [(parent_type, child.get(parent_key))
for parent_type, parent_key in self.parent_tab.get(
child_type, (None, None))]
parent = next((x for x in parents if x[1] is not None), None)
if parent is None or not is_alive(parent):
kill_q.append((child_type, child_id))
return kill_q
def _remove_flag(self):
try:
self.registry.delete_raw("garbage_collection")
except Exception as e:
self.logger.writeWarning("Could not remove flag: {}".format(e))
class MDNSEngine(object):
def __init__(self):
self.running = False
self.dnsServiceControllers = []
def start(self):
if not self.running:
self.logger = Logger('mdns-engine')
self.subscriptionController = MDNSSubscriptionController()
self.registrationController = MDNSRegistrationController()
self.interfaceController = MDNSInterfaceController(self.logger)
self.running = True
def stop(self):
self.close()
def close(self):
self.logger.writeDebug("MDNS Engine Closed")
self.subscriptionController.close()
self.registrationController.close()
self.interfaceController.close()
for controller in self.dnsServiceControllers:
controller.close()
self.running = False
def run(self):
pass
def _autostart_if_required(self):
if not self.running:
self.start()
def register(self,
name,
regtype,
port,
txtRecord="",
callback=None,
interfaceIps=None):
self._autostart_if_required()
callbackHandler = MDNSAdvertisementCallbackHandler(
callback, regtype, name, port, txtRecord)
if not interfaceIps:
interfaceIps = []
try:
interfaces = self.interfaceController.getInterfaces(interfaceIps)
except InterfaceNotFoundException:
callbackHandler.entryFailed()
return
registration = MDNSRegistration(interfaces, name, regtype, port,
txtRecord)
self._add_registration_handle_errors(registration, callbackHandler)
def _add_registration_handle_errors(self, registration, callbackHandler):
try:
self.registrationController.addRegistration(registration)
except zeroconf.NonUniqueNameException:
callbackHandler.entryCollision()
except ServiceAlreadyExistsException:
callbackHandler.entryCollision()
except zeroconf.Error as e:
callbackHandler.entryFailed()
print(str(e))
else:
callbackHandler.entryEstablished()
def update(self, name, regtype, txtRecord=None):
self._autostart_if_required()
try:
registration = self.registrationController.registrations[regtype][
name]
except KeyError:
self.logger.writeError(
"Could not update registraton type: {} with name {}"
" - registration not found".format(regtype, name))
raise ServiceNotFoundException
registration.update(name=name, regtype=regtype, txtRecord=txtRecord)
def unregister(self, name, regType):
self._autostart_if_required()
self.registrationController.removeRegistration(name, regType)
def callback_on_services(self,
regtype,
callback,
registerOnly=True,
domain=None):
self._autostart_if_required()
doDNSDiscover = config['dns_discover']
domDNSDiscover = config['mdns_discover']
if domDNSDiscover:
listener = MDNSListener(callback, registerOnly)
self.subscriptionController.addSubscription(listener, regtype)
if doDNSDiscover:
dnsServiceController = DNSServiceController(
regtype, callback, self.logger, registerOnly, domain)
dnsServiceController.start()
self.dnsServiceControllers.append(dnsServiceController)
