def htmlify(r, mimetype, status=200):
# if the request was proxied via the nodefacade, use the original host in response.
# additional external proxies could cause an issue here, so we can override the hostname
# in the config to an externally-defined one if there are multiple reverse proxies
path = request.headers.get('X-Forwarded-Path', request.path)
host = _config.get('node_hostname')
if host is None:
host = request.headers.get('X-Forwarded-Host', request.host)
if _config.get('https_mode') == 'enabled':
scheme = 'https'
else:
scheme = request.headers.get('X-Forwarded-Proto', urlparse(request.url).scheme)
base_url = "{}://{}".format(scheme, host)
title = '<a href="' + base_url + '/">' + base_url + '</a>'
t = base_url
for x in path.split('/'):
if x != '':
t += '/' + x.strip('/')
title += '/<a href="' + t + '">' + x.strip('/') + '</a>'
return IppResponse(highlight(json.dumps(r, indent=4, cls=NMOSJSONEncoder),
JsonLexer(),
LinkingHTMLFormatter(linenos='table',
full=True,
title=title)),
mimetype='text/html',
status=status)
def updateHost():
if _config.get('node_hostname') is not None:
return _config.get('node_hostname')
elif _config.get('prefer_ipv6', False) is False:
return getLocalIP()
else:
return "[" + getLocalIP(None, socket.AF_INET6) + "]"
def __init__(self, logger=None, mdns_updater=None, auth_registry=None):
self.logger = Logger("aggregator_proxy", logger)
self.mdnsbridge = IppmDNSBridge(logger=self.logger)
self.aggregator_apiversion = None
self.service_type = None
self._set_api_version_and_srv_type(
_config.get('nodefacade').get('NODE_REGVERSION'))
self.aggregator = None
self.registration_order = [
"device", "source", "flow", "sender", "receiver"
]
self._mdns_updater = mdns_updater
# '_node_data' is a local mirror of aggregated items.
self._node_data = {
'node': None,
'registered': False,
'entities': {
'resource': {}
}
}
self._running = True
self._aggregator_list_stale = True
self._aggregator_failure = False # Variable to flag when aggregator has returned and unexpected error
self._backoff_active = False
self._backoff_period = 0
self.auth_registrar = None # Class responsible for registering with Auth Server
self.auth_registry = auth_registry # Top level class that tracks locally registered OAuth clients
self.auth_client = None # Instance of Oauth client responsible for performing token requests
self._reg_queue = gevent.queue.Queue()
self.main_thread = gevent.spawn(self._main_thread)
self.queue_thread = gevent.spawn(self._process_queue)
def _send_request(self, method, aggregator, url_path, data=None):
"""Low level method to send a HTTP request"""
url = urljoin(aggregator, url_path)
self.logger.writeDebug("{} {}".format(method, url))
# We give a long(ish) timeout below, as the async request may succeed after the timeout period
# has expired, causing the node to be registered twice (potentially at different aggregators).
# Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic
# to web clients - so, sacrifice a little timeliness for things working as designed the
# majority of the time...
kwargs = {"method": method, "url": url, "json": data, "timeout": 1.0}
if _config.get('prefer_ipv6') is True:
kwargs["proxies"] = {'http': ''}
# If not in OAuth mode, perform standard request
if OAUTH_MODE is False or self.auth_client is None:
return requests.request(**kwargs)
else:
# If in OAuth Mode, use OAuth client to automatically fetch token / refresh token if expired
with self.auth_registry.app.app_context():
try:
return self.auth_client.request(**kwargs)
# General OAuth Error (e.g. incorrect request details, invalid client, etc.)
except OAuth2Error as e:
self.logger.writeError(
"Failed to fetch token before making API call to {}. {}"
.format(url, e))
self.auth_registrar = self.auth_client = None
def __init__(self, node_name='nmos-root', _parent=None):
logFormat = logging.Formatter(
'%(asctime)s : %(name)s : %(levelname)s : %(message)s')
logLevel = getattr(logging, _config.get("logging").get("level"))
logOutputs = _config.get("logging").get("output")
self.log = logging.getLogger(node_name)
self.log.setLevel(logLevel)
if not PurePythonLogger.logsOpened:
if "file" in logOutputs:
try:
fileLoc = _config.get("logging").get("fileLocation")
fileHandler = logging.FileHandler(fileLoc)
fileHandler.setLevel(logLevel)
fileHandler.setFormatter(logFormat)
self.log.addHandler(fileHandler)
except:
pass
if "stdout" in logOutputs:
streamHandler = logging.StreamHandler(sys.stdout)
streamHandler.setLevel(logLevel)
streamHandler.setFormatter(logFormat)
self.log.addHandler(streamHandler)
if "stderr" in logOutputs:
streamHandler = logging.StreamHandler(sys.stderr)
streamHandler.setLevel(logLevel)
streamHandler.setFormatter(logFormat)
self.log.addHandler(streamHandler)
self.log.propagate = False
PurePythonLogger.logsOpened = True
logging.info("Logging started...")
def __init__(self, oauth_config=None):
self.app = Flask(__name__)
self.app.response_class = IppResponse
self.app.before_first_request(self.torun)
self.sockets = Sockets(self.app)
self.socks = dict()
self.port = 0
# authentication/authorisation
self._oauth_config = oauth_config
self._authorize = self.default_authorize
self._authenticate = self.default_authenticate
self.add_routes(self, basepath='')
# Enable ProxyFix middleware if required
if _config.get('fix_proxy') == 'enabled':
self.app.wsgi_app = ProxyFix(self.app.wsgi_app)
# MDNS Service Names
LEGACY_REG_MDNSTYPE = "nmos-registration"
REGISTRATION_MDNSTYPE = "nmos-register"
# Registry path
AGGREGATOR_APINAMESPACE = "x-nmos"
AGGREGATOR_APINAME = "registration"
AGGREGATOR_APIROOT = AGGREGATOR_APINAMESPACE + '/' + AGGREGATOR_APINAME
# Exponential back off global vars
BACKOFF_INITIAL_TIMOUT_SECONDS = 5
BACKOFF_MAX_TIMEOUT_SECONDS = 40
# OAuth client global vars
FQDN = getfqdn()
OAUTH_MODE = _config.get("oauth_mode", False)
ALLOWED_SCOPE = "registration"
ALLOWED_GRANTS = ["authorization_code", "refresh_token", "client_credentials"]
class InvalidRequest(Exception):
"""Client Side Error during request, HTTP 4xx"""
def __init__(self, status_code=400):
super(InvalidRequest,
self).__init__("Invalid Request, code {}".format(status_code))
self.status_code = status_code
class ServerSideError(Exception):
"""Exception raised when a HTTP 5xx, timeout or inability to connect returned during request.
This indicates a server side or connectivity issue"""
def _SEND(self, method, url, data=None):
if self.aggregator == "":
self.aggregator = self._get_api_href()
headers = None
if data is not None:
data = json.dumps(data)
headers = {"Content-Type": "application/json"}
url = AGGREGATOR_APINAMESPACE + "/" + AGGREGATOR_APINAME + "/" + AGGREGATOR_APIVERSION + url
for i in range(0, 3):
if self.aggregator == "":
self.logger.writeWarning(
"No aggregator available on the network or mdnsbridge unavailable"
)
raise NoAggregator(self._mdns_updater)
self.logger.writeDebug("{} {}".format(
method, urljoin(self.aggregator, url)))
# We give a long(ish) timeout below, as the async request may succeed after the timeout period
# has expired, causing the node to be registered twice (potentially at different aggregators).
# Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic
# to web clients - so, sacrifice a little timeliness for things working as designed the
# majority of the time...
try:
if _config.get('prefer_ipv6') is False:
R = requests.request(method,
urljoin(self.aggregator, url),
data=data,
timeout=1.0,
headers=headers)
else:
R = requests.request(method,
urljoin(self.aggregator, url),
data=data,
timeout=1.0,
headers=headers,
proxies={'http': ''})
if R is None:
# Try another aggregator
self.logger.writeWarning(
"No response from aggregator {}".format(
self.aggregator))
elif R.status_code in [200, 201]:
if R.headers.get(
"content-type",
"text/plain").startswith("application/json"):
return R.json()
else:
return R.content
elif R.status_code == 204:
return
elif (R.status_code / 100) == 4:
self.logger.writeWarning(
"{} response from aggregator: {} {}".format(
R.status_code, method,
urljoin(self.aggregator, url)))
raise InvalidRequest(R.status_code, self._mdns_updater)
else:
self.logger.writeWarning(
"Unexpected status from aggregator {}: {}, {}".format(
self.aggregator, R.status_code, R.content))
except requests.exceptions.RequestException as ex:
# Log a warning, then let another aggregator be chosen
self.logger.writeWarning("{} from aggregator {}".format(
ex, self.aggregator))
# This aggregator is non-functional
self.aggregator = self._get_api_href()
self.logger.writeInfo("Updated aggregator to {} (try {})".format(
self.aggregator, i))
raise TooManyRetries(self._mdns_updater)
import requests
import json
import time
import gevent
import gevent.queue
from nmoscommon.logger import Logger
from nmoscommon.mdnsbridge import IppmDNSBridge
from nmoscommon.mdns.mdnsExceptions import ServiceNotFoundException
from nmoscommon.nmoscommonconfig import config as _config
import traceback
AGGREGATOR_APIVERSION = _config.get('nodefacade').get('NODE_REGVERSION')
AGGREGATOR_APINAMESPACE = "x-nmos"
AGGREGATOR_APINAME = "registration"
LEGACY_REG_MDNSTYPE = "nmos-registration"
REGISTRATION_MDNSTYPE = "nmos-register"
class NoAggregator(Exception):
def __init__(self, mdns_updater=None):
if mdns_updater is not None:
mdns_updater.inc_P2P_enable_count()
super(NoAggregator, self).__init__("No Registration API found")
class InvalidRequest(Exception):
def start(self):
if self.running:
gevent.signal_handler(signal.SIGINT, self.sig_handler)
gevent.signal_handler(signal.SIGTERM, self.sig_handler)
gevent.signal_handler(signal.SIGHUP, self.sig_hup_handler)
self.mdns.start()
self.node_id = get_node_id()
node_version = str(ptptime.ptp_detail()[0]) + ":" + str(
ptptime.ptp_detail()[1])
node_data = {
"id":
self.node_id,
"label":
_config.get('node_label', FQDN),
"description":
_config.get('node_description', "Node on {}".format(FQDN)),
"tags":
_config.get('node_tags', {}),
"href":
self.generate_href(),
"host":
HOST,
"services": [],
"hostname":
HOSTNAME,
"caps": {},
"version":
node_version,
"api": {
"versions": NODE_APIVERSIONS,
"endpoints": self.generate_endpoints(),
},
"clocks": [],
"interfaces":
self.list_interfaces()
}
self.registry = FacadeRegistry(self.mappings.keys(), self.aggregator,
self.mdns_updater, self.node_id,
node_data, self.logger)
self.registry_cleaner = FacadeRegistryCleaner(self.registry)
self.registry_cleaner.start()
self.httpServer = HttpServer(
FacadeAPI,
PORT,
'0.0.0.0',
api_args=[self.registry, self.auth_registry])
self.httpServer.start()
while not self.httpServer.started.is_set():
self.logger.writeInfo('Waiting for httpserver to start...')
self.httpServer.started.wait()
if self.httpServer.failed is not None:
raise self.httpServer.failed
self.logger.writeInfo("Running on port: {}".format(
self.httpServer.port))
try:
self.logger.writeInfo("Registering as {}...".format(self.node_id))
self.aggregator.register(
'node', self.node_id,
**translate_api_version(node_data, "node", NODE_REGVERSION))
except Exception as e:
self.logger.writeWarning("Could not register: {}".format(
e.__repr__()))
self.interface = FacadeInterface(self.registry, self.logger)
self.interface.start()
from .api import NODE_APIVERSIONS, NODE_REGVERSION, PROTOCOL, FacadeAPI # noqa E402
from .registry import FacadeRegistry, FacadeRegistryCleaner # noqa E402
from .aggregator import Aggregator, MDNSUpdater, ALLOWED_SCOPE, FQDN # noqa E402
from .authclient import AuthRegistry # noqa E402
from .serviceinterface import FacadeInterface # noqa E402
NS = 'urn:x-bbcrd:ips:ns:0.1'
PORT = 12345
HOSTNAME = gethostname().split(".", 1)[0]
# HTTPS under test only at present
# enabled = Use HTTPS only in all URLs and mDNS adverts
# disabled = Use HTTP only in all URLs and mDNS adverts
# mixed = Use HTTP in all URLs, but additionally advertise an HTTPS endpoint for discovery of this API only
ENABLE_P2P = _config.get('node_p2p_enable', True)
OAUTH_MODE = _config.get('oauth_mode', False)
# BYPASS AUTHLIB SECURITY CHECK DUE TO REVERSE PROXY
environ["AUTHLIB_INSECURE_TRANSPORT"] = "1"
def updateHost():
if _config.get('node_hostname') is not None:
return _config.get('node_hostname')
elif _config.get('prefer_hostnames', False) is True:
return FQDN
elif _config.get('prefer_ipv6', False) is False:
return getLocalIP()
else:
return "[" + getLocalIP(None, socket.AF_INET6) + "]"
from nmoscommon.mdns import MDNSEngine
from nmoscommon.logger import Logger
from nmoscommon import ptptime
from nmoscommon.nmoscommonconfig import config as _config
import socket
NS = 'urn:x-bbcrd:ips:ns:0.1'
PORT = 12345
HOSTNAME = gethostname().split(".", 1)[0]
FQDN = getfqdn()
# HTTPS under test only at present
# enabled = Use HTTPS only in all URLs and mDNS adverts
# disabled = Use HTTP only in all URLs and mDNS adverts
# mixed = Use HTTP in all URLs, but additionally advertise an HTTPS endpoint for discovery of this API only
HTTPS_MODE = _config.get('https_mode', 'disabled')
ENABLE_P2P = _config.get('node_p2p_enable', True)
def updateHost():
if _config.get('node_hostname') is not None:
return _config.get('node_hostname')
elif _config.get('prefer_ipv6', False) is False:
return getLocalIP()
else:
return "[" + getLocalIP(None, socket.AF_INET6) + "]"
HOST = updateHost()
DNS_SD_HTTP_PORT = 80
DNS_SD_HTTPS_PORT = 443
from six.moves import range as xrange
from gevent import monkey
monkey.patch_all()
import gevent
import json
import requests
import websocket
import itertools
from nmoscommon.logger import Logger
from nmoscommon.nmoscommonconfig import config as _config
QUERY_APIVERSION = _config.get('nodefacade').get('NODE_REGVERSION')
QUERY_APINAMESPACE = "x-nmos"
QUERY_APINAME = "query"
QUERY_MDNSTYPE = "nmos-query"
class BadSubscriptionError(Exception):
pass
class QueryNotFoundError(Exception):
pass
class QueryService(object):
def __init__(self,
# Library not available, use fallback
IPP_UTILS_CLOCK_AVAILABLE = False
HEARTBEAT_TIMEOUT = 12 # Seconds
CLEANUP_INTERVAL = 5 # Seconds
# TODO: Enumerate return codes better?
RES_SUCCESS = 0
RES_EXISTS = 1
RES_NOEXISTS = 2
RES_UNAUTHORISED = 3
RES_UNSUPPORTED = 4
RES_OTHERERROR = 5
HTTPS_MODE = _config.get('https_mode', 'disabled')
class FacadeRegistryCleaner(threading.Thread):
def __init__(self, registry):
self.stopping = False
self.registry = registry
super(FacadeRegistryCleaner, self).__init__()
self.daemon = True
def run(self):
loopcount = 0
while not self.stopping:
time.sleep(1)
loopcount += 1
if loopcount >= CLEANUP_INTERVAL:
# limitations under the License.
from __future__ import print_function
import requests
from os import urandom
from flask import request, url_for, redirect
from six import itervalues
from six.moves.urllib.parse import urljoin
from nmoscommon.nmoscommonconfig import config as _config
from nmoscommon.webapi import WebAPI, route, resource_route, abort
# Config Parameters
PROTOCOL = "https" if _config.get('https_mode') == "enabled" else "http"
NODE_REGVERSION = _config.get('nodefacade', {}).get('NODE_REGVERSION', 'v1.2')
# Node API Path Information
NODE_APINAMESPACE = "x-nmos"
NODE_APINAME = "node"
NODE_APIROOT = '/' + NODE_APINAMESPACE + '/' + NODE_APINAME + '/'
NODE_APIVERSIONS = ["v1.0", "v1.1", "v1.2", "v1.3"]
if PROTOCOL == "https":
NODE_APIVERSIONS.remove("v1.0")
RESOURCE_TYPES = ["sources", "flows", "devices", "senders", "receivers"]
class FacadeAPI(WebAPI):
def __init__(self, nmos_registry, auth_registry=None):
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from nmoscommon.webapi import *
from nmoscommon.webapi import WebAPI, route, resource_route, abort
from six.moves.urllib.parse import urljoin
import requests
from socket import gethostname
from nmoscommon.nmoscommonconfig import config as _config
NODE_APIVERSIONS = ["v1.0", "v1.1", "v1.2", "v1.3"]
if _config.get("https_mode", "disabled") == "enabled":
NODE_APIVERSIONS.remove("v1.0")
NODE_REGVERSION = _config.get('nodefacade', {}).get('NODE_REGVERSION', 'v1.2')
NODE_APINAMESPACE = "x-nmos"
NODE_APINAME = "node"
HOSTNAME = gethostname().split(".", 1)[0]
RESOURCE_TYPES = ["sources", "flows", "devices", "senders", "receivers"]
class FacadeAPI(WebAPI):
def __init__(self, registry):
self.registry = registry
self.node_id = registry.node_id
super(FacadeAPI, self).__init__()
@route('/')
