def htmlify(r, mimetype, status=200): # if the request was proxied via the nodefacade, use the original host in response. # additional external proxies could cause an issue here, so we can override the hostname # in the config to an externally-defined one if there are multiple reverse proxies path = request.headers.get('X-Forwarded-Path', request.path) host = _config.get('node_hostname') if host is None: host = request.headers.get('X-Forwarded-Host', request.host) if _config.get('https_mode') == 'enabled': scheme = 'https' else: scheme = request.headers.get('X-Forwarded-Proto', urlparse(request.url).scheme) base_url = "{}://{}".format(scheme, host) title = '<a href="' + base_url + '/">' + base_url + '</a>' t = base_url for x in path.split('/'): if x != '': t += '/' + x.strip('/') title += '/<a href="' + t + '">' + x.strip('/') + '</a>' return IppResponse(highlight(json.dumps(r, indent=4, cls=NMOSJSONEncoder), JsonLexer(), LinkingHTMLFormatter(linenos='table', full=True, title=title)), mimetype='text/html', status=status)
def updateHost(): if _config.get('node_hostname') is not None: return _config.get('node_hostname') elif _config.get('prefer_ipv6', False) is False: return getLocalIP() else: return "[" + getLocalIP(None, socket.AF_INET6) + "]"
def __init__(self, logger=None, mdns_updater=None, auth_registry=None): self.logger = Logger("aggregator_proxy", logger) self.mdnsbridge = IppmDNSBridge(logger=self.logger) self.aggregator_apiversion = None self.service_type = None self._set_api_version_and_srv_type( _config.get('nodefacade').get('NODE_REGVERSION')) self.aggregator = None self.registration_order = [ "device", "source", "flow", "sender", "receiver" ] self._mdns_updater = mdns_updater # '_node_data' is a local mirror of aggregated items. self._node_data = { 'node': None, 'registered': False, 'entities': { 'resource': {} } } self._running = True self._aggregator_list_stale = True self._aggregator_failure = False # Variable to flag when aggregator has returned and unexpected error self._backoff_active = False self._backoff_period = 0 self.auth_registrar = None # Class responsible for registering with Auth Server self.auth_registry = auth_registry # Top level class that tracks locally registered OAuth clients self.auth_client = None # Instance of Oauth client responsible for performing token requests self._reg_queue = gevent.queue.Queue() self.main_thread = gevent.spawn(self._main_thread) self.queue_thread = gevent.spawn(self._process_queue)
def _send_request(self, method, aggregator, url_path, data=None): """Low level method to send a HTTP request""" url = urljoin(aggregator, url_path) self.logger.writeDebug("{} {}".format(method, url)) # We give a long(ish) timeout below, as the async request may succeed after the timeout period # has expired, causing the node to be registered twice (potentially at different aggregators). # Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic # to web clients - so, sacrifice a little timeliness for things working as designed the # majority of the time... kwargs = {"method": method, "url": url, "json": data, "timeout": 1.0} if _config.get('prefer_ipv6') is True: kwargs["proxies"] = {'http': ''} # If not in OAuth mode, perform standard request if OAUTH_MODE is False or self.auth_client is None: return requests.request(**kwargs) else: # If in OAuth Mode, use OAuth client to automatically fetch token / refresh token if expired with self.auth_registry.app.app_context(): try: return self.auth_client.request(**kwargs) # General OAuth Error (e.g. incorrect request details, invalid client, etc.) except OAuth2Error as e: self.logger.writeError( "Failed to fetch token before making API call to {}. {}" .format(url, e)) self.auth_registrar = self.auth_client = None
def __init__(self, node_name='nmos-root', _parent=None): logFormat = logging.Formatter( '%(asctime)s : %(name)s : %(levelname)s : %(message)s') logLevel = getattr(logging, _config.get("logging").get("level")) logOutputs = _config.get("logging").get("output") self.log = logging.getLogger(node_name) self.log.setLevel(logLevel) if not PurePythonLogger.logsOpened: if "file" in logOutputs: try: fileLoc = _config.get("logging").get("fileLocation") fileHandler = logging.FileHandler(fileLoc) fileHandler.setLevel(logLevel) fileHandler.setFormatter(logFormat) self.log.addHandler(fileHandler) except: pass if "stdout" in logOutputs: streamHandler = logging.StreamHandler(sys.stdout) streamHandler.setLevel(logLevel) streamHandler.setFormatter(logFormat) self.log.addHandler(streamHandler) if "stderr" in logOutputs: streamHandler = logging.StreamHandler(sys.stderr) streamHandler.setLevel(logLevel) streamHandler.setFormatter(logFormat) self.log.addHandler(streamHandler) self.log.propagate = False PurePythonLogger.logsOpened = True logging.info("Logging started...")
def __init__(self, oauth_config=None): self.app = Flask(__name__) self.app.response_class = IppResponse self.app.before_first_request(self.torun) self.sockets = Sockets(self.app) self.socks = dict() self.port = 0 # authentication/authorisation self._oauth_config = oauth_config self._authorize = self.default_authorize self._authenticate = self.default_authenticate self.add_routes(self, basepath='') # Enable ProxyFix middleware if required if _config.get('fix_proxy') == 'enabled': self.app.wsgi_app = ProxyFix(self.app.wsgi_app)
# MDNS Service Names LEGACY_REG_MDNSTYPE = "nmos-registration" REGISTRATION_MDNSTYPE = "nmos-register" # Registry path AGGREGATOR_APINAMESPACE = "x-nmos" AGGREGATOR_APINAME = "registration" AGGREGATOR_APIROOT = AGGREGATOR_APINAMESPACE + '/' + AGGREGATOR_APINAME # Exponential back off global vars BACKOFF_INITIAL_TIMOUT_SECONDS = 5 BACKOFF_MAX_TIMEOUT_SECONDS = 40 # OAuth client global vars FQDN = getfqdn() OAUTH_MODE = _config.get("oauth_mode", False) ALLOWED_SCOPE = "registration" ALLOWED_GRANTS = ["authorization_code", "refresh_token", "client_credentials"] class InvalidRequest(Exception): """Client Side Error during request, HTTP 4xx""" def __init__(self, status_code=400): super(InvalidRequest, self).__init__("Invalid Request, code {}".format(status_code)) self.status_code = status_code class ServerSideError(Exception): """Exception raised when a HTTP 5xx, timeout or inability to connect returned during request. This indicates a server side or connectivity issue"""
def _SEND(self, method, url, data=None): if self.aggregator == "": self.aggregator = self._get_api_href() headers = None if data is not None: data = json.dumps(data) headers = {"Content-Type": "application/json"} url = AGGREGATOR_APINAMESPACE + "/" + AGGREGATOR_APINAME + "/" + AGGREGATOR_APIVERSION + url for i in range(0, 3): if self.aggregator == "": self.logger.writeWarning( "No aggregator available on the network or mdnsbridge unavailable" ) raise NoAggregator(self._mdns_updater) self.logger.writeDebug("{} {}".format( method, urljoin(self.aggregator, url))) # We give a long(ish) timeout below, as the async request may succeed after the timeout period # has expired, causing the node to be registered twice (potentially at different aggregators). # Whilst this isn't a problem in practice, it may cause excessive churn in websocket traffic # to web clients - so, sacrifice a little timeliness for things working as designed the # majority of the time... try: if _config.get('prefer_ipv6') is False: R = requests.request(method, urljoin(self.aggregator, url), data=data, timeout=1.0, headers=headers) else: R = requests.request(method, urljoin(self.aggregator, url), data=data, timeout=1.0, headers=headers, proxies={'http': ''}) if R is None: # Try another aggregator self.logger.writeWarning( "No response from aggregator {}".format( self.aggregator)) elif R.status_code in [200, 201]: if R.headers.get( "content-type", "text/plain").startswith("application/json"): return R.json() else: return R.content elif R.status_code == 204: return elif (R.status_code / 100) == 4: self.logger.writeWarning( "{} response from aggregator: {} {}".format( R.status_code, method, urljoin(self.aggregator, url))) raise InvalidRequest(R.status_code, self._mdns_updater) else: self.logger.writeWarning( "Unexpected status from aggregator {}: {}, {}".format( self.aggregator, R.status_code, R.content)) except requests.exceptions.RequestException as ex: # Log a warning, then let another aggregator be chosen self.logger.writeWarning("{} from aggregator {}".format( ex, self.aggregator)) # This aggregator is non-functional self.aggregator = self._get_api_href() self.logger.writeInfo("Updated aggregator to {} (try {})".format( self.aggregator, i)) raise TooManyRetries(self._mdns_updater)
import requests import json import time import gevent import gevent.queue from nmoscommon.logger import Logger from nmoscommon.mdnsbridge import IppmDNSBridge from nmoscommon.mdns.mdnsExceptions import ServiceNotFoundException from nmoscommon.nmoscommonconfig import config as _config import traceback AGGREGATOR_APIVERSION = _config.get('nodefacade').get('NODE_REGVERSION') AGGREGATOR_APINAMESPACE = "x-nmos" AGGREGATOR_APINAME = "registration" LEGACY_REG_MDNSTYPE = "nmos-registration" REGISTRATION_MDNSTYPE = "nmos-register" class NoAggregator(Exception): def __init__(self, mdns_updater=None): if mdns_updater is not None: mdns_updater.inc_P2P_enable_count() super(NoAggregator, self).__init__("No Registration API found") class InvalidRequest(Exception):
def start(self): if self.running: gevent.signal_handler(signal.SIGINT, self.sig_handler) gevent.signal_handler(signal.SIGTERM, self.sig_handler) gevent.signal_handler(signal.SIGHUP, self.sig_hup_handler) self.mdns.start() self.node_id = get_node_id() node_version = str(ptptime.ptp_detail()[0]) + ":" + str( ptptime.ptp_detail()[1]) node_data = { "id": self.node_id, "label": _config.get('node_label', FQDN), "description": _config.get('node_description', "Node on {}".format(FQDN)), "tags": _config.get('node_tags', {}), "href": self.generate_href(), "host": HOST, "services": [], "hostname": HOSTNAME, "caps": {}, "version": node_version, "api": { "versions": NODE_APIVERSIONS, "endpoints": self.generate_endpoints(), }, "clocks": [], "interfaces": self.list_interfaces() } self.registry = FacadeRegistry(self.mappings.keys(), self.aggregator, self.mdns_updater, self.node_id, node_data, self.logger) self.registry_cleaner = FacadeRegistryCleaner(self.registry) self.registry_cleaner.start() self.httpServer = HttpServer( FacadeAPI, PORT, '0.0.0.0', api_args=[self.registry, self.auth_registry]) self.httpServer.start() while not self.httpServer.started.is_set(): self.logger.writeInfo('Waiting for httpserver to start...') self.httpServer.started.wait() if self.httpServer.failed is not None: raise self.httpServer.failed self.logger.writeInfo("Running on port: {}".format( self.httpServer.port)) try: self.logger.writeInfo("Registering as {}...".format(self.node_id)) self.aggregator.register( 'node', self.node_id, **translate_api_version(node_data, "node", NODE_REGVERSION)) except Exception as e: self.logger.writeWarning("Could not register: {}".format( e.__repr__())) self.interface = FacadeInterface(self.registry, self.logger) self.interface.start()
from .api import NODE_APIVERSIONS, NODE_REGVERSION, PROTOCOL, FacadeAPI # noqa E402 from .registry import FacadeRegistry, FacadeRegistryCleaner # noqa E402 from .aggregator import Aggregator, MDNSUpdater, ALLOWED_SCOPE, FQDN # noqa E402 from .authclient import AuthRegistry # noqa E402 from .serviceinterface import FacadeInterface # noqa E402 NS = 'urn:x-bbcrd:ips:ns:0.1' PORT = 12345 HOSTNAME = gethostname().split(".", 1)[0] # HTTPS under test only at present # enabled = Use HTTPS only in all URLs and mDNS adverts # disabled = Use HTTP only in all URLs and mDNS adverts # mixed = Use HTTP in all URLs, but additionally advertise an HTTPS endpoint for discovery of this API only ENABLE_P2P = _config.get('node_p2p_enable', True) OAUTH_MODE = _config.get('oauth_mode', False) # BYPASS AUTHLIB SECURITY CHECK DUE TO REVERSE PROXY environ["AUTHLIB_INSECURE_TRANSPORT"] = "1" def updateHost(): if _config.get('node_hostname') is not None: return _config.get('node_hostname') elif _config.get('prefer_hostnames', False) is True: return FQDN elif _config.get('prefer_ipv6', False) is False: return getLocalIP() else: return "[" + getLocalIP(None, socket.AF_INET6) + "]"
from nmoscommon.mdns import MDNSEngine from nmoscommon.logger import Logger from nmoscommon import ptptime from nmoscommon.nmoscommonconfig import config as _config import socket NS = 'urn:x-bbcrd:ips:ns:0.1' PORT = 12345 HOSTNAME = gethostname().split(".", 1)[0] FQDN = getfqdn() # HTTPS under test only at present # enabled = Use HTTPS only in all URLs and mDNS adverts # disabled = Use HTTP only in all URLs and mDNS adverts # mixed = Use HTTP in all URLs, but additionally advertise an HTTPS endpoint for discovery of this API only HTTPS_MODE = _config.get('https_mode', 'disabled') ENABLE_P2P = _config.get('node_p2p_enable', True) def updateHost(): if _config.get('node_hostname') is not None: return _config.get('node_hostname') elif _config.get('prefer_ipv6', False) is False: return getLocalIP() else: return "[" + getLocalIP(None, socket.AF_INET6) + "]" HOST = updateHost() DNS_SD_HTTP_PORT = 80 DNS_SD_HTTPS_PORT = 443
from six.moves import range as xrange from gevent import monkey monkey.patch_all() import gevent import json import requests import websocket import itertools from nmoscommon.logger import Logger from nmoscommon.nmoscommonconfig import config as _config QUERY_APIVERSION = _config.get('nodefacade').get('NODE_REGVERSION') QUERY_APINAMESPACE = "x-nmos" QUERY_APINAME = "query" QUERY_MDNSTYPE = "nmos-query" class BadSubscriptionError(Exception): pass class QueryNotFoundError(Exception): pass class QueryService(object): def __init__(self,
# Library not available, use fallback IPP_UTILS_CLOCK_AVAILABLE = False HEARTBEAT_TIMEOUT = 12 # Seconds CLEANUP_INTERVAL = 5 # Seconds # TODO: Enumerate return codes better? RES_SUCCESS = 0 RES_EXISTS = 1 RES_NOEXISTS = 2 RES_UNAUTHORISED = 3 RES_UNSUPPORTED = 4 RES_OTHERERROR = 5 HTTPS_MODE = _config.get('https_mode', 'disabled') class FacadeRegistryCleaner(threading.Thread): def __init__(self, registry): self.stopping = False self.registry = registry super(FacadeRegistryCleaner, self).__init__() self.daemon = True def run(self): loopcount = 0 while not self.stopping: time.sleep(1) loopcount += 1 if loopcount >= CLEANUP_INTERVAL:
# limitations under the License. from __future__ import print_function import requests from os import urandom from flask import request, url_for, redirect from six import itervalues from six.moves.urllib.parse import urljoin from nmoscommon.nmoscommonconfig import config as _config from nmoscommon.webapi import WebAPI, route, resource_route, abort # Config Parameters PROTOCOL = "https" if _config.get('https_mode') == "enabled" else "http" NODE_REGVERSION = _config.get('nodefacade', {}).get('NODE_REGVERSION', 'v1.2') # Node API Path Information NODE_APINAMESPACE = "x-nmos" NODE_APINAME = "node" NODE_APIROOT = '/' + NODE_APINAMESPACE + '/' + NODE_APINAME + '/' NODE_APIVERSIONS = ["v1.0", "v1.1", "v1.2", "v1.3"] if PROTOCOL == "https": NODE_APIVERSIONS.remove("v1.0") RESOURCE_TYPES = ["sources", "flows", "devices", "senders", "receivers"] class FacadeAPI(WebAPI): def __init__(self, nmos_registry, auth_registry=None):
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function from nmoscommon.webapi import * from nmoscommon.webapi import WebAPI, route, resource_route, abort from six.moves.urllib.parse import urljoin import requests from socket import gethostname from nmoscommon.nmoscommonconfig import config as _config NODE_APIVERSIONS = ["v1.0", "v1.1", "v1.2", "v1.3"] if _config.get("https_mode", "disabled") == "enabled": NODE_APIVERSIONS.remove("v1.0") NODE_REGVERSION = _config.get('nodefacade', {}).get('NODE_REGVERSION', 'v1.2') NODE_APINAMESPACE = "x-nmos" NODE_APINAME = "node" HOSTNAME = gethostname().split(".", 1)[0] RESOURCE_TYPES = ["sources", "flows", "devices", "senders", "receivers"] class FacadeAPI(WebAPI): def __init__(self, registry): self.registry = registry self.node_id = registry.node_id super(FacadeAPI, self).__init__() @route('/')