def removeHost(self, msg): """Remove a host from the Mininet topology msg: a RemoveHostMessage object""" # find adjacent switch(es) sw = [intf.link.intf2.node.name for intf in self.net.get(msg.name).intfList() if self.net.topo.isSwitch(intf.link.intf2.node.name)] self.net.get(msg.name).terminate() self.net.topo.g.node.pop(msg.name, None) self.net.hosts = [h for h in self.net.hosts if h.name != msg.name] del self.net.nameToNode[msg.name] if len(sw) == 0: logger.debug("deleting host connected to 0 switches") return elif len(sw) > 1: raise Exception("cannot support hosts connected to %s switches", len(sw)) swname = str(sw[0]) swobj = self.net.get(swname) swport = self.net.topo.port(swname, msg.name)[0] intf = str(swobj.intfList()[swport]) swobj.detach(intf) del swobj.nameToIntf[intf] del swobj.intfs[swport] del self.cache_name[msg.name] del self.cache_id[msg.hid]
def start(self, cargs=None): """Start the Pox process cargs: arguments to pass to the controller""" pox = os.path.join(ravel.util.Config.PoxDir, "pox.py") if not os.path.exists(pox): logger.error("cannot find pox.py at: {0}. " "Is PoxDir set correctly in ravel.cfg?".format(pox)) sys.exit(0) if cargs is None: cargs = [ "log.level", "--DEBUG", "openflow.of_01", "--port={0}".format(ravel.util.Config.PoxPort), self.app, "openflow.discovery" ] ravel.util.append_path(ravel.util.resource_file()) env = os.environ.copy() env["PYTHONPATH"] = ":".join(sys.path) logger.debug("pox with params: %s", " ".join(cargs)) self.proc = subprocess.Popen([pox] + cargs, env=env, preexec_fn=preexec_fn, stdout=open("/tmp/pox.log", "wb"), stderr=open("/tmp/pox.err", "wb"))
def unload(self, db): """Unload the application from the specified database db: a ravel.db.RavelDb instance containing the application""" for component in self.components: component.drop(db) logger.debug("unloaded application %s", self.name)
def init(self, db, env): """Initialize the application without loading it into the database. db: a ravel.db.RavelDb instance to be passed to the application's sub-shell env: a ravel.env.Environment instance of the CLI's executing environment to be passed to the application's sub-shell""" if not self.pyfile: return # discover sql components (tables, views, functions) if self.sqlfile is not None: with open(self.sqlfile) as f: self.components = discoverComponents(f.read()) logger.debug("discovered {0} components: {1}" .format(self.name, self.components)) # if needed, add path filepath = os.path.dirname(self.pyfile) ravel.util.append_path(filepath) try: self.module = importlib.import_module(self.name) self.console = self.module.console(db, env, self.components) # force module prompt to app name self.console.prompt = self.name + "> " self.console.doc_header = self.name + \ " commands (type help <topic>):" except BaseException, e: errstr = "{0}: {1}".format(type(e).__name__, str(e)) logger.warning("error loading %s console: %s", self.name, e)
def start(self): "Start the environment, initialize the database and network provider" self.provider.start() # only load topo if connecting to a clean db if self.db.cleaned: self.db.load_topo(self.provider) ravel.util.update_trigger_path(ravel.db.FLOW_SQL, ravel.util.resource_file()) self.db.load_schema(ravel.db.FLOW_SQL) # delay loading topo triggers until after db is loaded # we only want to catch updates after initial load ravel.util.update_trigger_path(ravel.db.TOPO_SQL, ravel.util.resource_file()) self.db.load_schema(ravel.db.TOPO_SQL) self.db.load_schema(ravel.db.AUXILIARY_FUN_SQL) else: logger.debug("connecting to existing db, skipping load_topo()") # if running onlydb mode, remove network flow triggers if self.opts.onlydb: self.db.load_schema(ravel.db.NOFLOW_SQL) self.provider.cacheNodes() core_shortcuts = [] for app in self.coreapps: self.load_app(app) if self.loaded[app].shortcut is not None: core_shortcuts.append(self.loaded[app].shortcut) self.coreapps.extend(core_shortcuts)
def start(self): "Start the environment, initialize the database and network provider" self.provider.start() # only load topo if connecting to a clean db if self.db.cleaned: self.db.load_topo(self.provider) ravel.util.update_trigger_path(ravel.db.FLOW_SQL, ravel.util.resource_file()) self.db.load_schema(ravel.db.FLOW_SQL) # delay loading topo triggers until after db is loaded # we only want to catch updates after initial load ravel.util.update_trigger_path(ravel.db.TOPO_SQL, ravel.util.resource_file()) self.db.load_schema(ravel.db.TOPO_SQL) else: logger.debug("connecting to existing db, skipping load_topo()") # if running onlydb mode, remove network flow triggers if self.opts.onlydb: self.db.load_schema(ravel.db.NOFLOW_SQL) self.provider.cacheNodes() core_shortcuts = [] for app in self.coreapps: self.load_app(app) if self.loaded[app].shortcut is not None: core_shortcuts.append(self.loaded[app].shortcut) self.coreapps.extend(core_shortcuts)
def init(self, db, env): """Initialize the application without loading it into the database. db: a ravel.db.RavelDb instance to be passed to the application's sub-shell env: a ravel.env.Environment instance of the CLI's executing environment to be passed to the application's sub-shell""" if not self.pyfile: return # discover sql components (tables, views, functions) if self.sqlfile is not None: with open(self.sqlfile) as f: self.components = discoverComponents(f.read()) logger.debug("discovered {0} components: {1}".format( self.name, self.components)) # if needed, add path filepath = os.path.dirname(self.pyfile) ravel.util.append_path(filepath) try: self.module = importlib.import_module(self.name) self.console = self.module.console(db, env, self.components) # force module prompt to app name self.console.prompt = self.name + "> " self.console.doc_header = self.name + \ " commands (type help <topic>):" except BaseException, e: errstr = "{0}: {1}".format(type(e).__name__, str(e)) logger.warning("error loading %s console: %s", self.name, e)
def removeHost(self, msg): """Remove a host from the Mininet topology msg: a RemoveHostMessage object""" # find adjacent switch(es) sw = [ intf.link.intf2.node.name for intf in self.net.get(msg.name).intfList() if self.net.topo.isSwitch(intf.link.intf2.node.name) ] self.net.get(msg.name).terminate() self.net.topo.g.node.pop(msg.name, None) self.net.hosts = [h for h in self.net.hosts if h.name != msg.name] del self.net.nameToNode[msg.name] if len(sw) == 0: logger.debug("deleting host connected to 0 switches") return elif len(sw) > 1: raise Exception("cannot support hosts connected to %s switches", len(sw)) swname = str(sw[0]) swobj = self.net.get(swname) swport = self.net.topo.port(swname, msg.name)[0] intf = str(swobj.intfList()[swport]) swobj.detach(intf) del swobj.nameToIntf[intf] del swobj.intfs[swport] del self.cache_name[msg.name] del self.cache_id[msg.hid]
def start(self, cargs=None): """Start the Pox process cargs: arguments to pass to the controller""" pox = os.path.join(ravel.util.Config.PoxDir, "pox.py") if not os.path.exists(pox): logger.error("cannot find pox.py at: {0}. " "Is PoxDir set correctly in ravel.cfg?" .format(pox)) sys.exit(0) if cargs is None: cargs = ["log.level", "--DEBUG", "openflow.of_01", "--port={0}".format(ravel.util.Config.PoxPort), self.app, "openflow.discovery"] ravel.util.append_path(ravel.util.resource_file()) env = os.environ.copy() env["PYTHONPATH"] = ":".join(sys.path) logger.debug("pox with params: %s", " ".join(cargs)) self.proc = subprocess.Popen([pox] + cargs, env=env, preexec_fn = preexec_fn, stdout=open("/tmp/pox.log", "wb"), stderr=open("/tmp/pox.err", "wb"))
def send(self, msg): """Send the specified message msg: the message to send""" pc = ravel.profiling.PerfCounter("mq_send") pc.start() logger.debug("mq: sending message %s", msg) self.mq.send(pickle.dumps(msg)) pc.stop()
def send(self, msg): """Send the specified message msg: the message to send""" logger.debug("rpc: sending message %s", msg) pc = ravel.profiling.PerfCounter("rpc_send") pc.start() self.proxy.client_send(pickle.dumps(msg)) pc.stop()
def _run(self): while self.running: s, _ = self.mq.receive() msg = s.decode() obj = pickle.loads(msg) logger.debug("mq: received message %s", msg) if obj is not None: obj.consume(self.consumer)
def _run(self): while self.running: s,_ = self.mq.receive() msg = s.decode() obj = pickle.loads(msg) logger.debug("mq: received message %s", msg) if obj is not None: obj.consume(self.consumer)
def stop(self): "Stop the Mininet network" self.receiver.stop() self.net.stop() if self.controller is not None: self.controller.stop() logger.debug("cleaning up mininet") mininet.clean.cleanup()
def load_schema(self, script): """Load the specified schema into the database" script: path to a SQL script""" try: s = open(script, "r").read() logger.debug("loaded schema %s", script) self.cursor.execute(s) except psycopg2.DatabaseError, e: logger.warning("error loading schema: %s", self.fmt_errmsg(e))
def drop(self, db): """Drop the component from the specified database db: a ravel.db.RavelDb instance containing the component""" try: cmd = "DROP {0} IF EXISTS {1} CASCADE;".format(self.typ, self.name) db.cursor.execute(cmd) logger.debug("removing component: %s", cmd) except Exception, e: logger.error("error removing component {0}: {1}" .format(self.name, e))
def drop(self, db): """Drop the component from the specified database db: a ravel.db.RavelDb instance containing the component""" try: cmd = "DROP {0} IF EXISTS {1} CASCADE;".format(self.typ, self.name) db.cursor.execute(cmd) logger.debug("removing component: %s", cmd) except Exception, e: logger.error("error removing component {0}: {1}".format( self.name, e))
def do_load(self, line): """Start one or more applications with orchestration Usage: load [app1] [app2] ... (priority: low -> high)""" ordering = [app.lower() for app in line.split()] for app in ordering: if app.lower() not in self.env.apps: print "Unrecognized app", app return # load unloaded apps loads = [app for app in ordering if app not in self.env.loaded] for app in loads: logger.debug("loading unloaded app %s", app) self.env.load_app(app) # unload unlisted apps: if it's loaded but not a shortcut or core app unlisted = [ app for app in self.env.loaded if app not in ordering and app not in self.env.coreapps and app in self.env.apps ] # for app in unlisted: # logger.info("unloading unlisted app %s", app) # self.env.unload_app(app) # processing in ascending order of priority ordering.reverse() sql = "" for app in [x for x in ordering if x != "routing"]: vtable = "{0}_violation".format(app) sql += ptable_template.format(app) sql += runrule_template.format(app, vtable) if "routing" in ordering: sql += routing for app1, app2 in pairwise(ordering): sql += orderrule_template.format(app1, app2) sql += clock_template.format(ordering[-1]) self.ordering = [x for x in reversed(ordering)] self.sql = sql log = resource_file("orch_log.sql") f = open(log, 'w') f.write(self.sql) f.close() logger.debug("logged orchestration protocol to %s", log) try: self.db.cursor.execute(self.sql) except Exception, e: print e
def load(self, db): """Load the application from the specified database db: a ravel.db.RavelDb instance into which the application will be loaded""" if self.sqlfile is None: logger.debug("loaded application %s with no SQL file", self.name) return with open(self.sqlfile) as f: try: db.cursor.execute(f.read()) except psycopg2.ProgrammingError, e: print "Error loading app {0}: {1}".format(self.name, e)
def do_load(self, line): """Start one or more applications with orchestration Usage: load [app1] [app2] ... (priority: low -> high)""" ordering = [app.lower() for app in line.split()] for app in ordering: if app.lower() not in self.env.apps: print "Unrecognized app", app return # load unloaded apps loads = [app for app in ordering if app not in self.env.loaded] for app in loads: logger.debug("loading unloaded app %s", app) self.env.load_app(app) # unload unlisted apps: if it's loaded but not a shortcut or core app unlisted = [app for app in self.env.loaded if app not in ordering and app not in self.env.coreapps and app in self.env.apps] for app in unlisted: logger.info("unloading unlisted app %s", app) self.env.unload_app(app) # processing in ascending order of priority ordering.reverse() sql = "" for app in [x for x in ordering if x != "routing"]: vtable = "{0}_violation".format(app) sql += ptable_template.format(app) sql += runrule_template.format(app, vtable) if "routing" in ordering: sql += routing for app1, app2 in pairwise(ordering): sql += orderrule_template.format(app1, app2) sql += clock_template.format(ordering[-1]) self.ordering = [x for x in reversed(ordering)] self.sql = sql log = resource_file("orch_log.sql") f = open(log, 'w') f.write(self.sql) f.close() logger.debug("logged orchestration protocol to %s", log) try: self.db.cursor.execute(self.sql) except Exception, e: print e
def truncate(self): """Clean the database of any state Ravel components, except for topology tables. This rolls back the database to the state after the topology is first loaded""" try: tables = ["cf", "clock", "p_spv", "spatial_ref_sys", "spv_tb_del", "spv_tb_ins", "rm", "rm_delta", "urm"] self.cursor.execute("truncate %s;" % ", ".join(tables)) logger.debug("truncated tables") self.cursor.execute("INSERT INTO clock values (0);") except psycopg2.DatabaseError, e: logger.warning("error truncating databases: %s", self.fmt_errmsg(e))
def stop(self): "Stop the environment, including the database and network provider" self.provider.stop() if len(self.xterms) > 0: logger.debug("waiting for xterms") for t in self.xterms: t.wait() # delete xterm temp files for f in self.xterm_files: os.unlink(f) ravel.messaging.clear_queue(Config.QueueId)
def add_extensions(self): """If not already added, add extensions required by Ravel (plpythonu, postgis, pgrouting)""" try: self.cursor.execute("SELECT 1 FROM pg_catalog.pg_namespace n JOIN " + "pg_catalog.pg_proc p ON pronamespace = n.oid " + "WHERE proname = 'pgr_dijkstra';") fetch = self.cursor.fetchall() if fetch == []: self.cursor.execute("CREATE EXTENSION IF NOT EXISTS plpythonu;") self.cursor.execute("CREATE EXTENSION IF NOT EXISTS postgis;") self.cursor.execute("CREATE EXTENSION IF NOT EXISTS pgrouting;") self.cursor.execute("CREATE EXTENSION plsh;") logger.debug("created extensions") except psycopg2.DatabaseError, e: logger.warning("error loading extensions: %s", self.fmt_errmsg(e))
def do_load(self, line): """Start one or more applications with orchestration Usage: load [app1] [app2] ... (priority: low -> high)""" ordering = [app.lower() for app in line.split()] if len(ordering) == 0: return for app in ordering: if app.lower() not in self.env.apps: print "Unrecognized app", app return # load unloaded apps loads = [app for app in ordering if app not in self.env.loaded] for app in loads: logger.debug("loading unloaded app %s", app) self.env.load_app(app) # unload unlisted apps: if it's loaded but not a shortcut or core app unlisted = [app for app in self.env.loaded if app not in ordering and app not in self.env.coreapps and app in self.env.apps] # for app in unlisted: # logger.info("unloading unlisted app %s", app) # self.env.unload_app(app) # processing in ascending order of priority ordering.reverse() sql = "" for app in [x for x in ordering if x != "routing"]: sql += ptable_template.format(app) try: self.db.cursor.execute("SELECT violation FROM app_violation WHERE app = '{0}';".format(app)) violations = self.db.cursor.fetchall() if len(violations) > 0: vtable = violations[0][0] for v in violations[1:]: vtable += "; DELETE FROM {0}".format(v[0]) else: vtable = "{0}_violation".format(app) except Exception, e: print e sql += runrule_template.format(app, vtable)
def create(self): """If not created, create a database with the name specified in the constructor""" conn = None try: conn = psycopg2.connect(database="postgres", user=self.user, password=self.passwd) conn.set_isolation_level(ISOLEVEL) cursor = conn.cursor() cursor.execute("SELECT datname FROM pg_database WHERE " + "datistemplate = false;") fetch = cursor.fetchall() dblist = [fetch[i][0] for i in range(len(fetch))] if self.name not in dblist: cursor.execute("CREATE DATABASE %s;" % self.name) logger.debug("created databse %s", self.name) except psycopg2.DatabaseError, e: logger.warning("error creating database: %s", self.fmt_errmsg(e))
class Application(object): """A Ravel application. In Ravel, an application contains a SQL implementation and, optionally, a sub-shell implemented in Python to monitor and control the application. Application sub-shells are accesible from the main Ravel CLI by loading the application and typing its name or shortcut.""" def __init__(self, name): "name: the name of the application" self.name = name self.shortcut = None self.description = "" self.pyfile = None self.sqlfile = None self.module = None self.components = [] self.console = None def link(self, filename): """Link a resource or implementation file to the application filename: path to the file containing an application's resource""" if filename.endswith(".py"): self.pyfile = filename elif filename.endswith(".sql"): self.sqlfile = filename def is_loadable(self): """returns: true if the application's Python component (it's sub-shell) can be imported and instantiated""" return self.module is not None def load(self, db): """Load the application from the specified database db: a ravel.db.RavelDb instance into which the application will be loaded""" if self.sqlfile is None: logger.debug("loaded application %s with no SQL file", self.name) return with open(self.sqlfile) as f: try: db.cursor.execute(f.read()) except psycopg2.ProgrammingError, e: print "Error loading app {0}: {1}".format(self.name, e) logger.debug("loaded application %s", self.name)
def start(self): "Start a new thread to receive messages" logger.debug("mq_receiver starting") self.running = True self.t = threading.Thread(target=self._run) self.t.start()
def _client_send(self, msg): obj = pickle.loads(msg) logger.debug("rpc: received message %s", msg) if obj is not None: print "CONSUMING MESSAGE", obj obj.consume(self.consumer)
def start(self): "Start a new thread to receive messages" logger.debug("rpc_receiver starting") self.running = True self.t = threading.Thread(target=self._run) self.t.start()
sql += routing for app1, app2 in pairwise(ordering): sql += orderrule_template.format(app1, app2) sql += clock_template.format(ordering[-1]) self.ordering = [x for x in reversed(ordering)] self.sql = sql log = resource_file("orch_log.sql") f = open(log, 'w') f.write(self.sql) f.close() logger.debug("logged orchestration protocol to %s", log) try: self.db.cursor.execute(self.sql) except Exception, e: print e def do_unload(self, line): """Stop one or more applications Usage: unload [app1] [app2] ...""" self.do_reset("") apps = line.split() for app in apps: try: self.db.cursor.execute("DELETE FROM app_violation WHERE app = '{0}'".format(app))