def removeHost(self, msg):
"""Remove a host from the Mininet topology
msg: a RemoveHostMessage object"""
# find adjacent switch(es)
sw = [intf.link.intf2.node.name for
intf in self.net.get(msg.name).intfList() if
self.net.topo.isSwitch(intf.link.intf2.node.name)]
self.net.get(msg.name).terminate()
self.net.topo.g.node.pop(msg.name, None)
self.net.hosts = [h for h in self.net.hosts if h.name != msg.name]
del self.net.nameToNode[msg.name]
if len(sw) == 0:
logger.debug("deleting host connected to 0 switches")
return
elif len(sw) > 1:
raise Exception("cannot support hosts connected to %s switches",
len(sw))
swname = str(sw[0])
swobj = self.net.get(swname)
swport = self.net.topo.port(swname, msg.name)[0]
intf = str(swobj.intfList()[swport])
swobj.detach(intf)
del swobj.nameToIntf[intf]
del swobj.intfs[swport]
del self.cache_name[msg.name]
del self.cache_id[msg.hid]
def start(self, cargs=None):
"""Start the Pox process
cargs: arguments to pass to the controller"""
pox = os.path.join(ravel.util.Config.PoxDir, "pox.py")
if not os.path.exists(pox):
logger.error("cannot find pox.py at: {0}. "
"Is PoxDir set correctly in ravel.cfg?".format(pox))
sys.exit(0)
if cargs is None:
cargs = [
"log.level", "--DEBUG", "openflow.of_01",
"--port={0}".format(ravel.util.Config.PoxPort), self.app,
"openflow.discovery"
]
ravel.util.append_path(ravel.util.resource_file())
env = os.environ.copy()
env["PYTHONPATH"] = ":".join(sys.path)
logger.debug("pox with params: %s", " ".join(cargs))
self.proc = subprocess.Popen([pox] + cargs,
env=env,
preexec_fn=preexec_fn,
stdout=open("/tmp/pox.log", "wb"),
stderr=open("/tmp/pox.err", "wb"))
def unload(self, db):
"""Unload the application from the specified database
db: a ravel.db.RavelDb instance containing the application"""
for component in self.components:
component.drop(db)
logger.debug("unloaded application %s", self.name)
def init(self, db, env):
"""Initialize the application without loading it into the database.
db: a ravel.db.RavelDb instance to be passed to the application's
sub-shell
env: a ravel.env.Environment instance of the CLI's executing
environment to be passed to the application's sub-shell"""
if not self.pyfile:
return
# discover sql components (tables, views, functions)
if self.sqlfile is not None:
with open(self.sqlfile) as f:
self.components = discoverComponents(f.read())
logger.debug("discovered {0} components: {1}"
.format(self.name, self.components))
# if needed, add path
filepath = os.path.dirname(self.pyfile)
ravel.util.append_path(filepath)
try:
self.module = importlib.import_module(self.name)
self.console = self.module.console(db, env, self.components)
# force module prompt to app name
self.console.prompt = self.name + "> "
self.console.doc_header = self.name + \
" commands (type help <topic>):"
except BaseException, e:
errstr = "{0}: {1}".format(type(e).__name__, str(e))
logger.warning("error loading %s console: %s",
self.name, e)
def start(self):
"Start the environment, initialize the database and network provider"
self.provider.start()
# only load topo if connecting to a clean db
if self.db.cleaned:
self.db.load_topo(self.provider)
ravel.util.update_trigger_path(ravel.db.FLOW_SQL,
ravel.util.resource_file())
self.db.load_schema(ravel.db.FLOW_SQL)
# delay loading topo triggers until after db is loaded
# we only want to catch updates after initial load
ravel.util.update_trigger_path(ravel.db.TOPO_SQL,
ravel.util.resource_file())
self.db.load_schema(ravel.db.TOPO_SQL)
self.db.load_schema(ravel.db.AUXILIARY_FUN_SQL)
else:
logger.debug("connecting to existing db, skipping load_topo()")
# if running onlydb mode, remove network flow triggers
if self.opts.onlydb:
self.db.load_schema(ravel.db.NOFLOW_SQL)
self.provider.cacheNodes()
core_shortcuts = []
for app in self.coreapps:
self.load_app(app)
if self.loaded[app].shortcut is not None:
core_shortcuts.append(self.loaded[app].shortcut)
self.coreapps.extend(core_shortcuts)
def start(self):
"Start the environment, initialize the database and network provider"
self.provider.start()
# only load topo if connecting to a clean db
if self.db.cleaned:
self.db.load_topo(self.provider)
ravel.util.update_trigger_path(ravel.db.FLOW_SQL,
ravel.util.resource_file())
self.db.load_schema(ravel.db.FLOW_SQL)
# delay loading topo triggers until after db is loaded
# we only want to catch updates after initial load
ravel.util.update_trigger_path(ravel.db.TOPO_SQL,
ravel.util.resource_file())
self.db.load_schema(ravel.db.TOPO_SQL)
else:
logger.debug("connecting to existing db, skipping load_topo()")
# if running onlydb mode, remove network flow triggers
if self.opts.onlydb:
self.db.load_schema(ravel.db.NOFLOW_SQL)
self.provider.cacheNodes()
core_shortcuts = []
for app in self.coreapps:
self.load_app(app)
if self.loaded[app].shortcut is not None:
core_shortcuts.append(self.loaded[app].shortcut)
self.coreapps.extend(core_shortcuts)
def init(self, db, env):
"""Initialize the application without loading it into the database.
db: a ravel.db.RavelDb instance to be passed to the application's
sub-shell
env: a ravel.env.Environment instance of the CLI's executing
environment to be passed to the application's sub-shell"""
if not self.pyfile:
return
# discover sql components (tables, views, functions)
if self.sqlfile is not None:
with open(self.sqlfile) as f:
self.components = discoverComponents(f.read())
logger.debug("discovered {0} components: {1}".format(
self.name, self.components))
# if needed, add path
filepath = os.path.dirname(self.pyfile)
ravel.util.append_path(filepath)
try:
self.module = importlib.import_module(self.name)
self.console = self.module.console(db, env, self.components)
# force module prompt to app name
self.console.prompt = self.name + "> "
self.console.doc_header = self.name + \
" commands (type help <topic>):"
except BaseException, e:
errstr = "{0}: {1}".format(type(e).__name__, str(e))
logger.warning("error loading %s console: %s", self.name, e)
def unload(self, db):
"""Unload the application from the specified database
db: a ravel.db.RavelDb instance containing the application"""
for component in self.components:
component.drop(db)
logger.debug("unloaded application %s", self.name)
def removeHost(self, msg):
"""Remove a host from the Mininet topology
msg: a RemoveHostMessage object"""
# find adjacent switch(es)
sw = [
intf.link.intf2.node.name
for intf in self.net.get(msg.name).intfList()
if self.net.topo.isSwitch(intf.link.intf2.node.name)
]
self.net.get(msg.name).terminate()
self.net.topo.g.node.pop(msg.name, None)
self.net.hosts = [h for h in self.net.hosts if h.name != msg.name]
del self.net.nameToNode[msg.name]
if len(sw) == 0:
logger.debug("deleting host connected to 0 switches")
return
elif len(sw) > 1:
raise Exception("cannot support hosts connected to %s switches",
len(sw))
swname = str(sw[0])
swobj = self.net.get(swname)
swport = self.net.topo.port(swname, msg.name)[0]
intf = str(swobj.intfList()[swport])
swobj.detach(intf)
del swobj.nameToIntf[intf]
del swobj.intfs[swport]
del self.cache_name[msg.name]
del self.cache_id[msg.hid]
def start(self, cargs=None):
"""Start the Pox process
cargs: arguments to pass to the controller"""
pox = os.path.join(ravel.util.Config.PoxDir, "pox.py")
if not os.path.exists(pox):
logger.error("cannot find pox.py at: {0}. "
"Is PoxDir set correctly in ravel.cfg?"
.format(pox))
sys.exit(0)
if cargs is None:
cargs = ["log.level",
"--DEBUG",
"openflow.of_01",
"--port={0}".format(ravel.util.Config.PoxPort),
self.app,
"openflow.discovery"]
ravel.util.append_path(ravel.util.resource_file())
env = os.environ.copy()
env["PYTHONPATH"] = ":".join(sys.path)
logger.debug("pox with params: %s", " ".join(cargs))
self.proc = subprocess.Popen([pox] + cargs,
env=env,
preexec_fn = preexec_fn,
stdout=open("/tmp/pox.log", "wb"),
stderr=open("/tmp/pox.err", "wb"))
def send(self, msg):
"""Send the specified message
msg: the message to send"""
pc = ravel.profiling.PerfCounter("mq_send")
pc.start()
logger.debug("mq: sending message %s", msg)
self.mq.send(pickle.dumps(msg))
pc.stop()
def send(self, msg):
"""Send the specified message
msg: the message to send"""
logger.debug("rpc: sending message %s", msg)
pc = ravel.profiling.PerfCounter("rpc_send")
pc.start()
self.proxy.client_send(pickle.dumps(msg))
pc.stop()
def send(self, msg):
"""Send the specified message
msg: the message to send"""
pc = ravel.profiling.PerfCounter("mq_send")
pc.start()
logger.debug("mq: sending message %s", msg)
self.mq.send(pickle.dumps(msg))
pc.stop()
def _run(self):
while self.running:
s, _ = self.mq.receive()
msg = s.decode()
obj = pickle.loads(msg)
logger.debug("mq: received message %s", msg)
if obj is not None:
obj.consume(self.consumer)
def _run(self):
while self.running:
s,_ = self.mq.receive()
msg = s.decode()
obj = pickle.loads(msg)
logger.debug("mq: received message %s", msg)
if obj is not None:
obj.consume(self.consumer)
def send(self, msg):
"""Send the specified message
msg: the message to send"""
logger.debug("rpc: sending message %s", msg)
pc = ravel.profiling.PerfCounter("rpc_send")
pc.start()
self.proxy.client_send(pickle.dumps(msg))
pc.stop()
def stop(self):
"Stop the Mininet network"
self.receiver.stop()
self.net.stop()
if self.controller is not None:
self.controller.stop()
logger.debug("cleaning up mininet")
mininet.clean.cleanup()
def load_schema(self, script):
"""Load the specified schema into the database"
script: path to a SQL script"""
try:
s = open(script, "r").read()
logger.debug("loaded schema %s", script)
self.cursor.execute(s)
except psycopg2.DatabaseError, e:
logger.warning("error loading schema: %s", self.fmt_errmsg(e))
def stop(self):
"Stop the Mininet network"
self.receiver.stop()
self.net.stop()
if self.controller is not None:
self.controller.stop()
logger.debug("cleaning up mininet")
mininet.clean.cleanup()
def drop(self, db):
"""Drop the component from the specified database
db: a ravel.db.RavelDb instance containing the component"""
try:
cmd = "DROP {0} IF EXISTS {1} CASCADE;".format(self.typ, self.name)
db.cursor.execute(cmd)
logger.debug("removing component: %s", cmd)
except Exception, e:
logger.error("error removing component {0}: {1}"
.format(self.name, e))
def drop(self, db):
"""Drop the component from the specified database
db: a ravel.db.RavelDb instance containing the component"""
try:
cmd = "DROP {0} IF EXISTS {1} CASCADE;".format(self.typ, self.name)
db.cursor.execute(cmd)
logger.debug("removing component: %s", cmd)
except Exception, e:
logger.error("error removing component {0}: {1}".format(
self.name, e))
def do_load(self, line):
"""Start one or more applications with orchestration
Usage: load [app1] [app2] ... (priority: low -> high)"""
ordering = [app.lower() for app in line.split()]
for app in ordering:
if app.lower() not in self.env.apps:
print "Unrecognized app", app
return
# load unloaded apps
loads = [app for app in ordering if app not in self.env.loaded]
for app in loads:
logger.debug("loading unloaded app %s", app)
self.env.load_app(app)
# unload unlisted apps: if it's loaded but not a shortcut or core app
unlisted = [
app for app in self.env.loaded if app not in ordering
and app not in self.env.coreapps and app in self.env.apps
]
# for app in unlisted:
# logger.info("unloading unlisted app %s", app)
# self.env.unload_app(app)
# processing in ascending order of priority
ordering.reverse()
sql = ""
for app in [x for x in ordering if x != "routing"]:
vtable = "{0}_violation".format(app)
sql += ptable_template.format(app)
sql += runrule_template.format(app, vtable)
if "routing" in ordering:
sql += routing
for app1, app2 in pairwise(ordering):
sql += orderrule_template.format(app1, app2)
sql += clock_template.format(ordering[-1])
self.ordering = [x for x in reversed(ordering)]
self.sql = sql
log = resource_file("orch_log.sql")
f = open(log, 'w')
f.write(self.sql)
f.close()
logger.debug("logged orchestration protocol to %s", log)
try:
self.db.cursor.execute(self.sql)
except Exception, e:
print e
def load(self, db):
"""Load the application from the specified database
db: a ravel.db.RavelDb instance into which the application will be loaded"""
if self.sqlfile is None:
logger.debug("loaded application %s with no SQL file", self.name)
return
with open(self.sqlfile) as f:
try:
db.cursor.execute(f.read())
except psycopg2.ProgrammingError, e:
print "Error loading app {0}: {1}".format(self.name, e)
def load(self, db):
"""Load the application from the specified database
db: a ravel.db.RavelDb instance into which the application will be loaded"""
if self.sqlfile is None:
logger.debug("loaded application %s with no SQL file", self.name)
return
with open(self.sqlfile) as f:
try:
db.cursor.execute(f.read())
except psycopg2.ProgrammingError, e:
print "Error loading app {0}: {1}".format(self.name, e)
def do_load(self, line):
"""Start one or more applications with orchestration
Usage: load [app1] [app2] ... (priority: low -> high)"""
ordering = [app.lower() for app in line.split()]
for app in ordering:
if app.lower() not in self.env.apps:
print "Unrecognized app", app
return
# load unloaded apps
loads = [app for app in ordering if app not in self.env.loaded]
for app in loads:
logger.debug("loading unloaded app %s", app)
self.env.load_app(app)
# unload unlisted apps: if it's loaded but not a shortcut or core app
unlisted = [app for app in self.env.loaded if app not in ordering
and app not in self.env.coreapps and app in self.env.apps]
for app in unlisted:
logger.info("unloading unlisted app %s", app)
self.env.unload_app(app)
# processing in ascending order of priority
ordering.reverse()
sql = ""
for app in [x for x in ordering if x != "routing"]:
vtable = "{0}_violation".format(app)
sql += ptable_template.format(app)
sql += runrule_template.format(app, vtable)
if "routing" in ordering:
sql += routing
for app1, app2 in pairwise(ordering):
sql += orderrule_template.format(app1, app2)
sql += clock_template.format(ordering[-1])
self.ordering = [x for x in reversed(ordering)]
self.sql = sql
log = resource_file("orch_log.sql")
f = open(log, 'w')
f.write(self.sql)
f.close()
logger.debug("logged orchestration protocol to %s", log)
try:
self.db.cursor.execute(self.sql)
except Exception, e:
print e
def truncate(self):
"""Clean the database of any state Ravel components, except for
topology tables. This rolls back the database to the state after
the topology is first loaded"""
try:
tables = ["cf", "clock", "p_spv", "spatial_ref_sys", "spv_tb_del",
"spv_tb_ins", "rm", "rm_delta", "urm"]
self.cursor.execute("truncate %s;" % ", ".join(tables))
logger.debug("truncated tables")
self.cursor.execute("INSERT INTO clock values (0);")
except psycopg2.DatabaseError, e:
logger.warning("error truncating databases: %s", self.fmt_errmsg(e))
def stop(self):
"Stop the environment, including the database and network provider"
self.provider.stop()
if len(self.xterms) > 0:
logger.debug("waiting for xterms")
for t in self.xterms:
t.wait()
# delete xterm temp files
for f in self.xterm_files:
os.unlink(f)
ravel.messaging.clear_queue(Config.QueueId)
def stop(self):
"Stop the environment, including the database and network provider"
self.provider.stop()
if len(self.xterms) > 0:
logger.debug("waiting for xterms")
for t in self.xterms:
t.wait()
# delete xterm temp files
for f in self.xterm_files:
os.unlink(f)
ravel.messaging.clear_queue(Config.QueueId)
def add_extensions(self):
"""If not already added, add extensions required by Ravel (plpythonu,
postgis, pgrouting)"""
try:
self.cursor.execute("SELECT 1 FROM pg_catalog.pg_namespace n JOIN " +
"pg_catalog.pg_proc p ON pronamespace = n.oid " +
"WHERE proname = 'pgr_dijkstra';")
fetch = self.cursor.fetchall()
if fetch == []:
self.cursor.execute("CREATE EXTENSION IF NOT EXISTS plpythonu;")
self.cursor.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
self.cursor.execute("CREATE EXTENSION IF NOT EXISTS pgrouting;")
self.cursor.execute("CREATE EXTENSION plsh;")
logger.debug("created extensions")
except psycopg2.DatabaseError, e:
logger.warning("error loading extensions: %s", self.fmt_errmsg(e))
def do_load(self, line):
"""Start one or more applications with orchestration
Usage: load [app1] [app2] ... (priority: low -> high)"""
ordering = [app.lower() for app in line.split()]
if len(ordering) == 0:
return
for app in ordering:
if app.lower() not in self.env.apps:
print "Unrecognized app", app
return
# load unloaded apps
loads = [app for app in ordering if app not in self.env.loaded]
for app in loads:
logger.debug("loading unloaded app %s", app)
self.env.load_app(app)
# unload unlisted apps: if it's loaded but not a shortcut or core app
unlisted = [app for app in self.env.loaded if app not in ordering
and app not in self.env.coreapps and app in self.env.apps]
# for app in unlisted:
# logger.info("unloading unlisted app %s", app)
# self.env.unload_app(app)
# processing in ascending order of priority
ordering.reverse()
sql = ""
for app in [x for x in ordering if x != "routing"]:
sql += ptable_template.format(app)
try:
self.db.cursor.execute("SELECT violation FROM app_violation WHERE app = '{0}';".format(app))
violations = self.db.cursor.fetchall()
if len(violations) > 0:
vtable = violations[0][0]
for v in violations[1:]:
vtable += "; DELETE FROM {0}".format(v[0])
else:
vtable = "{0}_violation".format(app)
except Exception, e:
print e
sql += runrule_template.format(app, vtable)
def create(self):
"""If not created, create a database with the name specified in
the constructor"""
conn = None
try:
conn = psycopg2.connect(database="postgres",
user=self.user,
password=self.passwd)
conn.set_isolation_level(ISOLEVEL)
cursor = conn.cursor()
cursor.execute("SELECT datname FROM pg_database WHERE " +
"datistemplate = false;")
fetch = cursor.fetchall()
dblist = [fetch[i][0] for i in range(len(fetch))]
if self.name not in dblist:
cursor.execute("CREATE DATABASE %s;" % self.name)
logger.debug("created databse %s", self.name)
except psycopg2.DatabaseError, e:
logger.warning("error creating database: %s", self.fmt_errmsg(e))
class Application(object):
"""A Ravel application. In Ravel, an application contains a SQL
implementation and, optionally, a sub-shell implemented in Python to
monitor and control the application. Application sub-shells are accesible
from the main Ravel CLI by loading the application and typing its name
or shortcut."""
def __init__(self, name):
"name: the name of the application"
self.name = name
self.shortcut = None
self.description = ""
self.pyfile = None
self.sqlfile = None
self.module = None
self.components = []
self.console = None
def link(self, filename):
"""Link a resource or implementation file to the application
filename: path to the file containing an application's resource"""
if filename.endswith(".py"):
self.pyfile = filename
elif filename.endswith(".sql"):
self.sqlfile = filename
def is_loadable(self):
"""returns: true if the application's Python component (it's sub-shell)
can be imported and instantiated"""
return self.module is not None
def load(self, db):
"""Load the application from the specified database
db: a ravel.db.RavelDb instance into which the application will be loaded"""
if self.sqlfile is None:
logger.debug("loaded application %s with no SQL file", self.name)
return
with open(self.sqlfile) as f:
try:
db.cursor.execute(f.read())
except psycopg2.ProgrammingError, e:
print "Error loading app {0}: {1}".format(self.name, e)
logger.debug("loaded application %s", self.name)
def start(self):
"Start a new thread to receive messages"
logger.debug("mq_receiver starting")
self.running = True
self.t = threading.Thread(target=self._run)
self.t.start()
def _client_send(self, msg):
obj = pickle.loads(msg)
logger.debug("rpc: received message %s", msg)
if obj is not None:
print "CONSUMING MESSAGE", obj
obj.consume(self.consumer)
def _client_send(self, msg):
obj = pickle.loads(msg)
logger.debug("rpc: received message %s", msg)
if obj is not None:
print "CONSUMING MESSAGE", obj
obj.consume(self.consumer)
def start(self):
"Start a new thread to receive messages"
logger.debug("rpc_receiver starting")
self.running = True
self.t = threading.Thread(target=self._run)
self.t.start()
sql += routing
for app1, app2 in pairwise(ordering):
sql += orderrule_template.format(app1, app2)
sql += clock_template.format(ordering[-1])
self.ordering = [x for x in reversed(ordering)]
self.sql = sql
log = resource_file("orch_log.sql")
f = open(log, 'w')
f.write(self.sql)
f.close()
logger.debug("logged orchestration protocol to %s", log)
try:
self.db.cursor.execute(self.sql)
except Exception, e:
print e
def do_unload(self, line):
"""Stop one or more applications
Usage: unload [app1] [app2] ..."""
self.do_reset("")
apps = line.split()
for app in apps:
try:
self.db.cursor.execute("DELETE FROM app_violation WHERE app = '{0}'".format(app))
