def parse_entry(self, entry):
# initialize routes in case this is the first entry (ok to call more
# than once - does nothing)
argosroutes.initialize(entry.datetime)
rv = self.parser(entry)
for (host, val) in rv:
if self.host_filter is not None:
if host not in self.host_filter:
return
else:
# use default host filtering rule (which is "exclude 'citysense'
# host from all plots except route additions & route changes
if self.metric not in ["route-additions", "route-changes"]:
if host == "citysense":
return
if self.metric in ["net", "net-links"]:
# hostmap is a map from hostnames to NetworkData objects
# val is a tuple: (overlay-bytes, mergestream-bytes, rawstream-bytes)
if host not in self.hostmap:
self.hostmap[host] = NetworkData(entry.datetime)
self.hostmap[host].add_bytes("overlay", entry.datetime, val[0])
self.hostmap[host].add_bytes("mergestream", entry.datetime, val[1])
self.hostmap[host].add_bytes("rawstream", entry.datetime, val[2])
else:
# hostmap is a map from hostnames to ScatterVals objects
# val is a scalar (int or float)
# perform any metric-specific processing of the value
val = val*self.value_mult
if host not in self.hostmap:
metric_type = AVAIL_METRICS[self.metric]
if metric_type == METRIC_TYPE_PACKET_RATE:
# packet rates should be added together
aggregate = sum
elif metric_type == METRIC_TYPE_BYTE_RATE:
# byte rates should be added together
aggregate = sum
elif metric_type == METRIC_TYPE_MEASURE:
# measurements should be averaged
aggregate = lambda li: sum(li)/len(li)
elif metric_type == METRIC_TYPE_COUNT:
# counts should be added together
aggregate = sum
else:
raise ValueError(metric_type)
self.hostmap[host] = ScatterVals(60, aggregate)
self.hostmap[host].putvals(entry.datetime, val)
def main():
parser = OptionParser(usage="%prog [options] DIR")
(opts, args) = parser.parse_args()
if len(args) != 1:
parser.error("expected exactly 1 argument")
directory = args[0]
coordinator = "www.citysense.net" # hard-coded
print "fetching node names from database..."
argosroutes.initialize()
# first do all outdoor nodes together as one unit
testbeds = {}
for testbed in argosroutes.get_outdoor_testbeds():
testbeds[testbed] = argosroutes.get_nodes(testbed)
print "starting to write files..."
filename = os.path.join(args[0], "wifioverlay-node-outdoor.script")
fi = open(filename, "w")
print >>fi, create_wifioverlay_script(testbeds, coordinator, server=False)
fi.close()
print "created %s" % filename
filename = os.path.join(args[0], "wifioverlay-server-outdoor.script")
fi = open(filename, "w")
print >>fi, create_wifioverlay_script(testbeds, coordinator, server=True)
fi.close()
print "created %s" % filename
# next do all indoor (citymd) nodes
testbeds = {"citymd": argosroutes.get_nodes("citymd") }
filename = os.path.join(args[0], "wifioverlay-node-citymd.script")
fi = open(filename, "w")
print >>fi, create_wifioverlay_script(testbeds, coordinator, server=False)
fi.close()
print "created %s" % filename
filename = os.path.join(args[0], "wifioverlay-server-citymd.script")
fi = open(filename, "w")
print >>fi, create_wifioverlay_script(testbeds, coordinator, server=True)
fi.close()
print "created %s" % filename
def run(click_exe, filename, gen_pkt_src, opts, coordinator="www.citysense.net",
citymd=False, ol_elt_params={}):
logging = argos.get_logging_defaults()
if opts.debug:
elts = opts.debug.split(",")
for elt in elts:
if elt.lower() == "all":
for key in logging:
logging[key] = "DEBUG"
break
else:
if elt in logging:
logging[elt] = "DEBUG"
else:
raise ValueError("no such element: %s" % elt)
# look up nodes from testbed(s)
argosroutes.initialize(init_routes=False)
testbeds = {}
if citymd:
try:
testbeds["citymd"] = argosroutes.get_nodes("citymd")
except KeyError:
raise ValueError("testbed not found: citymd")
else:
for tb in argosroutes.get_outdoor_testbeds():
testbeds[tb] = argosroutes.get_nodes(tb)
argos.init()
macros = {}
if citymd:
macros["ARGOS_USE_CITYMD"] = None
if opts.db is not None:
macros["ARGOS_USE_DATABASE"] = None
config = argos.preprocess_config(filename, macros)
queries = argos.parse_config(config)
# port that ArgosCtrlServer listens on and argosniffer connects on
if opts.port:
config_port = opts.port
else:
config_port = argos.DEF_CONFIG_PORT
# port that NetworkProxyServer listens on (on the server) and each nodes'
# NetworkProxy connects to
toserver_port = config_port + 1
# port that every node (incl. server) listens on and connects to on other
# nodes
overlay_port = toserver_port + 1
# port that the server's ControlSocket element listens on
control_port = overlay_port + 1
args = (queries, gen_pkt_src, toserver_port, overlay_port, control_port,
testbeds)
kwargs = {"coordinator": coordinator, "loglvls": logging,
"ol_elt_params": ol_elt_params}
node_router = argos.gen_node_router(*args, **kwargs)
(fd, filename) = tempfile.mkstemp()
fi = os.fdopen(fd, "w")
print >>fi, node_router
fi.close()
definitions = {}
if opts.db is not None:
db_name = opts.db
db_user = opts.db_user
prompt = "user '%s' on database '%s' password: "******"ARGOS_DATABASE_DBNAME"] = db_name
definitions["ARGOS_DATABASE_USER"] = db_user
definitions["ARGOS_DATABASE_PASSWORD"] = db_pass
args = (queries, filename, toserver_port, overlay_port, control_port,
testbeds)
kwargs = {"coordinator": coordinator, "loglvls": logging,
"ol_elt_params": ol_elt_params,
"config_port": config_port, "definitions": definitions}
server_router = argos.gen_server_router(*args, **kwargs)
if opts.dump:
print "#"*60
print "#%s#" % "Node Router".center(58)
print "#"*60
print
print node_router
print
print "#"*60
print "#%s#" % "Server Router".center(58)
print "#"*60
print
print server_router
return
elif opts.dump_node:
print node_router
return
elif opts.dump_server:
print server_router
return
argos.verify_router(node_router, "node router")
print "~~ node router checks out ok"
argos.verify_router(server_router, "server router")
print "~~ server router checks out ok"
if opts.verify:
# done!
return
print "Running server..."
if (opts.outdir is not None) and not os.path.exists(opts.outdir):
os.mkdir(opts.outdir)
# daemonize if requested
if opts.daemon:
cwd = os.getcwd()
daemon.daemonize(pidfile=opts.pidfile)
os.chdir(cwd)
else:
if opts.pidfile:
# have to handle pidfile ourselves (instead of in daemonize())
fi = daemon.openpidfile(opts.pidfile, ex_exist=False)
print >>fi, "%d" % os.getpid()
fi.close()
# start click process
proc = Popen("%s 2>&1" % click_exe, stdin=PIPE, stdout=PIPE, shell=True)
proc.stdin.write(server_router)
proc.stdin.close()
click_sig_handler = lambda signum, frame: os.kill(proc.pid, signum)
# set a signal handler to pass SIGHUP, SIGINT and SIGTERM on to click
signal.signal(signal.SIGHUP, click_sig_handler)
signal.signal(signal.SIGINT, click_sig_handler)
signal.signal(signal.SIGTERM, click_sig_handler)
closers = []
log_handlers = []
string_handlers = []
# set up logging either to disk or to stdout
if opts.outdir is None:
if isapipe(sys.stdout.fileno()):
# if the output of this script is a pipe, then turn off stdout buffering
unbuffered_stdout = os.fdopen(sys.stdout.fileno(), "w", 0)
h = lambda e: unbuffered_stdout.write("%s\n" % str(e).strip())
log_handlers.append(("stdout", h))
string_handlers.append(("stdout", h))
chatter = lambda s: unbuffered_stdout.write("%s\n" % s.strip())
else:
h = lambda e: sys.stdout.write("%s\n" % str(e).strip())
log_handlers.append(("stdout", h))
string_handlers.append(("stdout", h))
chatter = lambda s: sys.stdout.write("%s\n" % s.strip())
else:
import rotatinglog
log = rotatinglog.HourlyLog(opts.outdir)
chatter = lambda s: log.write("%s\n" % s.strip())
h = lambda entry: disk_log_filtered(entry, log)
log_handlers.append(("disk", h))
string_handlers.append(("disk", chatter))
closers.append(log.close)
# set up database logging, if enabled
if opts.db is not None:
import argosdb
db = argosdb.ArgosDB(db_user, db_pass, db_name)
db_parser = argosdb.Parser(db)
h = lambda entry: db_log_filtered(entry, db_parser)
log_handlers.append(("db", h))
close_db = lambda : db.close(commit=True)
closers.append(close_db)
# set up http server to display system status
import argosstatusserver
server = argosstatusserver.SystemStatusServer(opts.http_port)
server.start()
log_handlers.append(("web", server.handle_log_entry))
closers.append(server.stop)
# all output-handlers have been set up; now, in a loop, read and process
# individual lines from click's output
while 1:
try:
line = proc.stdout.readline()
except IOError, e:
if e.errno == errno.EINTR:
continue
else:
raise
if line == "":
# EOF received - check if process quit
if proc.poll() is not None:
# yes, process is dead so we should just quit
if proc.returncode > 128:
chatter("[click exitted from signal %d]" % (proc.returncode-128))
else:
chatter("[click exitted with code %d]" % proc.returncode)
break
# else, weird - I don't think we should have gotten an EOF...
# oh well, just ignore it
else:
# parse the line and pass it to each log-handler in case they want
# it; if the line fails to parse, then pass it to the string-handlers
# instead
entry = argoslog.parse_line(line)
if entry is None:
for name, handler in string_handlers:
try:
handler(line)
except StandardError, e:
chatter("string-handler '%s' failed: %s: %s" % (name, e.__class__.__name__, str(e)))
chatter(traceback.format_exc())
else: