def _on_shape(self, shape):
if not shape: return None
field = shape.get('max', None)
if field is None:
field = shape.get('min', None)
if field is None:
logger.dump("no shape function defined")
return None
reverse = False
else:
reverse = True
if field == 'bytes':
nm = self.bytesid
elif field == 'packets':
nm = self.packetsid
elif field is None:
pass
else:
logger.dump("unexpected sort field '%s'" % field)
pos = 0
for c in self.counters:
if c == nm:
return reverse, pos, shape.get('count', 0)
pos += 1
return None
def on_flow(self, msg):
#print msg
dd = zmq.utils.jsonapi.loads(msg[1])
flowkey = ''
self.count += 1
for ft in self.flowtags:
flowkey += ':'+str(dd[ft])
frec = self.flowrepo.get(flowkey, None)
if frec is None:
frec = [0]
self.flowrepo[flowkey] = frec
now = datetime.datetime.now()
if self.prev is None or (self.prev + self.every) <= now:
for ht in self.hosttags:
host = dd[ht]
hrec = self.hostrepo.get(host, None)
if hrec is None:
hrec = [0]
self.hostrepo[host] = hrec
self.pub.send_multipart(['hosts', zmq.utils.jsonapi.dumps({'host':host,
'count':len(self.hostrepo)})])
hrec[0] += 1
logger.dump("%4d[%d/%d] %s %s"%(len(self.flowrepo)*1000/self.count,
len(self.flowrepo), self.count, flowkey, now))
self.prev = now
frec[0] += 1
def _on_shape(self, shape):
if not shape: return None
field = shape.get('max', None)
if field is None:
field = shape.get('min', None)
if field is None:
logger.dump("no shape function defined")
return None
reverse = False
else:
reverse = True
if field == 'bytes':
nm = self.bytesid
elif field == 'packets':
nm = self.packetsid
elif field is None:
pass
else:
logger.dump("unexpected sort field '%s'"%field)
pos = 0
for c in self.counters:
if c == nm:
return reverse, pos, shape.get('count', 0)
pos += 1
return None
def _cleanuptimefields(fields):
if Source.startstamp in fields:
logger.dump("ignoring `start time`[%s] field in query"%(Source.startstamp))
del fields[Source.startstamp]
if Source.endstamp in fields:
logger.dump("ignoring `end time`[%s] field in query"%(Source.endstamp))
del fields[Source.endstamp]
def _cleanuptimefields(fields):
if Source.startstamp in fields:
logger.dump("ignoring `start time`[%s] field in query" %
(Source.startstamp))
del fields[Source.startstamp]
if Source.endstamp in fields:
logger.dump("ignoring `end time`[%s] field in query" %
(Source.endstamp))
del fields[Source.endstamp]
def showschedule(self, nm):
msg = '%s schedule:\n'%(nm)
for k, v in self._schedule.items():
try:
iter(v)
msg += " %s: %s --> %s\n"%(k, tostamp(datetime.datetime.utcfromtimestamp(v[0])),
tostamp(datetime.datetime.utcfromtimestamp(v[1])))
except:
msg += " %s: %s\n"%(k, str(v))
logger.dump(msg)
def showschedule(self, nm):
msg = '%s schedule:\n' % (nm)
for k, v in self._schedule.items():
try:
iter(v)
msg += " %s: %s --> %s\n" % (
k, tostamp(datetime.datetime.utcfromtimestamp(v[0])),
tostamp(datetime.datetime.utcfromtimestamp(v[1])))
except:
msg += " %s: %s\n" % (k, str(v))
logger.dump(msg)
def _on_shape(self, shape):
if not shape: return (None, None, 0)
field = shape.get('max', None)
if field is None:
field = shape.get('min', None)
if field is None:
logger.dump("no shape function defined")
return (None, None, 0)
direction = 'min'
else:
direction = 'max'
if self.bytestp.name != field and self.packetstp.name != field:
logger.dump("unexpected sort field '%s'"%field)
return (None, None, 0)
count = shape.get('count', 0)
try:
count = int(count)
except:
logger.dump("unexpected count field '%s'"%(count))
return (None, None, 0)
if count < 0:
logger.dump("negative count field '%d'"%(count))
return (None, None, 0)
return field, direction, count
def _on_shape(self, shape):
if not shape: return (None, None, 0)
field = shape.get('max', None)
if field is None:
field = shape.get('min', None)
if field is None:
logger.dump("no shape function defined")
return (None, None, 0)
direction = 'min'
else:
direction = 'max'
if self.bytestp.name != field and self.packetstp.name != field:
logger.dump("unexpected sort field '%s'" % field)
return (None, None, 0)
count = shape.get('count', 0)
try:
count = int(count)
except:
logger.dump("unexpected count field '%s'" % (count))
return (None, None, 0)
if count < 0:
logger.dump("negative count field '%d'" % (count))
return (None, None, 0)
return field, direction, count
def _on_backup(self, loc):
if loc is None:
return {"error":"backup folder was not provided at startup"}
start = datetime.datetime.now()
fname = os.path.join(loc, 'collector.backup')
try:
with tables.open_file(fname, mode='w', title='Collector Backup') as fileh:
for nm, obj in self._objmap.items():
grp = fileh.create_group(fileh.root, nm)
obj.backup(fileh, grp)
fileh.flush()
except Exception, e:
emsg = traceback.format_exc()
logger.dump('Backup failed: %s\n%s'%(str(e), emsg))
return {"error":"can not do backup to '%s': %s"%(loc, str(e))}
def _on_backup(self, loc):
if loc is None:
return {"error": "backup folder was not provided at startup"}
start = datetime.datetime.now()
fname = os.path.join(loc, 'collector.backup')
try:
with tables.open_file(fname, mode='w',
title='Collector Backup') as fileh:
for nm, obj in self._objmap.items():
grp = fileh.create_group(fileh.root, nm)
obj.backup(fileh, grp)
fileh.flush()
except Exception, e:
emsg = traceback.format_exc()
logger.dump('Backup failed: %s\n%s' % (str(e), emsg))
return {"error": "can not do backup to '%s': %s" % (loc, str(e))}
def on_msg(self, msg):
req = msg[1]
addr = msg[0]
if req == self.hbmessage:
arec = self._addresses.get(addr, None)
if not arec:
logger.dump("unexpected heartbeat from %s" % ([addr]))
return
arec.stamp()
return
try:
qry = zmq.utils.jsonapi.loads(req)
hb = int(qry.get('heartbeat', self.heartbeat))
except Exception, e:
logger.dump("invalid query: %s" % (req))
err = {'error': str(e), 'badmsg': req}
self._send(addr, err)
return
def setup(insock, servsock, qrysock, backup):
try:
conn = connector.Connector()
recvr = native.receiver.Receiver(insock, conn.loop)
fproc = FlowProc(recvr, backup)
if backup:
fproc._on_restore(backup)
conn.timer(1, fproc.on_time)
conn.listen(qrysock, fproc)
except KeyboardInterrupt:
logger.dump("closing")
finally:
conn.close()
def setup(insock, servsock, qrysock, backup):
try:
conn = connector.Connector()
recvr = native.receiver.Receiver(insock, conn.loop)
fproc = FlowProc(recvr, backup)
if backup:
fproc._on_restore(backup)
conn.timer(1, fproc.on_time)
conn.listen(qrysock, fproc)
except KeyboardInterrupt:
logger.dump("closing")
finally:
conn.close()
def on_msg(self, msg):
req = msg[1]
addr = msg[0]
if req == self.hbmessage:
arec = self._addresses.get(addr, None)
if not arec:
logger.dump("unexpected heartbeat from %s"%([addr]))
return
arec.stamp()
return
try:
qry = zmq.utils.jsonapi.loads(req)
hb = int(qry.get('heartbeat', self.heartbeat))
except Exception, e:
logger.dump("invalid query: %s"%(req))
err = {'error':str(e), 'badmsg':req}
self._send(addr, err)
return
def _on_restore(self, loc):
fname = os.path.join(loc, 'collector.backup')
if not os.path.isfile(fname):
logger.dump("Nothing to restore from. %s does not exits" % (fname))
return
now = self._history.now()
nowstamp = self._history.seconds().now
try:
with tables.open_file(fname) as fileh:
for nm, obj in self._objmap.items():
grp = fileh.get_node(fileh.root, nm)
obj.restore(fileh, grp)
#TODO adjust time according to current value
renow = self._history.now()
if now < renow:
raise Exception(
"Can not restore from future: %s < %s" %
(querymod.tostamp(now), querymod.tostamp(renow)))
logger.dump("Restoring with time gap of %s" %
(elapsed(now - renow)))
renowstamp = self._history.seconds().now
if renowstamp < nowstamp:
units = [
self._history.seconds(),
self._history.minutes(),
self._history.hours(),
self._history.days()
]
step = 0
end = 0
while renowstamp < nowstamp:
if units and renowstamp > end: # time to switch to bigger step
renowstamp -= step # step back to previous time
unit = units.pop(0)
step = unit.one
end = renowstamp + unit.count * step
renowstamp += step
self._tick_sources(renowstamp)
renowstamp += step
self._tick_sources(nowstamp)
logger.dump("Restored in %s" %
(elapsed(datetime.datetime.utcnow() - now)))
except Exception, e:
emsg = traceback.format_exc()
logger.dump("Restore from %s failed: %s\n%s" %
(fname, str(e), emsg))
shutil.move(fname, fname + '.failed')
sys.exit(-1)
def setup(insock, outsock, qrysock):
try:
conn = connector.Connector()
pub = conn.publish(outsock)
fproc = FlowRaw(pub)
fproc.header()
conn.subscribe(insock, 'flow', fproc.on_flow)
conn.timer(1, fproc.on_time)
conn.listen(qrysock, fproc)
except KeyboardInterrupt:
logger.dump("closing")
finally:
conn.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--input',
type=str,
help='publisher interface to subscribe to',
required=True)
parser.add_argument('-s',
'--server',
type=str,
help='server interface to serve stats requests',
required=True)
parser.add_argument('-q',
'--query',
type=str,
help='listening interface for queries',
required=True)
parser.add_argument('-d',
'--dumpfile',
type=str,
help='output file (defaults: stdout)',
default=None)
parser.add_argument('-b',
'--backup',
type=str,
help='backup folder',
default=None)
args = parser.parse_args()
if args.dumpfile is None:
out = sys.stdout
else:
out = open(args.dumpfile)
logger.setout(out)
if args.backup and not os.path.isdir(args.backup):
logger.dump("backup folder does not exist: %s " % (args.backup))
return
try:
processor.setup(args.input, args.server, args.query, args.backup)
finally:
out.close()
def _check_peers(self, now):
for addr, rec in self._addresses.items():
qids = rec.check(now)
if qids is not None:
del self._addresses[addr]
logger.dump("dropping peer from %s"%([addr]))
for qid in qids:
qrec = self._long_queries.get(qid, None)
if qrec is None: continue
qrec.aset.discard(addr)
if not qrec.aset: # if all addresses are gone for this query
del self._long_queries[qid]
q = qrec.query
logger.dump("dropping query %s"%(q.value()))
if type(q) == querymod.RawQuery:
self._nreceiver.unregisterraw(qid)
if type(q) == querymod.PeriodicQuery:
del self._periodic[qid]
def _check_peers(self, now):
for addr, rec in self._addresses.items():
qids = rec.check(now)
if qids is not None:
del self._addresses[addr]
logger.dump("dropping peer from %s" % ([addr]))
for qid in qids:
qrec = self._long_queries.get(qid, None)
if qrec is None: continue
qrec.aset.discard(addr)
if not qrec.aset: # if all addresses are gone for this query
del self._long_queries[qid]
q = qrec.query
logger.dump("dropping query %s" % (q.value()))
if type(q) == querymod.RawQuery:
self._nreceiver.unregisterraw(qid)
if type(q) == querymod.PeriodicQuery:
del self._periodic[qid]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--input', type=str, help='publisher interface to subscribe to', required=True)
parser.add_argument('-s', '--server', type=str, help='server interface to serve stats requests', required=True)
parser.add_argument('-q', '--query', type=str, help='listening interface for queries', required=True)
parser.add_argument('-d', '--dumpfile', type=str, help='output file (defaults: stdout)', default=None)
parser.add_argument('-b', '--backup', type=str, help='backup folder', default=None)
args = parser.parse_args()
if args.dumpfile is None:
out = sys.stdout
else:
out = open(args.dumpfile)
logger.setout(out)
if args.backup and not os.path.isdir(args.backup):
logger.dump("backup folder does not exist: %s "%(args.backup))
return
try:
processor.setup(args.input, args.server, args.query, args.backup)
finally:
out.close()
def _on_restore(self, loc):
fname = os.path.join(loc, 'collector.backup')
if not os.path.isfile(fname):
logger.dump("Nothing to restore from. %s does not exits"%(fname))
return
now = self._history.now()
nowstamp = self._history.seconds().now
try:
with tables.open_file(fname) as fileh:
for nm, obj in self._objmap.items():
grp = fileh.get_node(fileh.root, nm)
obj.restore(fileh, grp)
#TODO adjust time according to current value
renow = self._history.now()
if now < renow: raise Exception("Can not restore from future: %s < %s"%(querymod.tostamp(now),
querymod.tostamp(renow)))
logger.dump("Restoring with time gap of %s"%(elapsed(now-renow)))
renowstamp = self._history.seconds().now
if renowstamp < nowstamp:
units = [self._history.seconds(), self._history.minutes(), self._history.hours(), self._history.days()]
step = 0
end = 0
while renowstamp < nowstamp:
if units and renowstamp > end: # time to switch to bigger step
renowstamp -= step # step back to previous time
unit = units.pop(0)
step = unit.one
end = renowstamp+unit.count*step
renowstamp += step
self._tick_sources(renowstamp)
renowstamp += step
self._tick_sources(nowstamp)
logger.dump("Restored in %s"%(elapsed(datetime.datetime.utcnow()-now)))
except Exception, e:
emsg = traceback.format_exc()
logger.dump("Restore from %s failed: %s\n%s"%(fname, str(e), emsg))
shutil.move(fname, fname+'.failed')
sys.exit(-1)
def _cleanuptimefields(fields):
for ftype in ntypes.TimeType.timetypes.values():
if ftype.id in fields:
logger.dump("ignoring `%s`[%s] field in query"%(ftype.name, ftype.id))
del fields[ftype.id]
def _cleanuptimefields(fields):
for ftype in ntypes.TimeType.timetypes.values():
if ftype.id in fields:
logger.dump("ignoring `%s`[%s] field in query" %
(ftype.name, ftype.id))
del fields[ftype.id]
logger.dump("invalid query: %s" % (req))
err = {'error': str(e), 'badmsg': req}
self._send(addr, err)
return
arec = self._addresses.get(addr, None)
if not arec:
arec = ARecord(addr, hb)
self._addresses[addr] = arec
query = qry.get('query', None)
if query:
try:
q = querymod.Query.create(query, self._history)
except Exception, e:
traceback.print_exc()
logger.dump("bad query: %s from %s (hb:%ds): %s" %
(query, [addr], hb, str(e)))
err = {'error': str(e), 'badmsg': req}
self._send(addr, err)
return
logger.dump("got query %s from %s (hb:%ds):\n%s\n" %
(q.id, [addr], hb, pprint.pformat(query)))
if q.is_live():
self._register(q, addr)
arec.add(q.id)
else:
res = q.collect(self._nbuf, self._nreceiver.sources())
self.send_multipart([addr, res])
return
logger.dump("invalid query: %s"%(req))
err = {'error':str(e), 'badmsg':req}
self._send(addr, err)
return
arec = self._addresses.get(addr, None)
if not arec:
arec = ARecord(addr, hb)
self._addresses[addr] = arec
query = qry.get('query', None)
if query:
try:
q = querymod.Query.create(query, self._history)
except Exception, e:
traceback.print_exc()
logger.dump("bad query: %s from %s (hb:%ds): %s"%(query, [addr], hb, str(e)))
err = {'error':str(e), 'badmsg':req}
self._send(addr, err)
return
logger.dump("got query %s from %s (hb:%ds):\n%s\n"%(q.id, [addr], hb, pprint.pformat(query)))
if q.is_live():
self._register(q, addr)
arec.add(q.id)
else:
res = q.collect(self._nbuf, self._nreceiver.sources())
self.send_multipart([addr, res])
return
stat = qry.get('status', None)
def header(self):
msg = "capturing tags:\n"
for ft in self.flowtags:
msg += ' %s\n'%(flowtypes[ft])
logger.dump(msg)