def image_dump(path): s = Session() for image in s.query(Image).all(): with open(path + '/%d-%s.%s' %\ (image.timestamp, image.md5sum, image.filetype), 'w') as ifile: ifile.write(image.data) print 'Dumped : %d-%s.%s' % (image.timestamp, image.md5sum, image.filetype) s.close()
def gen_report(title): s = Session() # New Unique Image Counts over the course of the day. trend = {} for image in s.query(Image).all(): dts = datetime.datetime.fromtimestamp(image.timestamp) hrtime = int(time.mktime((dts.year, dts.month, dts.day, dts.hour, 0, 0, 0, 0, 0))) if hrtime not in trend: trend[hrtime] = 0 trend[hrtime] += 1 od = collections.OrderedDict(sorted(trend.items())) itrend = [{'data': [[i * 1000, od[i]] for i in od], 'label': 'Unique Images'}] # Top 10 Most common images top100 = s.query(Image).order_by(desc(Image.count)).limit(100).all() # Total Unique Images total_images = s.query(Image).count() # accounts accounts = s.query(Account).all() # Stats proto_top10 = get_stats(10) protos = s.query(Stat.proto, func.sum(Stat.count))\ .group_by(Stat.proto)\ .order_by(desc(func.sum(Stat.count))).all() report = env.get_template('report.html').render( title = title, accounts = accounts, itrend = json.dumps(itrend), itop = top100, itotal = total_images, pt10 = json.dumps(proto_top10), protos = protos, jquery = '\n'.join([ open('/usr/share/dofler/static/jquery.min.js').read(), open('/usr/share/dofler/static/jquery.flot.min.js').read(), open('/usr/share/dofler/static/jquery.flot.time.min.js').read(), ]).encode('utf-8') ) with open('DoFler-%s.html' % title.replace(' ','_'), 'w') as reportfile: reportfile.write(report) s.close()
def get_stats(limit): s = Session() data = [] protos = s.query(Stat.proto, func.sum(Stat.count))\ .group_by(Stat.proto)\ .order_by(desc(func.sum(Stat.count)))\ .limit(limit).all() for proto in protos: data.append({ 'label': proto[0], 'data': [[int(a[0] * 1000), int(a[1])] for a in s.query(Stat.timestamp, func.sum(Stat.count))\ .filter(Stat.proto == proto[0])\ .group_by(Stat.timestamp)\ .order_by(desc(Stat.timestamp))\ .all()] }) s.close() return data
def run(self): ''' Process startup. ''' s = Session() while int(time.time()) < self.delay: log.debug('%s: Parser Waiting til %s currently %s. sleeping 1s.' %( self.name, self.delay, int(time.time()))) time.sleep(1) self.command = setting('%s_command' % self.name).value\ .replace('{IF}', setting('listen_interface').value) self.api = DoflerClient( host=setting('server_host').value, port=setting('server_port').intvalue, username=setting('server_username').value, password=setting('server_password').value, ssl=setting('server_ssl').boolvalue, anon=setting('server_anonymize').boolvalue) s.close() self.realtime_process()