def check_mem(job, logger): """check if there is enough mem to execute task""" if 'issingle' in job and job['issingle']: return mem = gen_avail_mem() counter = 1 msgid = job['message_id'] memlimit = MEMLIMITS[job['num']] memlimits = format_byte_size(memlimit) while counter < 11: if mem < memlimit: logger.info("Deferring processing of: %(msgid)s " "%(avail)s available %(req)s required", dict(msgid=msgid, avail=format_byte_size(mem), req=memlimits)) if counter == 10: raise OSError("Insufficient memory") else: logger.info("Processing %(msgid)s: " "%(avail)s available %(req)s required", dict(msgid=msgid, avail=format_byte_size(mem), req=memlimits)) break time.sleep(30) mem = gen_avail_mem() counter += 1
def check_mem(job, logger): """check if there is enough mem to execute task""" if "issingle" in job and job["issingle"]: return mem = gen_avail_mem() counter = 1 msgid = job["message_id"] memlimit = MEMLIMITS[job["num"]] memlimits = format_byte_size(memlimit) while counter < 11: if mem < memlimit: logger.info( "Deferring processing of: %(msgid)s " "%(avail)s available %(req)s required", dict(msgid=msgid, avail=format_byte_size(mem), req=memlimits), ) if counter == 10: raise OSError("Insufficient memory") else: logger.info( "Processing %(msgid)s: " "%(avail)s available %(req)s required", dict(msgid=msgid, avail=format_byte_size(mem), req=memlimits), ) break time.sleep(30) mem = gen_avail_mem() counter += 1
def build_barchart(data, rows=None, header=None): "Builds a BarChart from Data supplied" if rows is None: graph = BarChart(950, 250) graph.chart.x = 30 graph.chart.y = 15 graph.chart.width = 940 graph.chart.height = 220 graph.plot.x = 30 graph.plot.y = 15 graph.plot.width = 940 graph.plot.height = 220 else: graph = BarChart() dates = [] mail_size = [] mail_total = [] spam_total = [] virus_total = [] for ind, msg in enumerate(data): if ind % 10: dates.append('') else: dates.append(str(msg.date)) mail_total.append(int(msg.mail_total)) spam_total.append(int(msg.spam_total)) virus_total.append(int(msg.virus_total)) mail_size.append(int(msg.total_size)) if rows: rows.append((str(msg.date), msg.mail_total, msg.spam_total, msg.virus_total, format_byte_size(msg.total_size))) graph.chart.data = [tuple(mail_total), tuple(spam_total), tuple(virus_total)] graph.plot.data = [tuple(mail_size)] graph.chart.categoryAxis.categoryNames = dates graph.plot.categoryAxis.categoryNames = dates if rows: rows.append((header, sum(mail_total), sum(spam_total), sum(virus_total), format_byte_size(sum(mail_size)))) return graph, rows return graph
def json(self): "recent messages json" value = 'white' if (self.spam and not self.highspam and not self.blacklisted and not self.nameinfected and not self.otherinfected and not self.virusinfected): value = 'spam' if self.highspam and (not self.blacklisted): value = 'highspam' if self.whitelisted: value = 'whitelisted' if self.blacklisted: value = 'blacklisted' if self.nameinfected or self.virusinfected or self.otherinfected: value = 'infected' if not self.scaned: value = 'gray' if (self.spam and (not self.blacklisted) and (not self.virusinfected) and (not self.nameinfected) and (not self.otherinfected)): status = _('Spam') if self.blacklisted: status = _('BS') if (self.virusinfected or self.nameinfected or self.otherinfected): status = _('Infected') if ((not self.spam) and (not self.virusinfected) and (not self.nameinfected) and (not self.otherinfected) and (not self.whitelisted)): status = _('Clean') if self.whitelisted: status = _('AS') if not self.scaned: status = _('NS') return dict( id=self.id, timestamp=self.timestamp.strftime('%Y-%m-%d %H:%M:%S %Z'), sascore=self.sascore, size=format_byte_size(self.size), subject=escape(truncate((self.subject and self.subject.strip()) or '---', 50)), from_address=escape( wrap_paragraphs(self.from_address, 32)), to_address=escape(wrap_paragraphs(self.to_address or '---', 32)), style=value, status=status, )
def jsonify_msg_list(element): """ Fixes the converting error in converting DATETIME objects to JSON """ value = 'white' if (element.spam and not element.highspam and not element.blacklisted and not element.nameinfected and not element.otherinfected and not element.virusinfected): value = 'spam' if element.highspam and (not element.blacklisted): value = 'highspam' if element.whitelisted: value = 'whitelisted' if element.blacklisted: value = 'blacklisted' if (element.nameinfected or element.virusinfected or element.otherinfected): value = 'infected' if not element.scaned: value = 'gray' if (element.spam and (not element.blacklisted) and (not element.virusinfected) and (not element.nameinfected) and (not element.otherinfected)): status = _('Spam') if element.blacklisted: status = _('BL') if (element.virusinfected or element.nameinfected or element.otherinfected): status = _('Infected') if ((not element.spam) and (not element.virusinfected) and (not element.nameinfected) and (not element.otherinfected) and (not element.whitelisted)): status = _('Clean') if element.whitelisted: status = _('WL') if not element.scaned: status = _('NS') return dict( id=element.id, timestamp=element.timestamp.strftime('%A, %d %b %Y %H:%M:%S %Z'), sascore=element.sascore, size=format_byte_size(element.size), subject=truncate(escape(element.subject), 50), from_address=wrap_paragraphs(escape(element.from_address), 32), to_address=wrap_paragraphs(escape(element.to_address), 32), style=value, status=status, )
def jsonify_msg_list(element): """ Fixes the converting error in converting DATETIME objects to JSON """ value = 'white' if (element.spam and not element.highspam and not element.blacklisted and not element.nameinfected and not element.otherinfected and not element.virusinfected): value = 'spam' if element.highspam and (not element.blacklisted): value = 'highspam' if element.whitelisted: value = 'whitelisted' if element.blacklisted: value = 'blacklisted' if (element.nameinfected or element.virusinfected or element.otherinfected): value = 'infected' if not element.scaned: value = 'gray' if (element.spam and (not element.blacklisted) and (not element.virusinfected) and (not element.nameinfected) and (not element.otherinfected)): status = _('Spam') if element.blacklisted: status = _('BL') if (element.virusinfected or element.nameinfected or element.otherinfected): status = _('Infected') if ((not element.spam) and (not element.virusinfected) and (not element.nameinfected) and (not element.otherinfected) and (not element.whitelisted)): status = _('Clean') if element.whitelisted: status = _('WL') if not element.scaned: status = _('NS') return dict( id=element.id, timestamp=element.timestamp.strftime('%Y-%m-%d %H:%M:%S %Z'), sascore=element.sascore, size=format_byte_size(element.size), subject=truncate(escape(element.subject), 50), from_address=wrap_paragraphs(escape(element.from_address), 32), to_address=wrap_paragraphs(escape(element.to_address), 32), style=value, status=status)
def get_database_size(engine): """Returns human formatted SQL database size. Example: 3.04 GB """ if engine.name == 'sqlite': size_bytes = engine.execute('PRAGMA page_count;').scalar() * engine.execute('PRAGMA page_size;').scalar() elif engine.name == 'mysql': size_bytes = engine.execute('SELECT sum(ROUND((DATA_LENGTH + INDEX_LENGTH),2)) AS Size ' 'FROM INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA like "%s";' % engine.url.database).scalar() elif engine.name == 'postgresql': size_bytes = engine.execute('SELECT pg_database_size(\'%s\');' % engine.url.database).scalar() else: raise NotImplemented return format_byte_size(float(size_bytes))
def get_database_size(engine): """Returns human formatted SQL database size. Example: 3.04 GB """ if engine.name == 'sqlite': size_bytes = engine.execute('PRAGMA page_count;').scalar() * engine.execute('PRAGMA page_size;').scalar() elif engine.name == 'mysql': size_bytes = engine.execute('SELECT sum(ROUND((DATA_LENGTH + INDEX_LENGTH - DATA_FREE),2)) AS Size ' 'FROM INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA like "%s";' % engine.url.database).scalar() elif engine.name == 'postgresql': size_bytes = engine.execute('SELECT pg_database_size(\'%s\');' % engine.url.database).scalar() else: raise NotImplemented return format_byte_size(float(size_bytes))
def json(self): "recent messages json" value = 'white' if (self.spam and not self.highspam and not self.blacklisted and not self.nameinfected and not self.otherinfected and not self.virusinfected): value = 'spam' if self.highspam and (not self.blacklisted): value = 'highspam' if self.whitelisted: value = 'whitelisted' if self.blacklisted: value = 'blacklisted' if self.nameinfected or self.virusinfected or self.otherinfected: value = 'infected' if not self.scaned: value = 'gray' if (self.spam and (not self.blacklisted) and (not self.virusinfected) and (not self.nameinfected) and (not self.otherinfected)): status = _('Spam') if self.blacklisted: status = _('BS') if (self.virusinfected or self.nameinfected or self.otherinfected): status = _('Infected') if ((not self.spam) and (not self.virusinfected) and (not self.nameinfected) and (not self.otherinfected) and (not self.whitelisted)): status = _('Clean') if self.whitelisted: status = _('AS') if not self.scaned: status = _('NS') return dict( id=self.id, timestamp=str(self.timestamp), sascore=self.sascore, size=format_byte_size(self.size), subject=escape(truncate((self.subject and self.subject.strip()) or '---', 50)), from_address=escape(wrap_paragraphs(self.from_address, 32)), to_address=escape(wrap_paragraphs(self.to_address or '---', 32)), style=value, status=status, )
def _pie_chart(self, data, title, headers, sortby): "Build PIE chart" headings = [headers] rows = [[ self._draw_square(PIE_CHART_COLORS[index]), getattr(row, 'address'), getattr(row, 'count'), format_byte_size(getattr(row, 'size')), '' ] for index, row in enumerate(data)] if len(rows) and len(rows) != 10: missing = 10 - len(rows) add_rows = [('', '', '', '', '') for ind in range(missing)] rows.extend(add_rows) if not rows: return headings.extend(rows) piedata = [getattr(row, sortby) for row in data] total = sum(piedata) labels = [("%.1f%%" % ((1.0 * getattr(row, sortby) / total) * 100)) for row in data] pie = PieChart() pie.chart.labels = labels pie.chart.data = piedata headings[1][4] = pie table_with_style = Table( headings, [0.2 * inch, 2.8 * inch, 0.5 * inch, 0.7 * inch, 3.2 * inch]) table_with_style.setStyle(TABLE_STYLE) paragraph = Paragraph(title, CUSTOM_HEADING1) self.parts.append(paragraph) self.parts.append(table_with_style) self.parts.append(Spacer(1, 70)) if (self.sentry % 2) == 0: self.parts.append(PageBreak())
def _pie_chart(self, data, title, headers, sortby): "Build PIE chart" headings = [headers] rows = [[self._draw_square(PIE_CHART_COLORS[index]), getattr(row, 'address'), getattr(row, 'count'), format_byte_size(getattr(row, 'size')), ''] for index, row in enumerate(data)] if len(rows) and len(rows) != 10: missing = 10 - len(rows) add_rows = [('', '', '', '', '') for ind in range(missing)] rows.extend(add_rows) if not rows: return headings.extend(rows) piedata = [getattr(row, sortby) for row in data] total = sum(piedata) labels = [("%.1f%%" % ((1.0 * getattr(row, sortby) / total) * 100)) for row in data] pie = PieChart() pie.chart.labels = labels pie.chart.data = piedata headings[1][4] = pie table_with_style = Table(headings, [0.2 * inch, 2.8 * inch, 0.5 * inch, 0.7 * inch, 3.2 * inch]) table_with_style.setStyle(TABLE_STYLE) paragraph = Paragraph(title, STYLES['Heading1']) self.parts.append(paragraph) self.parts.append(table_with_style) self.parts.append(Spacer(1, 70)) if (self.sentry % 2) == 0: self.parts.append(PageBreak())
def display(self, reportid, format=None): "Display a report" try: c.report_title = REPORTS[reportid]['title'] except KeyError: abort(404) filters = session.get('filter_by', []) if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']: rquery = ReportQuery(c.user, reportid, filters) query = rquery() cachekey = u'reportquery-%s-%s' % (c.user.username, reportid) query = query.options(FromCache('sql_cache_short', cachekey)) data = query[:10] if format == 'png': return self._generate_png(data, reportid) if format == 'csv': info = REPORTDL_MSG % dict(r=c.report_title, f='csv') audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return self._generate_csv(data, reportid) jsondata = [dict(tooltip=getattr(item, 'address'), y=getattr(item, REPORTS[reportid]['sort']), stroke='black', color=PIE_COLORS[index], size=getattr(item, 'size')) for index, item in enumerate(data)] template = '/reports/piereport.html' if reportid == '9': query = sa_scores(Session, c.user) if filters: dynq = DynaQuery(Message, query, filters) query = dynq.generate() cachekey = u'sascores-%s' % c.user.username query = query.options(FromCache('sql_cache_short', cachekey)) data = query.all() if format == 'json': scores = [] counts = [] for row in data: scores.append(dict(value=int(row.score), text=str(row.score))) counts.append(dict(y=int(row.count), tooltip=(_('Score ') + str(row.score) + ': ' + str(row.count)))) jsondata = dict(scores=scores, count=counts) elif format == 'png': return self._generate_png(data, reportid) else: jsondata = {} jsondata['labels'] = [{'value': index + 1, 'text': str(item.score)} for index, item in enumerate(data)] jsondata['scores'] = [item.count for item in data] template = '/reports/barreport.html' if reportid == '10': if format == 'json': data = [[item.address.strip(), get_hostname(item.address.strip()), country_flag(item.address.strip()), item.count, item.size] for item in data] template = '/reports/relays.html' if reportid == '11': query = message_totals(Session, c.user) if filters: dynq = DynaQuery(Message, query, filters) query = dynq.generate() cachekey = u'msgtotals-%s' % c.user.username query = query.options(FromCache('sql_cache_short', cachekey)) data = query.all() if format == 'png': return self._generate_png(data, reportid) elif format == 'json': dates = [] mail_total = [] spam_total = [] size_total = [] virus_total = [] for row in data: dates.append(str(row.ldate)) mail_total.append(int(row.mail_total)) spam_total.append(int(row.spam_total)) virus_total.append(int(row.virus_total)) size_total.append(int(row.total_size)) jsondata = dict(dates=[dict(value=index + 1, text=date) for index, date in enumerate(dates)], mail=[dict(y=total, tooltip=(_('Mail totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(mail_total)], spam=[dict(y=total, tooltip=(_('Spam totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(spam_total)], virii=[dict(y=total, tooltip=(_('Virus totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(virus_total)], volume=size_total, mail_total=sum(mail_total), spam_total=sum(spam_total), virus_total=sum(virus_total), volume_total=sum(size_total)) try: vpct = "%.1f" % ((1.0 * sum(virus_total) / sum(mail_total)) * 100) spct = "%.1f" % ((1.0 * sum(spam_total) / sum(mail_total)) * 100) except ZeroDivisionError: vpct = "0.0" spct = "0.0" jsondata['vpct'] = vpct jsondata['spct'] = spct data = [dict(date=str(row.ldate), mail_total=row.mail_total, spam_total=row.spam_total, virus_total=row.virus_total, size_total=format_byte_size(row.total_size), virus_percent="%.1f" % ((1.0 * int(row.virus_total) / int(row.mail_total)) * 100), spam_percent="%.1f" % ((1.0 * int(row.spam_total) / int(row.mail_total)) * 100)) for row in data] elif format == 'csv': info = REPORTDL_MSG % dict(r=c.report_title, f='csv') audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return self._generate_csv(data, reportid) else: jsondata = dict(mail=[], spam=[], virus=[], volume=[], labels=[]) for index, item in enumerate(data): jsondata['spam'].append(item.spam_total) jsondata['mail'].append(item.mail_total) jsondata['virus'].append(item.virus_total) jsondata['volume'].append(item.total_size) jsondata['labels'].append(dict(text=str(item.ldate), value=index)) template = '/reports/listing.html' if format == 'json': response.headers['Content-Type'] = JSON_HEADER return json.dumps(dict(items=list(data), pie_data=jsondata)) if format == 'pdf' and reportid != '9': info = REPORTDL_MSG % dict(r=c.report_title, f='pdf') audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return self._generate_pdf(data, reportid) c.reportid = reportid c.chart_data = json.dumps(jsondata) c.top_items = data c.active_filters = filters c.saved_filters = [] c.FILTER_BY = FILTER_BY c.FILTER_ITEMS = FILTER_ITEMS c.form = FilterForm(request.POST, csrf_context=session) info = REPORTVIEW_MSG % dict(r=c.report_title) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return self.render(template)
def display(self, reportid, format=None): "Display a report" c.report_title = REPORTS[reportid]['title'] filters = session.get('filter_by', []) if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']: rquery = ReportQuery(c.user, reportid, filters) query = rquery() cachekey = u'reportquery-%s-%s' % (c.user.username, reportid) query = query.options(FromCache('sql_cache_short', cachekey)) data = query[:10] if format == 'png': return self._generate_png(data, reportid) if format == 'csv': info = REPORTDL_MSG % dict(r=c.report_title, f='csv') audit_log(c.user.username, 1, info, request.host, request.remote_addr, datetime.now()) return self._generate_csv(data, reportid) jsondata = [ dict(tooltip=getattr(item, 'address'), y=getattr(item, REPORTS[reportid]['sort']), stroke='black', color=PIE_COLORS[index], size=getattr(item, 'size')) for index, item in enumerate(data) ] template = '/reports/piereport.html' if reportid == '9': query = sa_scores(Session, c.user) if filters: dynq = DynaQuery(Message, query, filters) query = dynq.generate() cachekey = u'sascores-%s' % c.user.username query = query.options(FromCache('sql_cache_short', cachekey)) data = query.all() if format == 'json': scores = [] counts = [] for row in data: scores.append( dict(value=int(row.score), text=str(row.score))) counts.append( dict(y=int(row.count), tooltip=(_('Score ') + str(row.score) + ': ' + str(row.count)))) jsondata = dict(scores=scores, count=counts) elif format == 'png': return self._generate_png(data, reportid) else: jsondata = {} jsondata['labels'] = [{ 'value': index + 1, 'text': str(item.score) } for index, item in enumerate(data)] jsondata['scores'] = [item.count for item in data] template = '/reports/barreport.html' if reportid == '10': if format == 'json': data = [[ item.address.strip(), get_hostname(item.address.strip()), country_flag(item.address.strip()), item.count, format_byte_size(item.size) ] for item in data] template = '/reports/relays.html' if reportid == '11': query = message_totals(Session, c.user) if filters: dynq = DynaQuery(Message, query, filters) query = dynq.generate() cachekey = u'msgtotals-%s' % c.user.username query = query.options(FromCache('sql_cache_short', cachekey)) data = query.all() if format == 'png': return self._generate_png(data, reportid) elif format == 'json': dates = [] mail_total = [] spam_total = [] size_total = [] virus_total = [] for row in data: dates.append(str(row.date)) mail_total.append(int(row.mail_total)) spam_total.append(int(row.spam_total)) virus_total.append(int(row.virus_total)) size_total.append(int(row.total_size)) jsondata = dict( dates=[ dict(value=index + 1, text=date) for index, date in enumerate(dates) ], mail=[ dict(y=total, tooltip=(_('Mail totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(mail_total) ], spam=[ dict(y=total, tooltip=(_('Spam totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(spam_total) ], virii=[ dict(y=total, tooltip=(_('Virus totals on ') + dates[index] + ': ' + str(total))) for index, total in enumerate(virus_total) ], volume=size_total, mail_total=sum(mail_total), spam_total=sum(spam_total), virus_total=sum(virus_total), volume_total=sum(size_total)) try: vpct = "%.1f" % ( (1.0 * sum(virus_total) / sum(mail_total)) * 100) spct = "%.1f" % ( (1.0 * sum(spam_total) / sum(mail_total)) * 100) except ZeroDivisionError: vpct = "0.0" spct = "0.0" jsondata['vpct'] = vpct jsondata['spct'] = spct data = [ dict(date=str(row.date), mail_total=row.mail_total, spam_total=row.spam_total, virus_total=row.virus_total, size_total=format_byte_size(row.total_size), virus_percent="%.1f" % ((1.0 * int(row.virus_total) / int(row.mail_total)) * 100), spam_percent="%.1f" % ((1.0 * int(row.spam_total) / int(row.mail_total)) * 100)) for row in data ] elif format == 'csv': info = REPORTDL_MSG % dict(r=c.report_title, f='csv') audit_log(c.user.username, 1, info, request.host, request.remote_addr, datetime.now()) return self._generate_csv(data, reportid) else: jsondata = dict(mail=[], spam=[], virus=[], volume=[], labels=[]) for index, item in enumerate(data): jsondata['spam'].append(item.spam_total) jsondata['mail'].append(item.mail_total) jsondata['virus'].append(item.virus_total) jsondata['volume'].append(item.total_size) jsondata['labels'].append( dict(text=str(item.date), value=index)) template = '/reports/listing.html' if format == 'json': response.headers['Content-Type'] = JSON_HEADER return json.dumps(dict(items=list(data), pie_data=jsondata)) if format == 'pdf' and reportid != '9': info = REPORTDL_MSG % dict(r=c.report_title, f='pdf') audit_log(c.user.username, 1, info, request.host, request.remote_addr, datetime.now()) return self._generate_pdf(data, reportid) c.reportid = reportid c.chart_data = json.dumps(jsondata) c.top_items = data c.active_filters = filters c.saved_filters = [] c.FILTER_BY = FILTER_BY c.FILTER_ITEMS = FILTER_ITEMS c.form = FilterForm(request.POST, csrf_context=session) info = REPORTVIEW_MSG % dict(r=c.report_title) audit_log(c.user.username, 1, info, request.host, request.remote_addr, datetime.now()) return render(template)
def format_byte_size(size): if size: # we use float since postgres driver will return decimal return format_byte_size(float(size)) else: return format_byte_size(0)
def test_bytes(self): eq_(format_byte_size(1), '1 B')
def test_kibibytes(self): eq_(format_byte_size(1000, binary=True), '1000 B') eq_(format_byte_size(1024, 0, True), '1 KiB') eq_(format_byte_size(1024, 2, True), '1.00 KiB')
def test_yottabytes(self): eq_(format_byte_size(123456789012345678901234567890, 2, True), '102121.06 YiB') eq_(format_byte_size(123456789012345678901234567890, 2, False), '123456.79 YB')
def test_zettabytes(self): eq_(format_byte_size(1234567890123456789012, 2, True), '1.05 ZiB') eq_(format_byte_size(1234567890123456789012, 2, False), '1.23 ZB')
def test_terabytes(self): eq_(format_byte_size(12345678901234, 2, True), '11.23 TiB') eq_(format_byte_size(12345678901234, 2, False), '12.35 TB')
def test_megabytes(self): eq_(format_byte_size(12345678, 2, True), '11.77 MiB') eq_(format_byte_size(12345678, 2, False), '12.35 MB')
def test_kilobytes(self): eq_(format_byte_size(1000), '1.0 kB') eq_(format_byte_size(1024, 0, False), '1 kB') eq_(format_byte_size(1024, 2, False), '1.02 kB') eq_(format_byte_size(1024, 0, False, True), '1 kilobytes') eq_(format_byte_size(1024, 2, False, True), '1.02 kilobytes')