def index(request):
custom_style = Style(background='transparent',
plot_background='transparent',
legend_font_size=30)
config = Config()
config.fill = True
pie_chart = pygal.Pie(legend_at_bottom=True,
inner_radius=.6,
style=custom_style,
margin=0,
width=400)
#pie_chart.title = 'Graduation Rate'
pie_chart.add('Graduated', grad_rate)
pie_chart.add('Ejected', eject_rate)
pie_chart.render_to_file('webapp/static/graduation_donut.svg')
pending_applications = Client.objects.filter(
client_phase='Applicant').count()
sum_residents = Client.objects.filter(client_phase='Resident').count()
applicants = Client.objects.filter(
client_phase='Applicant').order_by('-id')[:6]
sum_applicants = Client.objects.filter(
client_phase='Applicant').count() - 6
return render(
request, 'index.html', {
'pending_applications': pending_applications,
'applicants': applicants,
'sum_applicants': sum_applicants,
'sum_residents': sum_residents
})
def __init__(self,
mkchart,
title='Default',
show_legend=True,
x_title=None,
y_title=None,
show_dots=False,
range=None,
xrange=None,
show_x_guides=False,
show_y_guides=False,
fill=False,
human_readable=True,
style=DarkColorizedStyle):
self.context = App.get_running_app().root.ids.main_ui
style.background = 'transparent'
style.plot_background = 'transparent'
style.opacity = '.7'
config = Config()
config.show_legend = show_legend
config.human_readable = human_readable
config.fill = fill
config.title = title
config.x_title = x_title
config.y_title = y_title
config.show_dots = show_dots
config.xrange = xrange
config.range = range
config.show_x_guides = show_x_guides
config.show_y_guides = show_y_guides
config.style = style
self.config = config
self.chart = mkchart(config)
self.view = None
def define_config():
"""Defines some configuration settings for the pygal plot."""
config = Config()
config.show_legend = False
config.human_readable = True
config.fill = False
config.height = 1000
config.width = 2000
config.x_label_rotation = 300
config.show_legend = False
config.x_labels_major_every = 200 # 200
config.show_minor_x_labels = False
config.show_dots = False
config.fill = True
config.logarithmic = False
return config
def test_config_behaviours():
"""Test that all different way to set config produce same results"""
line1 = Line()
line1.show_legend = False
line1.fill = True
line1.pretty_print = True
line1.no_prefix = True
line1.x_labels = ['a', 'b', 'c']
line1.add('_', [1, 2, 3])
l1 = line1.render()
q = line1.render_pyquery()
assert len(q(".axis.x")) == 1
assert len(q(".axis.y")) == 1
assert len(q(".plot .series path")) == 1
assert len(q(".legend")) == 0
assert len(q(".x.axis .guides")) == 3
assert len(q(".y.axis .guides")) == 11
assert len(q(".dots")) == 3
assert q(".axis.x text").map(texts) == ['a', 'b', 'c']
line2 = Line(
show_legend=False,
fill=True,
pretty_print=True,
no_prefix=True,
x_labels=['a', 'b', 'c'])
line2.add('_', [1, 2, 3])
l2 = line2.render()
assert l1 == l2
class LineConfig(Config):
show_legend = False
fill = True
pretty_print = True
no_prefix = True
x_labels = ['a', 'b', 'c']
line3 = Line(LineConfig)
line3.add('_', [1, 2, 3])
l3 = line3.render()
assert l1 == l3
line4 = Line(LineConfig())
line4.add('_', [1, 2, 3])
l4 = line4.render()
assert l1 == l4
line_config = Config()
line_config.show_legend = False
line_config.fill = True
line_config.pretty_print = True
line_config.no_prefix = True
line_config.x_labels = ['a', 'b', 'c']
line5 = Line(line_config)
line5.add('_', [1, 2, 3])
l5 = line5.render()
assert l1 == l5
def test_config_behaviours():
"""Test that all different way to set config produce same results"""
line1 = Line()
line1.show_legend = False
line1.fill = True
line1.pretty_print = True
line1.no_prefix = True
line1.x_labels = ['a', 'b', 'c']
line1.add('_', [1, 2, 3])
l1 = line1.render()
q = line1.render_pyquery()
assert len(q(".axis.x")) == 1
assert len(q(".axis.y")) == 1
assert len(q(".plot .series path")) == 1
assert len(q(".legend")) == 0
assert len(q(".x.axis .guides")) == 3
assert len(q(".y.axis .guides")) == 11
assert len(q(".dots")) == 3
assert q(".axis.x text").map(texts) == ['a', 'b', 'c']
line2 = Line(
show_legend=False,
fill=True,
pretty_print=True,
no_prefix=True,
x_labels=['a', 'b', 'c'])
line2.add('_', [1, 2, 3])
l2 = line2.render()
assert l1 == l2
class LineConfig(Config):
show_legend = False
fill = True
pretty_print = True
no_prefix = True
x_labels = ['a', 'b', 'c']
line3 = Line(LineConfig)
line3.add('_', [1, 2, 3])
l3 = line3.render()
assert l1 == l3
line4 = Line(LineConfig())
line4.add('_', [1, 2, 3])
l4 = line4.render()
assert l1 == l4
line_config = Config()
line_config.show_legend = False
line_config.fill = True
line_config.pretty_print = True
line_config.no_prefix = True
line_config.x_labels = ['a', 'b', 'c']
line5 = Line(line_config)
line5.add('_', [1, 2, 3])
l5 = line5.render()
assert l1 == l5
def define_config():
"""Defines some configuration options for the pygal plot."""
config = Config()
config.show_legend = False
config.legend_at_bottom = True
config.human_readable = True
config.print_values = True
config.show_x_labels = True
config.show_y_labels = True
config.fill = True
return config
def test_config_behaviours():
line1 = Line()
line1.show_legend = False
line1.fill = True
line1.pretty_print = True
line1.no_prefix = True
line1.x_labels = ["a", "b", "c"]
line1.add("_", [1, 2, 3])
l1 = line1.render()
q = line1.render_pyquery()
assert len(q(".axis.x")) == 1
assert len(q(".axis.y")) == 1
assert len(q(".plot .series path")) == 1
assert len(q(".legend")) == 0
assert len(q(".x.axis .guides")) == 3
assert len(q(".y.axis .guides")) == 21
assert len(q(".dots")) == 3
assert q(".axis.x text").map(texts) == ["a", "b", "c"]
line2 = Line(show_legend=False, fill=True, pretty_print=True, no_prefix=True, x_labels=["a", "b", "c"])
line2.add("_", [1, 2, 3])
l2 = line2.render()
assert l1 == l2
class LineConfig(Config):
show_legend = False
fill = True
pretty_print = True
no_prefix = True
x_labels = ["a", "b", "c"]
line3 = Line(LineConfig)
line3.add("_", [1, 2, 3])
l3 = line3.render()
assert l1 == l3
line4 = Line(LineConfig())
line4.add("_", [1, 2, 3])
l4 = line4.render()
assert l1 == l4
line_config = Config()
line_config.show_legend = False
line_config.fill = True
line_config.pretty_print = True
line_config.no_prefix = True
line_config.x_labels = ["a", "b", "c"]
line5 = Line(line_config)
line5.add("_", [1, 2, 3])
l5 = line5.render()
assert l1 == l5
def genweightchart():
'''Generate weight chart with Pygal'''
weighthistory = session.query(Weight) \
.filter_by(user_id=current_user.id) \
.order_by(Weight.id.asc()) \
.all()
maxweight = session.query(Weight) \
.filter_by(user_id=current_user.id) \
.order_by(Weight.weight.desc()) \
.first()
if maxweight is None:
maxrange = 100
else:
maxrange = (int(maxweight.weight) + 5)
custom_style = Style(
background='transparent',
value_font_size=24,
title_font_size=36,
label_font_size=24,
margin=5,
plot_background='transparent',
foreground='#53E89B',
foreground_strong='#53A0E8',
foreground_subtle='#630C0D',
opacity='.6',
opacity_hover='.9',
transition='400ms ease-in',
colors=('#5cb85c', '#f0ad4e', '#d9534f'))
config = Config()
config.show_legend = False
config.y_labels = range(0, maxrange, 5)
config.margin_bottom=50
config.human_readable = True
config.fill = True
config.style=custom_style
config.print_labels=True
config.no_data_text='Add weight measurements!'
wlist = []
for entry in enumerate(weighthistory):
wlist.append(entry[1].weight)
line_chart = pygal.Line(config)
line_chart.title = "Weight History"
line_chart.add('Values', wlist)
chart = line_chart.render(is_unicode=True)
return chart
def genfoodchart(start, end):
'''Generate food chart with Pygal'''
now = datetime.datetime.now().strftime("%Y-%m-%d")
goodcount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=1).add_columns(Ranks.rank).count()
okaycount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=2).add_columns(Ranks.rank).count()
badcount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=3).add_columns(Ranks.rank).count()
custom_style = Style(
background='transparent',
value_font_size=24,
title_font_size=36,
margin=1,
plot_background='transparent',
foreground='#53E89B',
foreground_strong='#53A0E8',
foreground_subtle='#630C0D',
opacity='.6',
opacity_hover='.9',
transition='400ms ease-in',
colors=('#5cb85c', '#f0ad4e', '#d9534f'))
config = Config()
config.show_legend = True
config.legend_at_bottom=True
config.legend_at_bottom_columns=1
config.legend_box_size=10
config.human_readable = True
config.fill = True
config.style=custom_style
config.print_labels=True
config.print_values=True
config.no_data_text='Need to add some food!'
pie_chart = pygal.Pie(config)
pie_chart.title = "Current Food Stats"
pie_chart.add('Good', goodcount)
pie_chart.add('Okay', okaycount)
pie_chart.add('Bad', badcount)
chart = pie_chart.render(is_unicode=True)
return chart
def genfoodchart(start, end):
'''Generate food chart with Pygal'''
now = datetime.datetime.now().strftime("%Y-%m-%d")
goodcount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=1).add_columns(Ranks.rank).count()
okaycount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=2).add_columns(Ranks.rank).count()
badcount = session.query(Food) \
.filter_by(user_id=current_user.id) \
.filter(Food.food_date.between(start, end)).join(Ranks) \
.filter_by(rank=3).add_columns(Ranks.rank).count()
custom_style = Style(
background='transparent',
value_font_size=24,
title_font_size=36,
margin=1,
plot_background='transparent',
foreground='#53E89B',
foreground_strong='#53A0E8',
foreground_subtle='#630C0D',
opacity='.6',
opacity_hover='.9',
transition='400ms ease-in',
colors=('#5cb85c', '#f0ad4e', '#d9534f'))
config = Config()
config.show_legend = True
config.legend_at_bottom=True
config.legend_at_bottom_columns=1
config.legend_box_size=10
config.human_readable = True
config.fill = True
config.style=custom_style
config.print_labels=True
config.print_values=True
config.no_data_text='Need to add some food!'
pie_chart = pygal.Pie(config)
pie_chart.title = "Current Food Stats"
pie_chart.add('Good', goodcount)
pie_chart.add('Okay', okaycount)
pie_chart.add('Bad', badcount)
chart = pie_chart.render(is_unicode=True)
return chart
def genweightchart():
'''Generate weight chart with Pygal'''
weighthistory = session.query(Weight) \
.filter_by(user_id=current_user.id) \
.order_by(Weight.id.desc()) \
.all()
maxweight = session.query(Weight) \
.filter_by(user_id=current_user.id) \
.order_by(Weight.id.desc()) \
.first()
if maxweight is None:
maxrange = 100
else:
maxrange = (int(maxweight.weight) + 50)
custom_style = Style(
background='transparent',
value_font_size=24,
title_font_size=36,
margin=1,
plot_background='transparent',
foreground='#53E89B',
foreground_strong='#53A0E8',
foreground_subtle='#630C0D',
opacity='.6',
opacity_hover='.9',
transition='400ms ease-in',
colors=('#5cb85c', '#f0ad4e', '#d9534f'))
config = Config()
config.show_legend = True
config.legend_at_bottom=True
config.y_labels = range(0, maxrange, 25)
config.human_readable = True
config.fill = True
config.style=custom_style
config.print_labels=True
config.no_data_text='Add weight measurements!'
wlist = []
for entry in enumerate(weighthistory):
wlist.append(entry[1].weight)
line_chart = pygal.Line(config)
line_chart.title = "Weight History"
line_chart.add('Values', wlist)
chart = line_chart.render(is_unicode=True)
return chart
def languages_draw(prefix, file_name, dir_name, time_title, names, counts):
chart = pygal.Bar(interpolate='cubic', width=1000)
config = Config()
config.show_legend = False
config.human_readable = True
config.fill = True
config.style = DarkColorizedStyle
config.label_font_size = 36
config.x_label_rotation = 45
chart.config = config
chart.title = file_name + " [" + time_title + "]"
chart.x_labels = names
chart.add(prefix, counts)
save_name = os.path.join(dir_name, file_name + ".svg")
chart.render_to_file(save_name)
def __prepareSVGConfigGeneric(colors): config = Config() config.show_legend = True config.human_readable = True config.fill = True config.order_min = 0 config.x_label_rotation = 80 config.margin = 80 config.legend_font_size = 10 config.tooltip_border_radius = 10 config.legend_font_size = 10 config.no_data_text = "No changes found" config.style = Style(background='transparent', plot_background='#f5f5f5', foreground='#428bca', foreground_light='#000000', foreground_dark='#428bca', opacity='.6', opacity_hover='.9', transition='400ms ease-in', colors=colors) return config
def analyze(self):
self.report.filebasename = file_basename(self.config.filename)
self.report.filedirname = file_dirname(self.config.filename)
self.report.filemimetype = file_mimetype(self.config.filename)
magicresult = file_magic(self.config.filename)
self.report.filemagic = "%s (%s)" % (magicresult["match"]["longname"], magicresult["match"]["shortname"]) if magicresult["match"] else None
self.report.hashes.crc32 = file_hashes(self.config.filename, 'crc32')
self.report.hashes.md5 = file_hashes(self.config.filename, 'md5')
self.report.hashes.sha1 = file_hashes(self.config.filename, 'sha1')
self.report.hashes.sha256 = file_hashes(self.config.filename, 'sha256')
self.report.hashes.sha512 = file_hashes(self.config.filename, 'sha512')
self.report.hashes.ssdeep = file_hashes(self.config.filename, 'ssdeep')
with nostdout():
self.report.subfiles = file_subfiles(self.config.filename)
# this might take some time to finish
# based on the filesize, runtime might increase
# will be autodisabled based on statsfilesizelimit config option
if self.config.enableentropycompressionstats:
stats = objdict(file_entropy_compression_stats(self.config.filename))
self.report.filesize = stats.filesizeinbytes
self.report.fileminsize = float(stats.minfilesize)
self.report.filecompressionratio = float(stats.compressionratio)
self.report.fileentropy = float(stats.entropy)
self.report.fileentropycategory = stats.entropycategory
# this might take some time to finish
# based on the filesize, runtime might increase
# should be autodisabled based on (statsfilesizelimit) config option
if self.config.enablefilevisualization:
self.report.visual.pngrgb = file_to_pngimage(self.config.filename)
self.report.visual.pnggray = file_to_pngimage(self.config.filename, enable_colors=False)
rh = identicon(self.report.hashes.sha256)
self.report.visual.identicon = rh.identicon if rh.success else None
config = Config()
config.x_title = 'Bytes'
config.y_title = 'Frequency'
config.x_scale = .25
config.y_scale = .25
config.width = 900
config.height = 300
config.title_font_size = 9
config.tooltip_font_size = 0
config.tooltip_border_radius = 0
config.no_data_text = ""
config.show_legend = False
config.show_only_major_dots = True
config.human_readable = False
config.show_y_labels = False
config.fill = True
config.style = CleanStyle
bar_chart = pygal.Bar(config)
# if enableentropycompressionstats config option is disabled, stats won't be generated above
# as such we need to explicitly generate, on-demand
if not stats:
stats = objdict(file_entropy_compression_stats(self.config.filename))
bar_chart.add('', stats.bytefreqlist)
self.report.visual.bytefreqhistogram = bar_chart.render(is_unicode=False)
# pygal inserts a copyright symbol in rendered chart output
# need to explicitly clean it before returning
pygalregex = re.compile(r"\xc2\xa9")
self.report.visual.bytefreqhistogram = pygalregex.sub("", self.report.visual.bytefreqhistogram)
else:
self.report.visual.pngrgb = None
self.report.visual.pnggray = None
self.report.visual.identicon = None
self.report.visual.bytefreqhistogram = None
# done with analysis, normalize report and return
self.report = dict_normalize(self.report)
def using_pygal(DataFile):
print("--using_pygal...")
# Get the data as dataframe
RightData = mold_data(DataFile)
# Remove unnecessary information
RightData = RightData.drop("edit-no", axis=1)
RightData = RightData.drop("type", axis=1)
RightData = RightData.drop("analysis", axis=1)
RightData = RightData.drop("char-diff", axis=1)
RightData = RightData.drop("char-diff-abs", axis=1)
# RightData1: set levenshtein to zero for substantives (effectively ignoring them)
RightData1 = RightData.copy(deep=True)
RightData1.loc[RightData1["category"] =="other","levenshtein"] = 0
# Fix some details: levenshtein as numerical, remove x from line-id
RightData1["levenshtein"] = RightData1["levenshtein"].astype(str).convert_objects(convert_numeric=True)
RightData1["line-id"] = RightData1["line-id"].map(lambda x: x.rstrip("x"))
RightData1 = RightData1.groupby("line-id").sum()
#print(RightData1.head(20))
# RightData2: set levenshtein to zero for copyedits (effectively ignoring them)
RightData2 = RightData.copy(deep=True)
RightData2.loc[RightData2["category"] =="copyedit","levenshtein"] = 0
# Fix some details: levenshtein as numerical, remove x from line-id
RightData2["levenshtein"] = RightData2["levenshtein"].astype(str).convert_objects(convert_numeric=True)
RightData2["line-id"] = RightData2["line-id"].map(lambda x: x.rstrip("x"))
RightData2 = RightData2.groupby("line-id").sum()
#print(RightData2.head(20))
# Select the range of data we want
Lines = RightData1.index.values
Copyedits = RightData1.loc[:,"levenshtein"]
Substantives = RightData2.loc[:,"levenshtein"]
# Apply some interpolation
CopyeditsSF = sf(Copyedits, 35, 1, mode="nearest")
SubstantivesSF = sf(Substantives, 35, 1, mode="nearest")
# Graph general configuration
config = Config()
config.show_legend = False
config.human_readable = True
config.fill = False
# Line chart
LineChart = pygal.StackedLine(config,
height=1000,
width = 2000,
x_label_rotation=300,
show_legend=False,
x_labels_major_every=200,
show_minor_x_labels=False,
show_dots=False,
fill=True,
logarithmic=False,
range=(0, 60))
LineChart.title ="The Martian Modifications (copyedits and substantives)"
LineChart.y_title = "Levenshtein distance (smoothed)"
LineChart.x_title = "Lines in the novel"
LineChart.x_labels = Lines
LineChart.add("Copyedits", CopyeditsSF)
LineChart.add("Substantives", SubstantivesSF)
LineChart.render_to_file("MartianMods_copyedits+substantives.svg")
def identify(self, filename):
self.report["filename"] = os.path.basename(filename)
self.report["filename_absolute"] = filename
self.report["filemimetype"] = utils.file_mimetype(filename)
self.report["filemagic"] = utils.file_magic(filename)
if self.report["filemimetype"] in self.pcapmimetypes:
self.report["filecategory"] = "CAP"
self.report["filetype"] = "PCAP"
self.logger.info(
"Identified %s as type %s (%s)"
% (self.report["filename"], self.report["filetype"], self.report["filemimetype"])
)
else:
self.logger.info(
"File %s of type %s is not supported in the current version"
% (self.report["filename"], self.report["filemimetype"])
)
return None
self.logger.debug("Calculating file hashes for %s" % (self.report["filename"]))
self.report["hashes"]["crc32"] = utils.file_hashes(filename, "crc32")
self.report["hashes"]["md5"] = utils.file_hashes(filename, "md5")
self.report["hashes"]["sha1"] = utils.file_hashes(filename, "sha1")
self.report["hashes"]["sha256"] = utils.file_hashes(filename, "sha256")
self.report["hashes"]["sha512"] = utils.file_hashes(filename, "sha512")
self.report["hashes"]["ssdeep"] = utils.file_hashes(filename, "ssdeep")
self.logger.info("Completed crc32/md5/sha{1,256,512}/ssdeep hash calculations")
if self.config["enable_entropy_compression_stats"]:
self.logger.debug("Collecting entropy compression stats for %s" % (self.report["filename"]))
stats = utils.entropy_compression_stats(filename)
self.report["filesize"] = stats["filesizeinbytes"]
self.report["fileminsize"] = float(stats["minfilesize"])
self.report["filecompressionratio"] = float(stats["compressionratio"])
self.report["fileentropy"] = float(stats["shannonentropy"])
# if entropy falls within the 0 - 1 or 7 - 8 range, categorize as suspicious
if (self.report["fileentropy"] > 0 and self.report["fileentropy"] < 1) or self.report["fileentropy"] > 7:
self.report["fileentropy_category"] = "SUSPICIOUS"
else:
self.report["fileentropy_category"] = "NORMAL"
self.logger.info("Completed entropy compression stats collection")
else:
stats = {}
if self.config["enable_bytefreq_histogram"]:
self.logger.debug("Generating Byte-Frequency histogram for %s" % (self.report["filename"]))
config = Config()
config.x_title = "Bytes"
config.y_title = "Frequency"
config.x_scale = 0.25
config.y_scale = 0.25
config.width = 900
config.height = 300
config.title_font_size = 9
config.tooltip_font_size = 0
config.tooltip_border_radius = 0
config.no_data_text = ""
config.show_legend = False
config.show_only_major_dots = True
config.human_readable = False
config.show_y_labels = False
config.fill = True
config.style = CleanStyle
bar_chart = pygal.Bar(config)
if "bytefreqlist" in stats.keys():
bar_chart.add("", stats["bytefreqlist"])
self.report["filebytefreqhistogram"] = bar_chart.render(is_unicode=False)
self.logger.info("Completed Byte-Frequency histogram generation")
# testing and db support to identify visually similar files/sessions
if self.config["enable_file_visualization"]:
self.logger.debug("Generating file visualization for %s" % (self.report["filename"]))
self.report["filevis_png"] = utils.file_to_pngimage(filename)
self.report["filevis_png_bw"] = utils.file_to_pngimage(filename, enable_colors=False)
self.logger.info("Completed file visualization generation")
return dict(self.report)
def decom(rec):
return rec['decom'].upper()
if __name__ == '__main__':
# Read in CSV report, strip the headers, blank lines
print(colored('Reading CSV report ...','cyan'))
csv = read_csv(args['from'])
data_1, data_2, data_3 = tee(gen_csv(csv),3)
# Make a renderable radar
# where the `decomposition algorithms` are plotted on X (corners of Radar)
# and `variations of parameters` are plotted on Y (contour lines)
print(colored('Preparing radar ...','cyan'))
cfg = Config()
cfg.fill = True
radar = pygal.Radar(cfg, range=(70,85))
radar.title = 'Clustering/Feature Comparison'
bar = {} # Bar charts, indexed by cluster type
# NOTE: Need to do the loop, not the `set` conversion
# otherwise the list gets sorted which becomes out of sync
labels = []
for n in data_1:
if label(n) not in labels:
labels.append(label(n))
# Create a bar chart for each of the cluster type
bar[label(n)] = pygal.HorizontalBar(cfg, range=(70,90))
bar[label(n)].title = 'Cluster with ' + label(n).upper()
radar.x_labels = labels
def create_chartguardian(id_station, station):
data_status = dict()
data_time = dict()
list_status = list()
list_time = list()
station_name = str()
station_id = str()
data_down = list()
data_up = list()
data_label = list()
last_problem = str()
status_alert = str()
provider_alert = str()
mediatype_alert = str()
streamuri_alert = str()
stationid_alert = str()
ts = int(time.time())
date_query = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
with drt.connect(db=data_config[env_app]['RETHINK-DB']) as conn:
cursor_up = drt.table(data_config[env_app]['RETHINK-TABLE']).filter((drt.row["date"] == date_query) & (drt.row["station_id"] == id_station)).run(conn)
cursor_down = drt.table(data_config[env_app]['RETHINK-TABLE']).filter((drt.row["date"] == date_query) & (drt.row["station_id"] == id_station)).run(conn)
for document_down in cursor_down:
station_name = document_down["station_name"]
station_id = document_down["station_id"]
if len(document_down["status_array"]) < 31:
print("Size array status DOWN")
print(len(document_down["status_array"]))
start_point = 0
else:
start_point = len(document_down["status_array"]) - 31
stat_array = document_down["status_array"]
provider_alert = document_down["provider"]
mediatype_alert = stream_config[env_app][station]['media-type']
streamuri_alert = stream_config[env_app][station]['uri-stream']
stationid_alert = id_station
for index, item in enumerate(stat_array[start_point:]):
status_len = len(document_down["status_array"])
ts = int(item["time"])
date_down = datetime.datetime.fromtimestamp(ts).strftime('%H:%M:%S')
#[None, None, None, None, None, None, {'value': 0, 'label': '10:10:10'}
if item["status"] == "down":
item_down = {
'value': 0,
'label': date_down
}
item_label = date_down
data_down.append(item_down)
data_label.append(item_label)
last_problem = date_down
status_alert = item["status"]
else:
item_none = None
data_down.append(item_none)
item_label = None
data_label.append(item_label)
ts = int(item["time"])
date_day = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
data_status.update({ index: item["status"] })
data_time.update({ index: date_day })
list_status.append(item["status"])
list_time.append(date_day)
for document_up in cursor_up:
print("==== document_up ===")
print(document_up)
station_name = document_up["station_name"]
station_id = document_up["station_id"]
if len(document_up["status_array"]) < 31:
print("Size array status UP")
len(document_up["status_array"])
start_point = 0
else:
start_point = len(document_up["status_array"]) - 31
stat_array = document_up["status_array"]
for index, item in enumerate(stat_array[start_point:]):
status_len = len(document_up["status_array"])
#{'value': 1, 'label': ''}
if item["status"] == "up":
print("====== ITEM STATUS =====")
print(item)
status_val = 1
status_label = "Sin Problemas"
item_up = {
'value': 1,
'label': ''
}
data_up.append(item_up)
else:
status_val = 0
status_label = "Problemas"
item_down = None
data_up.append(item_down)
try:
custom_style = Style(
opacity='.6',
opacity_hover='.9',
transition='400ms ease-in',
label_font_family='Liberation Mono',
label_font_size=8,
major_label_font_size=10,
colors=('#e74c3c', '#2980b9', '#e67e22', '#E87653', '#E89B53'))
configpg = Config()
configpg.human_readable = True
configpg.fill = False
configpg.x_title='Ultimo Problema: ' + last_problem
configpg.show_y_labels=False
configpg.legend_at_bottom_columns=False
configpg.legend_at_bottom=True
configpg.width=850
configpg.height=400
configpg.print_labels=False
configpg.margin_bottom=30
configpg.x_label_rotation=60
configpg.range=(0, 1)
configpg.show_y_labels=True
configpg.pretty_print=True
extra_args = {
'ACL': 'public-read',
'CacheControl': 'no-cache, no-store, must-revalidate'
}
line_chart = pygal.Line(style=custom_style, config=configpg)
line_chart.title = station_name + ' - ' + date_query
line_chart.x_labels = (data_label)
line_chart.x_labels_major = (data_label)
line_chart.add('Problemas', data_down , allow_interruptions=True, dots_size=5, stroke_style={'width': 4, 'linecap': 'round', 'linejoin': 'round'})
line_chart.add('Activo',data_up , allow_interruptions=True, show_dots=True, dots_size=5, stroke_style={'width': 4, 'linecap': 'round', 'linejoin': 'round'})
line_chart.add('Rango: 30 minutos', None)
line_chart.render_to_png(station_id + '-' + date_query + '.png')
#connection.close()
#connection.close()
upload_s3 = s3.upload_file(station_id + '-' + date_query + '.png', bucket_name, env_app+'/'+station_id + '-' + str(ts) + '.png', ExtraArgs=extra_args)
os.remove(station_id + '-' + date_query + '.png')
print(last_problem)
alert_data = {
's3_key': env_app+'/'+station_id + '-' + str(ts) + '.png',
'time_alert': last_problem
}
if status_alert == "down":
#alert_error(status_alert, station_name, provider_alert, ts, mediatype_alert, streamuri_alert, stationid_alert, date_query, alert_data)
streaming_alert(status_alert, station_name, provider_alert, ts, mediatype_alert, streamuri_alert, stationid_alert, date_query, alert_data)
return alert_data
else:
message = "No se tiene data que mostrar"
print(message)
return message
except IOError as e:
print("Se tiene un problema")
print(e)
message = e
return message
#create_chartguardian("tv-capital", "capital-tv")
# if iDataNum % nDataStepCount == 0:
# aMajorXData.append( aData[aFields[0]][iDataNum] )
# Import Style
# from pygal.style import LightSolarizedStyle
# from pygal.style import DefaultStyle
from pygal.style import RotateStyle
# Create Config object
from pygal import Config
chartCfg = Config()
chartCfg.show_legend = True
chartCfg.human_readable = True
chartCfg.pretty_print = True
if bFilledLineChart:
chartCfg.fill = True
else:
chartCfg.fill = False
chartCfg.x_scale = 1
chartCfg.y_scale = 1
chartCfg.x_label_rotation = 45
chartCfg.include_x_axis = True
chartCfg.show_dots = False
if nMaxDataCount > 0:
chartCfg.show_minor_x_labels = False
chartCfg.x_labels_major_count = nMaxDataCount
#chartCfg.style = DefaultStyle #LightSolarizedStyle
chartCfg.style = RotateStyle('#FF0000')
chartCfg.print_values = True #False
chartCfg.print_zeroes = True
chartCfg.no_data_text = "All values are 0"
def drawcapitalcostcomponentsfigure(techlist):
config = Config()
config.show_legend = True
config.human_readable = True
config.dots_size = 3
config.x_label_rotation = 270
config.legend_at_bottom = True
config.x_title = "capital cost of power based components [$/kW]"
config.y_title = "capital cost of energy based components [$/kWh]"
#config.show_dots = False
config.fill = True
#config.show_minor_x_labels = False
config.stroke_style = {'width': 1}
config.style = pygal.style.styles['default'](
label_font_size=12,
stroke_opacity=0,
stroke_opacity_hover=0,
transition='100000000000s ease-in')
xy_chart = pygal.XY(config)
xmin = 10000
ymin = 10000
xmax = 0.001
ymax = 0.001
for tech in techlist:
power_minstring = "capital_cost_of_power_based_components_min"
power_maxstring = "capital_cost_of_power_based_components_max"
energy_minstring = "capital_cost_of_energy_based_components_min"
energy_maxstring = "capital_cost_of_energy_based_components_max"
minpowerlink = ''
maxpowerlink = ''
minenergylink = ''
maxenergylink = ''
minpowerlabel = ''
maxpowerlabel = ''
minenergylabel = ''
maxenergylabel = ''
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_minstring).
first().source_id).first() is not None:
minpowerlink = Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_minstring).first().source_id).first().link
minpowerlabel = 'min. cost of power based components: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_minstring).
first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_minstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_maxstring).
first().source_id).first() is not None:
maxpowerlink = Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_maxstring).first().source_id).first().link
maxpowerlabel = 'max. cost of power based components: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_maxstring).
first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_maxstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_minstring).
first().source_id).first() is not None:
minenergylink = Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_minstring).first().source_id).first().link
minenergylabel = 'min. cost of energy based components: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_minstring)
.first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_minstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_maxstring).
first().source_id).first() is not None:
maxenergylink = Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_maxstring).first().source_id).first().link
maxenergylabel = 'max. cost of energy based components: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_maxstring)
.first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_maxstring).first().source_id).first().
releaseyear)
xy_chart.add(f"{tech.name}", [{
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
minenergylabel,
'xlink': {
'href': minenergylink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value),
'label':
minpowerlabel,
'xlink': {
'href': minpowerlink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value),
'label':
maxenergylabel,
'xlink': {
'href': maxenergylink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
maxpowerlabel,
'xlink': {
'href': maxpowerlink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
minenergylabel,
'xlink': {
'href': minenergylink,
'target': '_blank'
}
}])
if Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value is not None:
xmin = min(
xmin,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value)
xmax = max(
xmax,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value)
if Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value is not None:
ymin = min(
ymin,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value)
ymax = max(
ymax,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value)
xup = 10**int(math.floor(math.log10(xmax)))
yup = 10**int(math.floor(math.log10(ymax)))
while xup < xmax:
xup = xup + 10**int(math.floor(math.log10(xmax)))
while yup < ymax:
yup = yup + 10**int(math.floor(math.log10(ymax)))
xy_chart.xrange = (10**int(math.floor(math.log10(xmin))), xup)
xy_chart.range = (10**int(math.floor(math.log10(ymin))), yup)
xy_chart.render()
return xy_chart.render_data_uri()
'time': time,
'displayTime': displayTime,
'location': location,
'people': people
}
key = {'id': time + "_" + location}
graphs.update(key, insert, upsert=True)
# Generate graph
graph = graphs.find({'location': location}).sort("key", pymongo.ASCENDING)
obj = loads(dumps(graph))
config = Config()
config.show_legend = False
config.fill = True
config.style = LightGreenStyle
config.x_label_rotation = 80
config.show_only_major_dots = True
config.x_labels_major_every = 2
config.show_minor_x_labels = False
config.show_minor_y_labels = False
config.major_label_font_size = 12
config.print_values = False
config.print_zeroes = True
config.disable_xml_declaration = True
chart = pygal.Line(config)
t = []
p = []
for l in obj:
def using_pygal(DataFile):
print("using_pygal...")
## Getting the data
with open(DataFile, "r") as infile:
AllData = pd.read_table(infile)
#print(AllData.head())
# Select the data we want
Lines = AllData.loc[:,"line-no"][0:50]
ItemIDs = AllData.loc[:,"item-id"][0:50]
Version1s = AllData.loc[:,"version1"][0:50]
Version2s = AllData.loc[:,"version2"][0:50]
Categories = AllData.loc[:,"category"][0:50]
Types = AllData.loc[:,"type"][0:50]
LevDists = AllData.loc[:,"levenshtein"][0:50]
CharDiffsAbs = AllData.loc[:,"char-delta-abs"][0:50]
CharDiffs = AllData.loc[:,"char-delta"][0:50]
# Apply very simple smoothing to LevDists
LevDistsSmth = []
Smthp0 = (LevDists[0] + LevDists[1])/2
Smthp1 = (LevDists[1] + LevDists[2])/2
LevDistsSmth.append(Smthp0)
LevDistsSmth.append(Smthp1)
for i in range(2, len(LevDists)-2):
NewValue = (LevDists[i-2]/2 + LevDists[i-1] + LevDists[i] + LevDists[i+1] + LevDists[i+2]/2) / 4
LevDistsSmth.append(NewValue)
Smthm0 = (LevDists[len(LevDists)-2] + LevDists[len(LevDists)-3]) / 2
Smthm1 = (LevDists[len(LevDists)-1] + LevDists[len(LevDists)-2]) / 2
LevDistsSmth.append(Smthm0)
LevDistsSmth.append(Smthm1)
# Graph general configuration
config = Config()
config.show_legend = False
config.human_readable = True
config.fill = False
# Graphing: bar chart
BarChart1 = pygal.Bar(config)
BarChart1.title = "The Martian Modifications (Char-Diff)"
BarChart1.x_labels = map(str, range(0, 50))
#BarChart.add("Levenshtein", LevDists)
BarChart1.add("Char-Diff", CharDiffs)
#chart.add("Char-Diff-Abs", CharDiffsAbs)
BarChart1.render_to_file("BarChart1.svg")
# Graphing: another bar chart
BarChart2 = pygal.Bar(config, x_label_rotation=270)
BarChart2.title ="The Martian Modifications (Labels)"
#LineChart.x_labels = map(str, ItemIDs)
for i in range(0,30):
BarChart2.add(ItemIDs[i], [{"value" : CharDiffs[i], "label" : str(Version1s[i])+" => "+str(Version2s[i]), 'color': 'red'}])
BarChart2.render_to_file("BarChart2.svg")
# Line chart
LineChart1 = pygal.Line(config, x_label_rotation=270, show_legend=True)
LineChart1.title ="The Martian Modifications (smoothed)"
LineChart1.add("LevDists", LevDists)
LineChart1.add("LevDistsSmth", LevDistsSmth)
LineChart1.render_to_file("LineChart1.svg")
title_font_size=20,
legend_box_size=20,
colors=('#04B404', 'green', 'green', 'green'))
#Style für die Top20 Weltkarten
world_style2 = Style(
background='transparent',
plot_background='#f0efff',
title_font_size=17,)
#Diagramm-Einstellungen
config = Config()
config.rounded_bars=5
config.legend_box_size=20
config.x_label_rotation=90
config.fill=True
#Horizontales Histogramm mit Vergleich zwischen Installationszahlen 2015 und 2016
chart = pygal.HorizontalBar(style=chart_style1,legend_box_size=20)
chart.title = 'Installationen 2015 und 2016'
chart.add('2015', summe_installs_2015, rounded_bars=5)
chart.add('2016', summe_installs_2016, rounded_bars=5)
chart.render_to_file('charts/installs_compared_2015_2016.svg')
#Kreisdiagramm mit Deinstallations- und "Behalten"-Quote von 2015
pie_chart = pygal.Pie(style=chart_style1,legend_box_size=20)
pie_chart.title = 'Deinstallationsquote 2015'
pie_chart.add('Deinstalliert in %', round(deinstallationsquote_2015,1))
pie_chart.add('Behalten in %', round(behalten_quote_2015,1))
pie_chart.render_to_file('charts/uninstalls_2015.svg')
def drawdensityfigure(techlist, par):
config = Config()
config.show_legend = True
config.human_readable = True
config.dots_size = 3
config.x_label_rotation = 270
config.legend_at_bottom = True
if par == "gravimetric":
power_unit = "[W/kg]"
energy_unit = "[Wh/kg]"
else:
power_unit = "[kW/m^3]"
energy_unit = "[kWh/m^3]"
config.x_title = par + " power density " + power_unit
config.y_title = par + " energy density " + energy_unit
#config.show_dots = False
config.logarithmic = True
config.fill = True
#config.show_minor_x_labels = False
config.stroke_style = {'width': 1}
config.style = pygal.style.styles['default'](
label_font_size=12,
stroke_opacity=0,
stroke_opacity_hover=0,
transition='100000000000s ease-in')
xy_chart = pygal.XY(config)
xmin = 10000
ymin = 10000
xmax = 0.001
ymax = 0.001
for tech in techlist:
power_minstring = par + "_power_density_min"
power_maxstring = par + "_power_density_max"
energy_minstring = par + "_energy_density_min"
energy_maxstring = par + "_energy_density_max"
minpowerlink = ''
maxpowerlink = ''
minenergylink = ''
maxenergylink = ''
minpowerlabel = ''
maxpowerlabel = ''
minenergylabel = ''
maxenergylabel = ''
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_minstring).
first().source_id).first() is not None:
minpowerlink = Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_minstring).first().source_id).first().link
minpowerlabel = 'min. power density: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_minstring).
first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_minstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_maxstring).
first().source_id).first() is not None:
maxpowerlink = Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_maxstring).first().source_id).first().link
maxpowerlabel = 'max. power density: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=power_maxstring).
first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=power_maxstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_minstring).
first().source_id).first() is not None:
minenergylink = Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_minstring).first().source_id).first().link
minenergylabel = 'min. energy density: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_minstring)
.first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_minstring).first().source_id).first().
releaseyear)
if Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_maxstring).
first().source_id).first() is not None:
maxenergylink = Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_maxstring).first().source_id).first().link
maxenergylabel = 'max. energy density: ' + Source.query.filter_by(
id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(name=energy_maxstring)
.first().source_id).first().author + ', ' + str(
Source.query.filter_by(id=Parameter.query.filter_by(
technology_name=tech.name).filter_by(
name=energy_maxstring).first().source_id).first().
releaseyear)
xy_chart.add(f"{tech.name}", [{
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
minenergylabel,
'xlink': {
'href': minenergylink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value),
'label':
minpowerlabel,
'xlink': {
'href': minpowerlink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value),
'label':
maxenergylabel,
'xlink': {
'href': maxenergylink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
maxpowerlabel,
'xlink': {
'href': maxpowerlink,
'target': '_blank'
}
}, {
'value':
(Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value),
'label':
minenergylabel,
'xlink': {
'href': minenergylink,
'target': '_blank'
}
}])
if Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value is not None:
xmin = min(
xmin,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_minstring).first().value)
xmax = max(
xmax,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=power_maxstring).first().value)
if Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value is not None:
ymin = min(
ymin,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_minstring).first().value)
ymax = max(
ymax,
Parameter.query.filter_by(technology_name=tech.name).filter_by(
name=energy_maxstring).first().value)
xy_chart.xrange = (10**int(math.floor(math.log10(xmin))),
10**(int(math.floor(math.log10(xmax))) + 1) + 1)
xy_chart.range = (10**int(math.floor(math.log10(ymin))),
10**(int(math.floor(math.log10(ymax))) + 1) + 1)
xy_chart.render()
return xy_chart.render_data_uri()
def identify(self, filename):
self.report['filename'] = os.path.basename(filename)
self.report['filename_absolute'] = filename
self.report['filemimetype'] = utils.file_mimetype(filename)
self.report['filemagic'] = utils.file_magic(filename)
if self.report['filemimetype'] in self.pcapmimetypes:
self.report['filecategory'] = 'CAP'
self.report['filetype'] = 'PCAP'
self.logger.info('Identified %s as type %s (%s)' % (self.report['filename'], self.report['filetype'], self.report['filemimetype']))
else:
self.logger.info('File %s of type %s is not supported in the current version' % (self.report['filename'], self.report['filemimetype']))
return None
self.logger.debug('Calculating file hashes for %s' % (self.report['filename']))
self.report['hashes']['crc32'] = utils.file_hashes(filename, 'crc32')
self.report['hashes']['md5'] = utils.file_hashes(filename, 'md5')
self.report['hashes']['sha1'] = utils.file_hashes(filename, 'sha1')
self.report['hashes']['sha256'] = utils.file_hashes(filename, 'sha256')
self.report['hashes']['sha512'] = utils.file_hashes(filename, 'sha512')
self.report['hashes']['ssdeep'] = utils.file_hashes(filename, 'ssdeep')
self.logger.info('Completed crc32/md5/sha{1,256,512}/ssdeep hash calculations')
if self.config['enable_entropy_compression_stats']:
self.logger.debug('Collecting entropy compression stats for %s' % (self.report['filename']))
stats = utils.entropy_compression_stats(filename)
self.report['filesize'] = stats['filesizeinbytes']
self.report['fileminsize'] = float(stats['minfilesize'])
self.report['filecompressionratio'] = float(stats['compressionratio'])
self.report['fileentropy'] = float(stats['shannonentropy'])
# if entropy falls within the 0 - 1 or 7 - 8 range, categorize as suspicious
if (self.report['fileentropy'] > 0 and self.report['fileentropy'] < 1) or self.report['fileentropy'] > 7:
self.report['fileentropy_category'] = 'SUSPICIOUS'
else:
self.report['fileentropy_category'] = 'NORMAL'
self.logger.info('Completed entropy compression stats collection')
else:
stats = {}
if self.config['enable_bytefreq_histogram']:
self.logger.debug('Generating Byte-Frequency histogram for %s' % (self.report['filename']))
config = Config()
config.x_title = 'Bytes'
config.y_title = 'Frequency'
config.x_scale = .25
config.y_scale = .25
config.width = 900
config.height = 300
config.title_font_size = 9
config.tooltip_font_size = 0
config.tooltip_border_radius = 0
config.no_data_text = ""
config.show_legend = False
config.show_only_major_dots = True
config.human_readable = False
config.show_y_labels = False
config.fill = True
config.style = CleanStyle
bar_chart = pygal.Bar(config)
if 'bytefreqlist' in stats.keys():
bar_chart.add('', stats['bytefreqlist'])
self.report['filebytefreqhistogram'] = bar_chart.render(is_unicode=False)
self.logger.info('Completed Byte-Frequency histogram generation')
# testing and db support to identify visually similar files/sessions
if self.config['enable_file_visualization']:
self.logger.debug('Generating file visualization for %s' % (self.report['filename']))
self.report['filevis_png'] = utils.file_to_pngimage(filename)
self.report['filevis_png_bw'] = utils.file_to_pngimage(filename, enable_colors=False)
self.logger.info('Completed file visualization generation')
return dict(self.report)
# for iDataNum in reversed(range(0, nTotalDataCount)): # # Remove all entries who # if iDataNum % nDataStepCount == 0: # aMajorXData.append( aData[aFields[0]][iDataNum] ) # Import Style # from pygal.style import LightSolarizedStyle # Create Config object from pygal import Config chartCfg = Config() chartCfg.show_legend = True chartCfg.human_readable = True chartCfg.pretty_print=True if bFilledLineChart: chartCfg.fill = True else: chartCfg.fill = False chartCfg.x_scale = 1 chartCfg.y_scale = 1 chartCfg.x_label_rotation = 45 chartCfg.include_x_axis = True chartCfg.show_dots=False if nMaxDataCount > 0: chartCfg.show_minor_x_labels=False chartCfg.x_labels_major_count=nMaxDataCount chartCfg.js = [ 'svg.jquery.js','pygal-tooltips.js' ] # Use script from local # chartCfg.style = LightSolarizedStyle chartCfg.print_values = False chartCfg.print_zeroes = True chartCfg.no_data_text = "All values are 0"
sender = 'Monito admin <*****@*****.**>' receivers = ['*****@*****.**'] config = Config() config.range=(.0001, 5) config.legend_font_size=30 config.tooltip_font_size=30 config.legend_box_size=18 config.title_font_size=30 config.label_font_size=20 config.legend_at_bottom=True config.major_label_font_size=20 config.no_data_text='Fetching data..' config1 = Config() config1.fill=True config1.spacing=50 config1.range=(1, 100) config1.legend_font_size=30 config1.tooltip_font_size=30 config1.legend_box_size=18 config1.title_font_size=30 config1.label_font_size=20 config1.legend_at_bottom=True config1.major_label_font_size=20 config1.no_data_text='Fetching data..' style1 = Style( #background='transparent', #plot_background='transparent', foreground='#53E89B',