def get_result(self, request, graph_type, state_Abrv):
# define variables
usResult = request['usResult']
stateResult = request['stateResult']
type = graph_type
# Display pie chart
if (type == "pieChart"):
value = int(usResult) - int(stateResult)
chart = pieChart(name=type,
color_category='category20c',
height=450,
width=450)
xdata = ["United States", state_Abrv]
ydata = [int(value), int(stateResult)]
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
# else display bar chart
else:
chart = multiBarChart(width=500, height=400, x_axis_format=None)
xdata = ["United States", state_Abrv]
ydata = [usResult, stateResult]
chart.add_serie(name="", y=ydata, x=xdata)
chart.show_legend = False
# convert charts into html content (and JavaScript)
chart.buildcontent()
result = chart.htmlcontent
return result
def test_pieChart(self):
"""Test Pie Chart"""
type = "pieChart"
chart = pieChart(name=type,
color_category='category20c',
height=400,
width=400)
xdata = [
"Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry",
"Pineapple"
]
color_list = [
'orange', 'yellow', '#C5E946', '#95b43f', 'red', '#FF2259',
'#F6A641'
]
extra_serie = {
"tooltip": {
"y_start": "",
"y_end": " cal"
},
"color_list": color_list
}
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
def test_donutPieChart(self):
"""Test Donut Pie Chart"""
type = "pieChart"
chart = pieChart(name=type, height=400, width=400, donut=True, donutRatio=0.2)
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
def test_pieChart(self):
"""Test Pie Chart"""
type = "pieChart"
chart = pieChart(name=type, height=400, width=400)
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
def function_to_graph(x_axis, y_axis, title):
chart = pieChart(name=title, color_category='category20c', height=450, width=450)
xdata = x_axis
ydata = y_axis
extra_serie = {"tooltip": {"y_start": "", "y_end": "tweets"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
return chart.htmlcontent
def test_pieChart(self):
"""Test Pie Chart"""
type = "pieChart"
chart = pieChart(name=type, color_category='category20c', height=400, width=400)
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
def test_pieChart(self):
"""Test Pie Chart"""
type = "pieChart"
chart = pieChart(name=type, color_category='category20c', height=400, width=400)
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
color_list = ['orange', 'yellow', '#C5E946', '#95b43f', 'red', '#FF2259', '#F6A641']
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}, "color_list": color_list}
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
def get_chart(freq):
chart_type = 'pieChart'
chart = pieChart(name=chart_type, color_category='category20c', height=450, width=450)
extra_serie = {"tooltip": {"y_start": "", "y_end": " %"}}
xdata,ydata=zip(*freq.items())
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
# text_white="d3.selectAll('#pieChart text').style('fill', 'white');"
# chart.add_chart_extras(text_white)
chart.buildcontent()
return chart.htmlcontent
def pie_chart(labels, data, label=""):
chart = nvd3.pieChart(name="",
color_category='category20c',
height=450, width=450)
chart.jquery_on_ready = True
#Add the serie
extra_serie = {"tooltip": {"y_start": "", "y_end": label}}
chart.add_serie(y=data, x=labels, extra=extra_serie)
chart.buildcontent()
# assuming name is hardcoded to ""
return js_extract(chart.htmlcontent).replace("# svg", "#importances_graph")
def function_to_graph(x_axis, y_axis, title):
chart = pieChart(name=title,
color_category='category20c',
height=450,
width=450)
xdata = x_axis
ydata = y_axis
extra_serie = {"tooltip": {"y_start": "", "y_end": "tweets"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
return chart.htmlcontent
def get_chart(freq):
chart_type = 'pieChart'
chart = pieChart(name=chart_type,
color_category='category20c',
height=450,
width=450)
extra_serie = {"tooltip": {"y_start": "", "y_end": " %"}}
xdata, ydata = zip(*freq.items())
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
# text_white="d3.selectAll('#pieChart text').style('fill', 'white');"
# chart.add_chart_extras(text_white)
chart.buildcontent()
return chart.htmlcontent
def pie():
type = 'pieChart'
chart = pieChart(name=type,
color_category='category20c',
height=450,
width=450)
xdata = [
"Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"
]
ydata = [3, 4, 0, 1, 5, 7, 3]
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
return chart.htmlcontent
def allcounty(set_2, allpeople, html, county):
type = 'pieChart'
chart1 = pieChart(name=type, color_category='category20c', height=1100, width=900)
set_1 = ["พิการทางสายตา", "พิการทางการได้ยิน", "ทางการเคลื่อนไหวหรือทางร่างกาย", "ทางจิตใจหรือพฤติกรรม", "ทางสติปัญญา", "ทางการเรียนรู้", "ทางออทิสติก", "พิการมากกว่า 1 ประเภท", "ไม่ระบุ"]
set_2 = set_2
extra_serie = {"tooltip": {"y_start": "", "y_end": " person"}}
chart1.add_serie(y=set_2, x=set_1, extra=extra_serie)
chart1.buildhtml()
file = codecs.open(html, "w+", "utf-8")
p_title = "<body background=bg2.jpg><div align=center>จำนวนคนพิการทั้งหมดที่มีบัตรประจำตัวคนพิการจังหวัด"+county+"ทั้งหมดตั้งแต่ปี พ.ศ. 2537 ถึง พ.ศ. 2558 จำนวน "+str(allpeople)+" คน<br>"+" "+"<input type=button onclick=window.location.href='index.html' value=Home></div></body>"
file.write(p_title)
file.write("<br>")
file.write(chart1.htmlcontent)
file.close()
def hello():
crimes = list(csv.reader(open('noCrimeOutput.txt', 'rb'), delimiter='\t'))
chart = pieChart(name='pieChart', color_category='category20c', height=800, width=800)
xdata = []
ydata = []
for crime in crimes:
xdata.append(crime[0])
ydata.append(crime[1])
extra_serie = {"tooltip": {"y_start": "", "y_end": " counts"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
return chart.htmlcontent
def piechart(set_, maxx, name, name_type):
"""create a piechart by list"""
type = 'pieChart'
chart3 = pieChart(name=type, color_category='category20c', height=1100, width=900)
set_1 = ["เพศชาย", "เพศหญิง"]
set_2 = set_
extra_serie = {"tooltip": {"y_start": "", "y_end": " person"}}
chart3.add_serie(y=set_2, x=set_1, extra=extra_serie)
chart3.buildhtml()
file = codecs.open(name, "w+", "utf-8")
p_title = "<body background=bg2.jpg><div align=center>จำนวนคนพิการ"+name_type+"แต่ละประเภทที่มีบัตรประจำตัวคนพิการในประเทศไทยทั้งหมดตั้งแต่ปี พ.ศ. 2537 ถึง พ.ศ. 2558 จำนวน "+str(maxx)+" คน<br>"+" "+"<input type=button onclick=window.location.href='index.html' value=Home></div></body>"
file.write(p_title)
file.write("<br>")
file.write(chart3.htmlcontent)
file.close()
def d3js_htmls(self):
""" visualize data using D3.js(wrapped by python-nvd3)
"""
global html_lineChart, html_pieChart
if self.vars_dict.get('LINE_CHART'):
chart_name = self.vars_dict.get('TABULATE_FILE')
height = 768
width = 1366
line_chart = lineChart(name='[line]'+chart_name,
height=height,
width=width)
line_data = self.get_tabulate_data(self.vars_dict.get('LAST_DAYS'))
# x_data = [each[0].strip() for each in line_data][1:]
# x_data = map(lambda x: mktime(datetime.strptime(x, '%Y-%m-%d').timetuple()), x_data)
x_data = list(range(-30, 0))
name_list = line_data[0][1:]
y_data_list = []
for i in range(1, len(line_data[0])):
y_data_list.append([each[i] for each in line_data[1:]])
extra_serie = {
'tooltip': {
'y_start': '',
'y_end': 'counts'
}
}
for name, y_data in zip(name_list, y_data_list):
line_chart.add_serie(y=y_data, x=x_data, name=name, extra=extra_serie)
line_chart.buildhtml()
html_lineChart = line_chart.htmlcontent
if self.vars_dict.get('PIE_CHART'):
rows = self.res_counts
xdata, ydata = zip(*rows)
extra_serie = {
'tooltip': {
'y_start': '',
'y_end': 'counts'
}
}
chart_name = self.vars_dict.get('FILE_PATH')
height = 768
width = 1366
pie_chart = pieChart(name='[pie]'+chart_name, color_category='category20c', height=height, width=width)
pie_chart.set_containerheader("\n\n<h2>" + chart_name + "</h2>\n\n")
pie_chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
pie_chart.buildhtml()
html_pieChart = pie_chart.htmlcontent
return html_lineChart, html_pieChart
def piechart(set_, maxx, name, name_type):
"""create a piechart by list"""
type = 'pieChart'
chart3 = pieChart(name=type, color_category='category20c', height=1100, width=900)
set_1 = ["กรุงเทพมหานคร", "กาญจนบุรี", "จันทบุรี", "ฉะเชิงเทรา", "ชลบุรี","ชัยนาท", "ตราด", "นครนายก", "นครปฐม", "นนทบุรี", "ปทุมธานี", "ประจวบคีรีขันธ์", "ปราจีนบุรี", "พระนครศรีอยุธยา", "เพชรบุรี", "ระยอง", "ราชบุรี", "ลพบุรี", "สมุทรปราการ", "สมุทรสงคราม", "สมุทรสาคร", "สระแก้ว", "สระบุรี", "สิงห์บุรี", "สุพรรณบุรี", "อ่างทอง", "กาฬสินธุ์", "ขอนแก่น", "ชัยภูมิ",\
"นครพนม", "นครราชสีมา", "บึงกาฬ", "บุรีรัมย์", "มหาสารคาม", "มุกดาหาร", "ยโสธร", "ร้อยเอ็ด", "เลย", "ศรีสะเกษ", "สกลนคร", "สุรินทร์", "หนองคาย", "หนองบัวลำภู", "อำนาจเจริญ", "อุดรธานี", "อุบลราชธานี", "กระบี่", "ชุมพร", "ตรัง", "นครศรีธรรมราช", "นราธิวาส", "ปัตตานี", "พังงา", "พัทลุง", "ภูเก็ต", "ยะลา", "ระนอง", "สงขลา", "สตูล", "สุราษฎร์ธานี",\
"กำแพงเพชร", "เชียงราย", "เชียงใหม่", "ตาก", "นครสวรรค์", "น่าน", "พะเยา", "พิจิตร", "พิษณุโลก", "เพชรบูรณ์", "แพร่", "แม่ฮ่องสอน", "ลำปาง", "ลำพูน", "สุโขทัย", "อุตรดิตถ์", "อุทัยธานี"]
set_2 = set_
extra_serie = {"tooltip": {"y_start": "", "y_end": " person"}}
chart3.add_serie(y=set_2, x=set_1, extra=extra_serie)
chart3.buildhtml()
file = codecs.open(name, "w+", "utf-8")
p_title = "<body background=bg2.jpg><div align=center>จำนวนคนพิการ"+name_type+"เพศชายที่มีบัตรประจำตัวคนพิการทุกจังหวัดทั้งหมดตั้งแต่ปี พ.ศ. 2537 ถึง พ.ศ. 2558 จำนวน "+str(maxx)+" คน<br>"+" "+"<input type=button onclick=window.location.href='index.html' value=Home></div></body>"
file.write(p_title)
file.write("<br>")
file.write(chart3.htmlcontent)
file.close()
def graphic_time_activity_day():
"""
:return:
"""
output_file = open(_time_activity_day, 'w')
_num_week = int(this_week()) - 1
_week = WeekNumber.objects.get(num_week= _num_week)
_act = Activity()
_tp_acts = TypeActivity.objects.values('group').distinct()
# xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
# ydata = [3, 4, 0, 1, 5, 7, 3]
xdata = []
ydata = []
_time = 0
for _type_act in _tp_acts:
_time_week = _act.get_time_activity(_week, _type_act['group'])
if not _time_week is None and _time_week > 0:
xdata.append(_type_act['group'])
ydata.append(float(_time_week))
_time += float(_time_week)
type = 'Closed %s - time activity (%s hours weeks)' %(DayL[int(_time/24)-1], str(_time))
chart = pieChart(name=type, color_category='category20c', height=450, width=450)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
extra_serie = {"tooltip": {"y_start": "", "y_end": " hs"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
output_file.write(chart.htmlcontent)
output_file.close()
# data = {
# 'charttype': charttype,
# 'chartdata': chartdata,
# }
# return render_to_response("graphic.html")
# new_uri = '%s://%s%s%s' % (
# request.is_secure() and 'https' or 'http',
# site.domain,
# urlquote(request.path),
# (request.method == 'GET' and len(request.GET) > 0) and '?%s' % request.GET.urlencode() or ''
# ) http://eikke.com/django-domain-redirect-middleware/
graphic_view(_time_activity_day)
def hello():
crimes = list(csv.reader(open('noCrimeOutput.txt', 'rb'), delimiter='\t'))
chart = pieChart(name='pieChart',
color_category='category20c',
height=800,
width=800)
xdata = []
ydata = []
for crime in crimes:
xdata.append(crime[0])
ydata.append(crime[1])
extra_serie = {"tooltip": {"y_start": "", "y_end": " counts"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
return chart.htmlcontent
def generate_sentiment_chart(topics, name):
chart = pieChart(name=name, color_list=["green", "red"], height=400, width=400)
xdata = ["Positive sentiment", "Negative sentiment"]
ydata = []
total_population = []
for topic in topics:
population = retrieve_by_topic(topic, "posts")
for p in population:
total_population.append(p)
pos, neg = get_sentiment_stats(total_population)
ydata.append(pos)
ydata.append(neg)
extra_serie = {"tooltip": {"y_start": "", "y_end": "%"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
str(chart)
return chart.content
def graph_treatment():
key2 = 'piegraph'
requete="SELECT treatment.treatment_type,count(treatment.treatment_type)FROM chips.treatment,chips.experiment "\
"WHERE experiment.experiment_id = treatment.experiment_id AND experiment.project_id = treatment.project_id group by treatment.treatment_type;"
r, data = getdata(requete)
xdata = []
ydata = []
if len(data) > 0:
for i in range(0, len(data)):
xdata.append(data[i][0])
ydata.append(int(data[i][1]))
dimwith = 800
dimheight = 700
from nvd3 import pieChart
type = key2
chart = pieChart(name=type,
color_category='category20c',
height=dimheight,
width=dimwith)
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
text = chart.htmlcontent
argument1 = """chart.color(d3.scale.category20c().range());\n\n chart.tooltipContent(function(key, y, e, graph) {\n var x = String(key);\n var y = String(y) + \' cal\';\n\n tooltip_str = \'<center><b>\'+x+\'</b></center>\' + y;\n return tooltip_str;\n });\n"""
##$('#piechart svg.nvd3-svg').attr('width')=dimwith+100
argument2 = " "
argument7 = """});\n var datum ="""
argument8 = """});\n chart.legendPosition("right");\n var datum ="""
text = text.replace(argument7, argument8)
text = text.replace(argument1, argument2)
#text=text.replace('return tooltip_str;\n });\n ','return tooltip_str;\n });*\ \n ')
argument3 = "nv.addGraph(function() {\n"
argument4 = "function rungraphpi(){ \n nv.addGraph(function() {\n"
text = text.replace(argument3, argument4)
argument5 = "</script>"
argument6 = " $('#piegraph').ready(function(){ d3.selectAll('.nv-slice').on('click',function(e){ var col=e.data.label; window.location='treatment.html?treatment='+col; });}); } ;\n rungraphpi(); alert(); \n </script>"
text = text.replace(argument5, argument6)
return text
def generate_chart(data, file, type=None):
if not type:
type = 'multiBarHorizontalChart'
xdata, ydata = [], []
html = """
<link href="/home/chcao/cc-dev/gechart/static/nvd3/build/nv.d3.css" rel="stylesheet" type="text/css">
<script src="/home/chcao/cc-dev/gechart/static/d3/d3.js"></script>
<script src="/home/chcao/cc-dev/gechart/static/d3/d3.min.js"></script>
<script src="/home/chcao/cc-dev/gechart/static/nvd3/build/nv.d3.js"></script>
"""
for k, v in data.iteritems():
xdata.append(k)
ydata.append(v)
# piechart
chart = pieChart(name=type, color_category='category20c', height=900, width=1500)
extra_serie = {"tooltip": {"y_start": "Number: ", "y_end": ""}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
write_files(file, chart.htmlcontent, html)
def process_data():
pcapdata = []
for ts, buf in pcap:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
dtstring=datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S:%f') # Convert time to string
#dt=datetime.datetime.strptime(dtstring,'%d-%m-%Y %H:%M:%S:%f') # Convert string time to datetime
try:
if len(tcp.data) > 0:
http = dpkt.http.Request(tcp.data)
headers = http.headers
dest_ip = socket.inet_ntoa(ip.dst)
src_ip = socket.inet_ntoa(ip.src)
dest_host = headers['host']
uri = http.uri
useragent = headers['user-agent']
request = repr(http['body'])
#data = (dtstring, http.method)
data = ("{0},{1},{2},{3},{4},{5},{6}".format(dtstring, src_ip, useragent, dest_host, dest_ip, http.method, request))
pcapdata.append(data)
except:
continue
# Get data from db and save results to a list
#############################
gets = [] # Hold the get requests
posts = [] # Hold the posts
gets_and_posts = []
gets_full_entry = []
gets_with_a_request = []
posts_full_entry = []
total_entries = []
gets_and_posts_no_head = []
alltimes = []
for data in pcapdata:
data = data.split(',')
time = data[0]
useragent = data[2]
request_method = data[5]
request = data[6]
src_ip = data[1]
dest_ip = data[4]
dest_host = data[3]
if request_method == 'GET':
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
gets_with_a_request.append(line)
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
else:
line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method)
alltimes.append(time)
total_entries.append(line)
# Used for counting the GETs and POSTs, for making the pie chart
if request_method == 'GET':
gets_and_posts.append('GET Requests')
gets_and_posts_no_head.append(line)
if request_method == 'POST':
gets_and_posts.append('POSTs')
gets_and_posts_no_head.append(line)
###################################
# Get all the GETs (full line) and save to gets_full_entry
if request_method == 'GET':
gets.append(request_method)
gets_full_entry.append(line)
# Get all the POSTs (full line) and save to posts_full_entry
if request_method == 'POST':
posts.append(request_method)
posts_full_entry.append(line)
html_start = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link href="../../flask/static/bower_components/nvd3/build/nv.d3.min.css" rel="stylesheet" />
<script src="../../flask/static/bower_components/d3/d3.min.js"></script>
<script src="../../flask/static/bower_components/nvd3/build/nv.d3.min.js"></script>
<script src="../../flask/static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
###################################
# For a Pie Chart of GETS and POSTS:
count_request_method = count_stuff(gets_and_posts)
count_of_request_methods = []
temp = []
for key, value in count_request_method.iteritems():
temp = [key,value]
count_of_request_methods.append(temp)
xcategorylist = []
ycategorylist = []
for item in count_of_request_methods:
xcategorylist.append(item[0])
ycategorylist.append(item[1])
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400)
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie)
categorypiechart.buildcontent()
writefile = open('pcapstatic/pcap_category_piechart.html','w')
writefile.write(html_start + categorypiechart.htmlcontent)
writefile = open('pcapstatic/pcap_category_piechart.html','a') # Re-open for appending
header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>"
writefile.write(header)
#for line in gets_with_a_request:
for line in total_entries:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
for line in posts_full_entry:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
writefile.write(html_end)
# Timeseries of GETs vs POSTs with plotly
xtime = []
ytime = []
for item in gets_and_posts_no_head:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[5])
# Time Series
df_timeseries = pd.DataFrame(ytime,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('pcapstatic/pcap_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# Gets and Posts on a timeseries:
xtime = []
yrequest = []
posts_to_dst_ips = []
gets_to_dst_ips = []
total_dst_ips = []
total_src_ips = []
total_dst_hosts = []
for item in total_entries:
item = item.split(',')
time = item[0]
src_ip = item[2]
dst_ip = item[3]
dst_host = item[4]
request = item[5]
if src_ip not in total_src_ips:
total_src_ips.append(src_ip)
if dst_ip not in total_dst_ips:
total_dst_ips.append(dst_ip)
if dst_host not in total_dst_hosts:
total_dst_hosts.append(dst_host)
# Separate the dst_ip's based on GETS and POSTS (for a map)
if request == 'POST':
posts_to_dst_ips.append(dst_ip)
if request == 'GET':
gets_to_dst_ips.append(dst_ip)
# Get the count of POSTS for a timeseries
if request == 'POST':
xtime.append(time)
yrequest.append(1)
else:
xtime.append(time)
yrequest.append(0)
# Create maps of the POSTS vs GETS:
quickmap.ip_map_world('miller-2048x1502-color.jpg',posts_to_dst_ips,'pcapstatic/posts_to_dst_ips.svg')
quickmap.ip_map_world('miller-2048x1502-color.jpg',gets_to_dst_ips,'pcapstatic/gets_to_dst_ips.svg')
df_timeseries = pd.DataFrame(yrequest,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('pcapstatic/getsandposts_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# TO DO: Check out domains with investigate
#######################
# STATS:
number_of_gets = str(len(gets)) + " GET requests seen<br>"
number_of_posts = str(len(posts)) + " POSTS seen<br>"
stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>"
stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>"
stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>"
stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts
writefile = open('pcapstatic/stats.html','w')
writefile.write(stats)
type = "discreteBarChart"
chart = discreteBarChart(name='my graphname', height=400, width=800, jquery_on_ready=True)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
xdata = ["A", "B", "C", "D", "E", "F", "G"]
ydata = [3, 12, -10, 5, 25, -7, 2]
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
output_file.write(chart.htmlcontent)
# ---------------------------------------
type = "pie Chart"
chart = pieChart(name=type, color_category='category20c', height=400,
width=400, jquery_on_ready=True)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
color_list = ['orange', 'yellow', '#C5E946', '#95b43f', 'red', '#FF2259', '#F6A641']
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}, "color_list": color_list}
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
ydata = [3, 4, 2, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent()
output_file.write(chart.htmlcontent)
# ---------------------------------------
name = "lineChart-different-x-axis"
type = "lineChart"
chart = lineChart(name=name, height=400, width=800, x_is_date=False,
<html lang="th">
<head>
<meta charset="utf-8" />
<link href="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css" rel="stylesheet" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js"></script>
<title>2014 Annual Survey of State Government Tax Collections</title>
</head>
<body>
<center><h1>2014 Annual Survey of State Government Tax Collections</h1></center>
"""
pattern = {"tooltip": {"y_start": "", "y_end": " Million USD"}}
chart = multiBarChart(width=1024, height=768, x_axis_format=None)
chart1 = pieChart(name="chart1", color_category="category20c", height=800, width=768)
chart1.add_serie(y=total_taxes, x=states, extra={"tooltip": {"y_start": "", "y_end": " %"}})
chart1.buildcontent()
chart.add_serie(name="chart", y=total_taxes, x=states)
chart.buildcontent()
file.write(headhtml)
file.write("<center><h2>All Tax Collections By States</h2></center>")
file.write("<center>")
file.write(chart1.htmlcontent)
file.write("</center>")
file.write("<center><h2>State Government Tax CollectionsTax Collections By Category</h2></center>")
"""
Examples for Python-nvd3 is a Python wrapper for NVD3 graph library.
NVD3 is an attempt to build re-usable charts and chart components
for d3.js without taking away the power that d3.js gives you.
Project location : https://github.com/areski/python-nvd3
"""
from nvd3 import pieChart
#Open File for test
output_file = open('test_pieChart.html', 'w')
type = "pieChart"
chart = pieChart(name=type, color_category='category20c', height=400, width=400)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
#close Html file
output_file.close()
def gender(self, **kwargs):
"""
Creates gender distribution figures.
Parameters (generated during class initialization)
----------
NVD3 = False. If true, nvd3 interactive figure output.
Output
------
Saves figures to specified directories.
Returns
-------
None
"""
NVD3 = kwargs.get('NVD3', False)
### Removes those users not having the option to fill in edX registration data.
trim_data = self.person[(self.person.registered == 1)
& (self.person.user_id > 156633)]
### Data
gdict = {'f': "$Female$", 'm': "$Male$", 'o': "$Other$"}
glist = ['$Female$', '$Male$']
### Munge and Plot
gender = trim_data.gender.dropna().apply(
lambda x: gdict[x]).value_counts()
#print gender
certs = trim_data[trim_data.certified == 1]
if certs.username.count() > self.mincerts:
certs = certs.gender.dropna().apply(
lambda x: gdict[x]).value_counts()
else:
certs = pd.Series(index=gender.index)
gender = pd.concat([gender, certs],
join='inner',
axis=1,
keys=['$Non-Certified$', '$Certified$'])
gender = 100. * gender / gender.sum()
gender = gender.apply(lambda x: np.round(x, 1))
fig = plt.figure(figsize=(12, 6))
ax1 = fig.add_subplot(1, 1, 1)
gender.ix[glist, :].plot(
ax=ax1,
kind='bar',
color=[xff.colors['neutral'], xff.colors['institute']],
rot=0)
### Plot Details
ax1.set_xticklabels([r'%s' % x for x in glist])
ax1.set_yticklabels(
[r'${0}\%$'.format("%.0f" % (y)) for y in ax1.get_yticks()],
fontsize=30)
ax1.legend(loc=2, prop={'size': 28}, frameon=False)
### Generalized Plotting functions
figsavename = self.figpath + 'gender_distribution_' + self.nickname.replace(
'.', '_')
print figsavename
xff.texify(fig,
ax1,
xlabel=None,
ylabel='Count',
figsavename=figsavename + '.png')
# ### Output JSON Records
# gender.name = 'value'
# gender = gender.reset_index().rename(columns={'index':'label'})
# gender.dropna().to_json(figsavename+'.json',orient='records')
#----------------------------------------------------------------
### NVD3 Interactive http://nvd3.org/
if NVD3:
'http://nvd3.org/examples/pie.html'
X = [x.replace('$', '') for x in gender.index]
Y1 = gender.ix[glist, '$Non-Certified$'].values
Y2 = gender.ix[glist, '$Certified$'].values
#----------------------------------------------------------------
### BAR Chart
from nvd3 import multiBarChart
### Output File
figsavename = self.figpath + 'interactive_gender_distribution_' + self.nickname + '.html'
output_file = open(figsavename, 'w')
print figsavename
title = "Gender Distribution: %s" % self._xd.course_id
charttype = 'multiBarChart'
chart = multiBarChart(name=charttype,
height=350,
x_axis_format="",
y_axis_format=".1f")
chart.set_containerheader("\n\n<h2>" + title + "</h2>\n\n")
nb_element = len(gender.ix[glist, :])
### Series 1
extra_serie1 = {
"tooltip": {
"y_start": "",
"y_end": "%"
},
"color": xff.colors['neutral'],
"format": ".1f"
}
chart.add_serie(name="Participants", y=Y1, x=X, extra=extra_serie1)
### Series 2
extra_serie2 = {
"tooltip": {
"y_start": "",
"y_end": "%"
},
"color": xff.colors['institute'],
"format": ".1f"
}
chart.add_serie(name="Certificate Earners",
y=Y2,
x=X,
extra=extra_serie2)
### Final Output
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
#close Html file
output_file.close()
#----------------------------------------------------------------
### Pie Chart
from nvd3 import pieChart
### Output File
figsavename = self.figpath + 'interactive_gender_piechart_' + self.nickname + '.html'
output_file = open(figsavename, 'w')
print figsavename
title = "Gender Pie Chart: %s" % self._xd.course_id
charttype = 'multiBarChart'
chart = pieChart(name=charttype,
color_category='category20c',
height=400,
width=400)
chart.set_containerheader("\n\n<h2>" + title + "</h2>\n\n")
extra_serie = {"tooltip": {"y_start": "", "y_end": " certified"}}
chart.add_serie(y=Y1, x=X, extra=extra_serie)
### Final Output
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
#close Html file
output_file.close()
return None
chart.buildcontent() # Build HTML content only
output_file.write(chart.htmlcontent)
output_file.write("<p>All key word rank: <br>1st: ['service', 'manage', 'manage', 'manage', 'sale'], <br> 2nd: ['custom', 'work', 'work', 'work', 'custom'],<br>3rd: ['manage', 'service', 'service', 'service', 'product'], <br>4th: ['work', 'detail', 'custom', 'provide', 'service'], <br>5th: ['experiment', 'provide', 'experiment', 'response', 'work'],<br>6th: ['time', 'custom', 'require', 'custom', 'time'],<br>7th: ['sale', 'include', 'detail', 'sale', 'companies'],<br>8th: ['provid', 'require', 'provide', 'detail', 'manage'],<br>9th: ['require', 'companies', 'develop', 'develop', 'experiment'],<br>10th: ['detail', 'develop', 'response', 'experiment', 'detail']</p>")
output_file.write("<p>Interestingly the hightest appeared key words from Job titles and Job descriptions are slightly different, we can see that the hottest job key words are: management, work, customer, service, sales, experienced, provider.</p>")
# ----------------------------Title_11/15 Pie Chart
output_file.write('<h3>Closer look at Top 10 Key Words in Job Title</h3>')
output_file.write("<p>Following pie charts provides a closer look at Top 10 Key Words in Job Title</p>")
type = "Hottest Key Words in Job Title for 11/15"
chart = pieChart(name=type, color_category='category20c', height=350, width=1000)
#chart.set_containerheader("\n\n<h3>" + type + "</h3>\n\n")
#chart.add_chart_extras("\n\n<p>" + 'Here is the description.' + "<p>\n\n")
#chart.header_css = <link rel="stylesheet" href="styles.css">
extra_serie = {"tooltip": {"y_start": "", "y_end": " %"}}
xdata = ['service', 'manage', 'entry', 'custom', 'sale', 'member', 'food', 'crew', 'nurse', 'assist']
ydata = [0.19763513513513514, 0.13429054054054054, 0.11570945945945946, 0.10557432432432433, 0.10050675675675676, 0.0785472972972973, 0.07263513513513513, 0.07179054054054054, 0.06672297297297297, 0.056587837837837836]
ydata = [ydata[i]*100 for i in range(9)]
chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
chart.buildcontent() # Build HTML content only
output_file.write(chart.htmlcontent)
#----------------------------- Title_11/17 Pie Chart
type = "Hottest Key Words in Job Title for 11/17"
def process_data():
# Database details:
connection = MongoClient(MONGODB_HOST, MONGODB_PORT)
collection = connection[DBS_NAME][COLLECTION_NAME]
projects = collection.find(projection=fields)
# Count the number of times a something is seen:
def count_stuff(listofitems):
listofitems_counted = Counter()
for d in listofitems:
t = d.split(',')[0]
listofitems_counted[d] += 1
return(listofitems_counted)
# Get data from db and save results to a list
#############################
timeanddomain = [] # Hold the list of domains for counting
for project in projects:
dateandtime = project['time']
domain = project['domain']
line = ("{0},{1}").format(dateandtime, domain)
timeanddomain.append(line)
# Save the time of domain hits
all_times = [] # Hold the times
for item in timeanddomain:
#print item
time = item.split(',')[0]
all_times.append(time)
# Separate IP addresses from Domains
#############################
temp1 = [] # Hold the domains that are not IPV4
unique_domains = [] # Hold the domains that are not IPV4 and IPV6. domain_count(domain) will use this to count the domains
ip = [] # Hold the IP addresses (IPV4 and IPV6)
empty_items = 0
for item in timeanddomain:
domain = item.split(',')[1]
if valid_ipv4(domain) == True:
ip.append(domain)
if valid_ipv6(domain) == True:
ip.append(domain)
# This next one takes all non-ipv4 items (domains) and adds them to the temp1 list:
if valid_ipv4(domain) == False:
temp1.append(domain)
# This one reads the temp1 list that contains both domains and ipv6 addresses and takes only the non-ipv6 entries and adds them to the unique_domains list:
for domain in temp1:
if valid_ipv6(domain) == False:
unique_domains.append(domain)
count_of_domains = count_stuff(unique_domains) # Count the domains and save as count_of_domains
count_of_times = count_stuff(all_times) # Count the domains and save as count_of_domains
# Turn the count_of_domains into a dictionary
# Used to set a threshold and view domains contacted over or under a certain number
domainslist = []
temp = []
for key, value in count_of_domains.iteritems():
temp = [key,value]
domainslist.append(temp)
timeslist = []
temp = []
for key, value in count_of_times.iteritems():
temp = [key,value]
timeslist.append(temp)
#############################
# Determine normal in a loose manner:
#############################
normal_traffic = []
suspicious_traffic = []
# Print domains with a certain number of visits
for item in domainslist:
domain = item[0]
count = item[1]
# if count is greater than or equal to a number:
if count >= 2:
#print("{0}, {1}".format(domain,count))
normal_traffic.append(item)
# if count is equal to a number:
if count < 5:
#print("{0}, {1}".format(domain,count))
suspicious_traffic.append(item)
# if count is less than or equal to a number:
# if count <= 10:
# print("{0}, {1}".format(domain,count))
######################################################
# Take the suspicious domains and unique them, getting them ready to look at using third party tools:
domain_counts = [] # firstlevel.tld, count
domain_fullrequest_counts = [] # actual request, firstlevel.tld, count
temp = [] # Temporary holder to unique the firstlevel.tld, as used below
for line in suspicious_traffic:
fulldomain = line[0]
dom = tldextract.extract(fulldomain)
if dom.suffix != '':
domain = dom.domain + '.' + dom.suffix # Just the first level domain
# Create a newline, which has the firstlevel domain, the full domain with subdomains and the count
domain_fullrequest_count = "{0},{1},{2}".format(domain,fulldomain,line[1])
if domain in temp:
continue
else:
domain_count = "{0},{1}".format(domain,line[1])
domain_counts.append(domain_count)
domain_fullrequest_counts.append(domain_fullrequest_count)
temp.append(domain) # Unique the domains
#################################
# Check the suspicious traffic in
# OpenDNS Investigate:
token = ()
with open('investigate_token.txt') as API_KEY:
token = API_KEY.read()
token = token.rstrip()
inv = investigate.Investigate(token)
categories_list = []
security_categories_list = []
wl_domains = []
bl_domains = []
not_determined_domains = []
for line in domain_fullrequest_counts:
line = line.split(',')
domain = line[0]
domain_fullrequest_count = "{0},{1},{2}".format(line[0],line[1],line[2])
res = inv.categorization(domain, labels=True)
status = res[domain]['status']
if status == 0:
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
not_determined_domains.append(domain_fullrequest_count)
if status == 1:
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
wl_domains.append(domain_fullrequest_count)
#print domain_fullrequest_count
if status == -1:
bl_domains.append(domain_fullrequest_count)
security_category = res[domain]['security_categories']
if security_category == []:
continue
else:
for value in security_category:
security_categories_list.append(value)
categories_list.append(value)
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
#domain_and_cat = str(domain) + ":" + str(security_category)
#print (domain, security_category)
count_of_categories = count_stuff(categories_list) # Count the categories and save as count_of_categories
# Turn the count_of_categories into a dictionary
category_list = []
temp = []
for key, value in count_of_categories.iteritems():
temp = [key,value]
category_list.append(temp)
count_of_security_categories = count_stuff(security_categories_list) # Count the categories and save as count_of_security_categories
# Turn the count_of_security_categories into a dictionary
security_category_list = []
temp = []
for key, value in count_of_security_categories.iteritems():
temp = [key,value]
security_category_list.append(temp)
#print security_categories_list
#####################################
# CHART GENERATION
#####################################
# Time visit Barchart:
xtimedata = []
ytimedata = []
for item in timeanddomain:
item = item.split(',')
xtimedata.append(item[0])
ytimedata.append(item[1])
'''
timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
timebarchart.add_serie(y=ytimedata, x=xtimedata)
timebarchart.buildhtml()
timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
timebarchart.add_serie(y=df['date'], x=df['counts'])
#timebarchart.add_serie(y=df.index.values, x=df['domain'])
timebarchart.buildhtml()
writefile = open('../flask/templates/timebar.html','w')
writefile.write(timebarchart.htmlcontent)'''
#####################################
#alldomains Barchart:
xdomaindata = []
ydomaindata = []
for item in domainslist:
xdomaindata.append(item[0])
ydomaindata.append(item[1])
### Plotly does this better. All domains visit Barchart:
'''alldomains = discreteBarChart(name='discreteBarChart', height=300, width=1000)
alldomains.add_serie(y=ydomaindata, x=xdomaindata)
alldomains.buildhtml()
writefile = open('../flask/templates/alldomains.html','w')
writefile.write(alldomains.htmlcontent)'''
#####################################
# For a chart of suspicious domains:
xsuspicious = []
ysuspicious = []
for item in domain_counts:
item = item.split(',')
xsuspicious.append(item[0]) # Domains
ysuspicious.append(item[1]) # Count
### Plotly does this better. suspicious visit Barchart:
'''suspiciousbarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
suspiciousbarchart.add_serie(y=ysuspicious, x=xsuspicious)
suspiciousbarchart.buildhtml()
writefile = open('../flask/templates/suspicious_domains.html','w')
writefile.write(suspiciousbarchart.htmlcontent)'''
#####################################
# For a chart of blacklisted domains:
xblacklisted = []
yblacklisted = []
for item in bl_domains:
item = item.split(',')
xblacklisted.append(item[0]) # Domains
yblacklisted.append(item[2]) # Count
### Plotly does this better. Blacklisted Barchart:
'''blacklistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
blacklistedbarchart.add_serie(y=yblacklisted, x=xblacklisted)
blacklistedbarchart.buildhtml()
writefile = open('../flask/templates/blacklisted_domains.html','w')
writefile.write(blacklistedbarchart.htmlcontent)'''
#####################################
# For a chart of whitelisted domains:
xwhitelisted = []
ywhitelisted = []
for item in wl_domains:
item = item.split(',')
xwhitelisted.append(item[0]) # Domains
ywhitelisted.append(item[2]) # Count
### Plotly does this better. whitelisted Barchart:
'''whitelistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
whitelistedbarchart.add_serie(y=ywhitelisted, x=xwhitelisted)
whitelistedbarchart.buildhtml()
writefile = open('../flask/templates/whitelisted_domains.html','w')
writefile.write(whitelistedbarchart.htmlcontent)'''
#####################################
# For a Pie Chart of categories:
xcategorylist = []
ycategorylist = []
for item in category_list:
xcategorylist.append(item[0]) # Categories
ycategorylist.append(item[1]) # Count
### Blacklisted Barchart:
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=1000, width=1000)
xdata = xcategorylist
ydata = ycategorylist
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie)
categorypiechart.buildhtml()
writefile = open('../flask/templates/category_piechart.html','w')
writefile.write(categorypiechart.htmlcontent)
df_categories = pd.DataFrame(data=ycategorylist,index=xcategorylist)
#####################################
# For a Pie Chart of security categories:
xseccategorylist = []
yseccategorylist = []
for item in security_category_list:
xseccategorylist.append(item[0]) # Categories
yseccategorylist.append(item[1]) # Count
### security category piehart:
type = 'pieChart'
security_categorypiechart = pieChart(name=type, color_category='category20c', height=450, width=450)
xdata = xseccategorylist
ydata = yseccategorylist
extra_serie = {"tooltip": {"y_start": "", "y_end": ""}}
security_categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie)
security_categorypiechart.buildhtml()
writefile = open('../flask/templates/security_category_piechart.html','w')
writefile.write(security_categorypiechart.htmlcontent)
#####################################
# For a chart of neutral listed domains:
xneutrallisted = []
yneutrallisted = []
for item in not_determined_domains:
item = item.split(',')
xneutrallisted.append(item[0]) # Domains
yneutrallisted.append(item[2]) # Count
### Plotly does this better. neutral listed Barchart:
'''neutrallistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
neutrallistedbarchart.add_serie(y=yneutrallisted, x=xneutrallisted)
neutrallistedbarchart.buildhtml()
writefile = open('../flask/templates/neutral_domains.html','w')
writefile.write(neutrallistedbarchart.htmlcontent)'''
#################################
stats_number_of_u_domains = str(len(unique_domains)) + " unique domains seen<br>"
stats_number_of_t_domains = str(len(timeanddomain)) + " total domains<br>"
stats_number_of_ips = str(len(ip)) + " total IP addresses<br>"
stats_normaltraffic = ('Normal traffic (domains visited over 3 times): {0}<br>').format(len(normal_traffic))
stats_suspicioustraffic = ('Amount of suspicious traffic domains visited under 2 times): {0}<br>').format(len(suspicious_traffic))
stats_wl = ('<br>Of the suspicious domains (uniqued):<br>Whitelisted domains (OpenDNS): {0}<br>'.format(len(wl_domains)))
stats_bl = ('Blacklisted domains (OpenDNS): {0}<br>'.format(len(bl_domains)))
stats_neutral = ('Neutral domains (OpenDNS): {0}'.format(len(not_determined_domains)))
stats_categories_list = ('Number of categories seen: {0}'.format(len(categories_list)))
stats = stats_number_of_u_domains + stats_number_of_t_domains + stats_number_of_ips + stats_normaltraffic + stats_suspicioustraffic + stats_wl + stats_bl + stats_neutral
writefile = open('../flask/templates/stats.html','w')
writefile.write(stats)
# This one's for plotly:
xtime = []
ytime = []
for item in timeanddomain:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[1])
d = {'date':xtime,'domain':ytime}
df = pd.DataFrame(data=d)
df['counts'] = df.groupby('domain').transform('count')
# For plotly:
df2 = pd.DataFrame(data=ytime,index=xtime)
df2.index = pd.to_datetime(df2.index)
#############################
# HTML for plotly plots:
#############################
html_start = """<html>
<head>
<script src="../static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
# Line plot:
'''plot_html, plotdivid, width, height = _plot_html(df2.iplot(asFigure=True, kind ='scatter', subplots=True, shared_xaxes=True, fill=True, title='Count by day',dimensions=(800,800)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/plottest_scatter.html', 'w')
f.write(html_bar_chart)
f.close()'''
# Suspicious domains
df_suspicious = pd.DataFrame(ysuspicious, xsuspicious)
plot_html, plotdivid, width, height = _plot_html(df_suspicious.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Suspicious Traffic',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/suspicious_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# All domains visited
df_alldomains = pd.DataFrame(ydomaindata, xdomaindata)
plot_html, plotdivid, width, height = _plot_html(df_alldomains.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='All Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/all_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Whitelisted
df_whitelisted = pd.DataFrame(ywhitelisted, xwhitelisted)
plot_html, plotdivid, width, height = _plot_html(df_whitelisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Whitelisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/whitelisted_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Not categorized
df_not_categorized = pd.DataFrame(yneutrallisted, xneutrallisted)
plot_html, plotdivid, width, height = _plot_html(df_not_categorized.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Non-categorized Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/not_categorized_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Blacklisted
df_blacklisted = pd.DataFrame(yblacklisted, xblacklisted)
plot_html, plotdivid, width, height = _plot_html(df_blacklisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Blacklisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/blacklisted_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Time Series
df_timeseries = pd.DataFrame(ytimedata,xtimedata)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
def hello():
events = requests.get(
"https://autism-tracker-server.appspot.com/events").json()
emotion_dict = get_emotion_dict(events)
type = 'pieChart'
moodchart = pieChart(name=type,
color_category='category20c',
height=450,
width=450)
xdata = [
key
for key in ['😁 joy', '😢 sorrow', '😠 anger', '😲 surprise', '😐 neutral']
]
ydata = [emotion_dict[key] for key in emotion_dict]
extra_serie = {"tooltip": {"y_start": "", "y_end": " %"}}
moodchart.add_serie(y=ydata, x=xdata, extra=extra_serie)
moodchart.buildcontent()
stresschart = lineWithFocusChart(name='lineWithFocusChart',
x_is_date=True,
x_axis_format="%d %b %Y")
xdata, ydata_stress, ydata_harm, ydata_physical = get_stress_level(events)
extra_serie = {
"tooltip": {
"y_start": "",
"y_end": " ext"
},
"date_format": "%d %b %Y %H"
}
stresschart.add_serie(name="Stress Level",
y=ydata_stress,
x=xdata,
extra=extra_serie)
stresschart.add_serie(name="Self Harm Level",
y=ydata_harm,
x=xdata,
extra=extra_serie)
stresschart.add_serie(name="Physical Activity Level",
y=ydata_physical,
x=xdata,
extra=extra_serie)
stresschart.buildhtml()
html = """<table>
<tr>
<th>Trigger</th>
<th>Resolution</th>
<th>Additional Notes</th>
</tr>
{0}
</table>"""
items = get_table_items(events)
tr = "<tr>{0}</tr>"
td = "<td>{0}</td>"
subitems = [
tr.format(''.join([td.format(a) for a in item])) for item in items
]
table = html.format("".join(subitems))
return render_template('index.html',
table=table,
moodchart=moodchart,
stresschart=stresschart)
def process_data():
# Database details:
connection = MongoClient(MONGODB_HOST, MONGODB_PORT)
collection = connection[DBS_NAME][COLLECTION_NAME]
projects = collection.find(projection=fields)
#projects = collection.find()
# Get data from db and save results to a list
#############################
gets = [] # Hold the get requests
posts = [] # Hold the posts
gets_and_posts = []
gets_full_entry = []
gets_with_a_request = []
posts_full_entry = []
total_entries = []
gets_and_posts_no_head = []
alltimes = []
for data in projects:
time = data['time']
useragent = data['useragent']
request_method = data['request_method']
request = data['request']
src_ip = data['src_ip']
dest_ip = data['dest_ip']
dest_host = data['dest_host']
if request_method == 'GET':
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
gets_with_a_request.append(line)
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
else:
line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method)
alltimes.append(time)
total_entries.append(line)
# Used for counting the GETs and POSTs, for making the pie chart
if request_method == 'GET':
gets_and_posts.append('GET Requests')
gets_and_posts_no_head.append(line)
if request_method == 'POST':
gets_and_posts.append('POSTs')
gets_and_posts_no_head.append(line)
###################################
# Get all the GETs (full line) and save to gets_full_entry
if request_method == 'GET':
gets.append(request_method)
gets_full_entry.append(line)
# Get all the POSTs (full line) and save to posts_full_entry
if request_method == 'POST':
posts.append(request_method)
posts_full_entry.append(line)
html_start = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link href="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css" rel="stylesheet" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js"></script>
<script src="../../static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
###################################
# For a Pie Chart of GETS and POSTS:
count_request_method = count_stuff(gets_and_posts)
count_of_request_methods = []
temp = []
for key, value in count_request_method.iteritems():
temp = [key,value]
count_of_request_methods.append(temp)
xcategorylist = []
ycategorylist = []
for item in count_of_request_methods:
xcategorylist.append(item[0])
ycategorylist.append(item[1])
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400)
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie)
categorypiechart.buildcontent()
writefile = open('../flask/templates/pcap/pcap_category_piechart.html','w')
writefile.write(html_start + categorypiechart.htmlcontent)
writefile = open('../flask/templates/pcap/pcap_category_piechart.html','a') # Re-open for appending
header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>"
writefile.write(header)
#for line in gets_with_a_request:
for line in total_entries:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
for line in posts_full_entry:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
writefile.write(html_end)
# Timeseries of GETs vs POSTs with plotly
xtime = []
ytime = []
for item in gets_and_posts_no_head:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[5])
# Time Series
df_timeseries = pd.DataFrame(ytime,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/pcap/pcap_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# Gets and Posts on a timeseries:
xtime = []
yrequest = []
posts_to_dst_ips = []
gets_to_dst_ips = []
total_dst_ips = []
total_src_ips = []
total_dst_hosts = []
for item in total_entries:
item = item.split(',')
time = item[0]
src_ip = item[2]
dst_ip = item[3]
dst_host = item[4]
request = item[5]
if src_ip not in total_src_ips:
total_src_ips.append(src_ip)
if dst_ip not in total_dst_ips:
total_dst_ips.append(dst_ip)
if dst_host not in total_dst_hosts:
total_dst_hosts.append(dst_host)
# Separate the dst_ip's based on GETS and POSTS (for a map)
if request == 'POST':
posts_to_dst_ips.append(dst_ip)
if request == 'GET':
gets_to_dst_ips.append(dst_ip)
# Get the count of POSTS for a timeseries
if request == 'POST':
xtime.append(time)
yrequest.append(1)
else:
xtime.append(time)
yrequest.append(0)
# Create maps of the POSTS vs GETS:
quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',posts_to_dst_ips,'../flask/templates/pcap/posts_to_dst_ips.svg')
quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',gets_to_dst_ips,'../flask/templates/pcap/gets_to_dst_ips.svg')
df_timeseries = pd.DataFrame(yrequest,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/pcap/getsandposts_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# TO DO: Check out domains with investigate
#######################
# STATS:
number_of_gets = str(len(gets)) + " GET requests seen<br>"
number_of_posts = str(len(posts)) + " POSTS seen<br>"
stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>"
stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>"
stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>"
stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts
writefile = open('../flask/templates/pcap/stats.html','w')
writefile.write(stats)
def gender(self,**kwargs):
"""
Creates gender distribution figures.
Parameters (generated during class initialization)
----------
NVD3 = False. If true, nvd3 interactive figure output.
Output
------
Saves figures to specified directories.
Returns
-------
None
"""
NVD3 = kwargs.get('NVD3',False)
### Removes those users not having the option to fill in edX registration data.
trim_data = self.person[(self.person.registered==1) & (self.person.user_id>156633)]
### Data
gdict = {'f': "$Female$",'m': "$Male$",'o':"$Other$"}
glist = ['$Female$','$Male$']
### Munge and Plot
gender = trim_data.gender.dropna().apply(lambda x: gdict[x]).value_counts()
#print gender
certs = trim_data[trim_data.certified==1]
if certs.username.count() > self.mincerts:
certs = certs.gender.dropna().apply(lambda x: gdict[x]).value_counts()
else:
certs = pd.Series(index=gender.index)
gender = pd.concat([gender,certs],join='inner',axis=1,keys=['$Non-Certified$','$Certified$'])
gender = 100.*gender/gender.sum()
gender = gender.apply(lambda x: np.round(x,1))
fig = plt.figure(figsize=(12,6))
ax1 = fig.add_subplot(1,1,1)
gender.ix[glist,:].plot(ax=ax1,kind='bar',color=[xff.colors['neutral'],xff.colors['institute']],rot=0)
### Plot Details
ax1.set_xticklabels([r'%s' % x for x in glist])
ax1.set_yticklabels([r'${0}\%$'.format("%.0f" % (y)) for y in ax1.get_yticks()],fontsize=30)
ax1.legend(loc=2,prop={'size':28},frameon=False)
### Generalized Plotting functions
figsavename = self.figpath+'gender_distribution_'+self.nickname.replace('.','_')
print figsavename
xff.texify(fig,ax1,xlabel=None,ylabel='Count',figsavename=figsavename+'.png')
# ### Output JSON Records
# gender.name = 'value'
# gender = gender.reset_index().rename(columns={'index':'label'})
# gender.dropna().to_json(figsavename+'.json',orient='records')
#----------------------------------------------------------------
### NVD3 Interactive http://nvd3.org/
if NVD3:
'http://nvd3.org/examples/pie.html'
X = [ x.replace('$','') for x in gender.index ]
Y1 = gender.ix[glist,'$Non-Certified$'].values
Y2 = gender.ix[glist,'$Certified$'].values
#----------------------------------------------------------------
### BAR Chart
from nvd3 import multiBarChart
### Output File
figsavename = self.figpath+'interactive_gender_distribution_'+self.nickname+'.html'
output_file = open(figsavename, 'w')
print figsavename
title = "Gender Distribution: %s" % self._xd.course_id
charttype = 'multiBarChart'
chart = multiBarChart(name=charttype, height=350, x_axis_format="", y_axis_format=".1f")
chart.set_containerheader("\n\n<h2>" + title + "</h2>\n\n")
nb_element = len(gender.ix[glist,:])
### Series 1
extra_serie1 = {"tooltip": {"y_start": "", "y_end": "%"},
"color":xff.colors['neutral'],
"format":".1f"
}
chart.add_serie(name="Participants", y=Y1, x=X, extra=extra_serie1)
### Series 2
extra_serie2 = {"tooltip": {"y_start": "", "y_end": "%"},
"color":xff.colors['institute'],
"format":".1f"
}
chart.add_serie(name="Certificate Earners", y=Y2, x=X, extra=extra_serie2)
### Final Output
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
#close Html file
output_file.close()
#----------------------------------------------------------------
### Pie Chart
from nvd3 import pieChart
### Output File
figsavename = self.figpath+'interactive_gender_piechart_'+self.nickname+'.html'
output_file = open(figsavename, 'w')
print figsavename
title = "Gender Pie Chart: %s" % self._xd.course_id
charttype = 'multiBarChart'
chart = pieChart(name=charttype, color_category='category20c', height=400, width=400)
chart.set_containerheader("\n\n<h2>" + title + "</h2>\n\n")
extra_serie = {"tooltip": {"y_start": "", "y_end": " certified"}}
chart.add_serie(y=Y1, x=X, extra=extra_serie)
### Final Output
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
#close Html file
output_file.close()
return None
from nvd3 import pieChart
%load_ext rpy2.ipython
xs = %R sort(unique(mtcars$cyl))
ys = %R table(mtcars$cyl)
chart = pieChart(name="Cylinders", color_category='category10', height=500, width=400)
chart.add_serie(x=xs.tolist(), y=ys.tolist())
output_file = open('nvd3-3.html', 'w')
chart.buildhtml()
output_file.write(chart.htmlcontent)
output_file.close()
#Open File for test
output_file = open('test1.html', 'w')
type = "discreteBarChart"
chart = discreteBarChart(name=type, height=400, width=400)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
xdata = ["A", "B", "C", "D", "E", "F", "G"]
ydata = [3, 12, -10, 5, 35, -7, 2]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
type = "pieChart"
chart = pieChart(name=type, height=400, width=400)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
type = "multiBarChart"
chart = multiBarChart(name=type, height=350)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
nb_element = 10
xdata = range(nb_element)
ydata = [random.randint(1, 10) for i in range(nb_element)]
#Open File for test
output_file = open('test1.html', 'w')
type = "discreteBarChart"
chart = discreteBarChart(name=type, height=400, width=400)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
xdata = ["A", "B", "C", "D", "E", "F", "G"]
ydata = [3, 12, -10, 5, 35, -7, 2]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
type = "pieChart"
chart = pieChart(name=type, height=400, width=400)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
xdata = [
"Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"
]
ydata = [3, 4, 0, 1, 5, 7, 3]
chart.add_serie(y=ydata, x=xdata)
chart.buildhtml()
output_file.write(chart.htmlcontent)
#---------------------------------------
type = "multiBarChart"
chart = multiBarChart(name=type, height=350)
chart.set_containerheader("\n\n<h2>" + type + "</h2>\n\n")
nb_element = 10
"<p>All key word rank: <br>1st: ['service', 'manage', 'manage', 'manage', 'sale'], <br> 2nd: ['custom', 'work', 'work', 'work', 'custom'],<br>3rd: ['manage', 'service', 'service', 'service', 'product'], <br>4th: ['work', 'detail', 'custom', 'provide', 'service'], <br>5th: ['experiment', 'provide', 'experiment', 'response', 'work'],<br>6th: ['time', 'custom', 'require', 'custom', 'time'],<br>7th: ['sale', 'include', 'detail', 'sale', 'companies'],<br>8th: ['provid', 'require', 'provide', 'detail', 'manage'],<br>9th: ['require', 'companies', 'develop', 'develop', 'experiment'],<br>10th: ['detail', 'develop', 'response', 'experiment', 'detail']</p>"
)
output_file.write(
"<p>Interestingly the hightest appeared key words from Job titles and Job descriptions are slightly different, we can see that the hottest job key words are: management, work, customer, service, sales, experienced, provider.</p>"
)
# ----------------------------Title_11/15 Pie Chart
output_file.write('<h3>Closer look at Top 10 Key Words in Job Title</h3>')
output_file.write(
"<p>Following pie charts provides a closer look at Top 10 Key Words in Job Title</p>"
)
type = "Hottest Key Words in Job Title for 11/15"
chart = pieChart(name=type,
color_category='category20c',
height=350,
width=1000)
#chart.set_containerheader("\n\n<h3>" + type + "</h3>\n\n")
#chart.add_chart_extras("\n\n<p>" + 'Here is the description.' + "<p>\n\n")
#chart.header_css = <link rel="stylesheet" href="styles.css">
extra_serie = {"tooltip": {"y_start": "", "y_end": " %"}}
xdata = [
'service', 'manage', 'entry', 'custom', 'sale', 'member', 'food', 'crew',
'nurse', 'assist'
]
ydata = [
0.19763513513513514, 0.13429054054054054, 0.11570945945945946,
0.10557432432432433, 0.10050675675675676, 0.0785472972972973,
0.07263513513513513, 0.07179054054054054, 0.06672297297297297,
0.056587837837837836
]