def monthly_create_multi_bar(config):
"""
Creates the bar charts that uses data from
multiple months.
Arguments:
config -- configuration dictionary
"""
file_paths = config['file_paths']
year = config['year']
month = config['month']
months = [month_string_format(year, month-2), month_string_format(year, month-1), month_string_format(year, month)]
output_dir = config['output_dir']
ssfiles = map(lambda x: x + 'serverSummary.csv', file_paths)
create_server_summary(ssfiles, config)
bot_data = []
ccfiles = map(lambda x: x + 'C&CServers.csv', file_paths)
for ccf in ccfiles:
with open(ccf) as csv_file:
dreader = csv.DictReader(csv_file)
hold = []
for row in dreader:
if row['ip'] not in hold:
hold.append(row['ip'])
bot_data.append(len(hold))
plot_url = rutil.plotly_bar_chart(months, [(bot_data, 'Botnet (C&C)s')], 'Botnet (C&Cs) security event distribution')
rutil.plotly_download_png(plot_url, output_dir + 'BotCCDis.png')
cc_data = [0,0,0]
with open(ccfiles[2]) as csv_file:
dreader = csv.DictReader(csv_file)
hold = []
for row in dreader:
if row['ip'] not in hold:
if row['channel'] == '-': #HTTP
cc_data[1] += 1
else: #IRC
cc_data[0] += 1
hold.append(row['ip'])
plot_url = rutil.plotly_bar_chart(['IRC', 'HTTP', 'P2P'], [(cc_data, 'Count')], 'Botnet (C&Cs) by communication type')
rutil.plotly_download_png(plot_url, output_dir + 'BotCCType.png')
bot_data = []
bnfiles = map(lambda x: x + 'botnetDailyMax.csv', file_paths)
for bnf in bnfiles:
with open(bnf) as csv_file:
dreader = csv.DictReader(csv_file)
total_count = 0
for row in dreader:
if row['Count'] != '':
total_count += int(row['Count'])
bot_data.append(total_count)
plot_url = rutil.plotly_bar_chart(months,
[(bot_data, 'Botnet (Bots)')],
'Botnet (Bots) security event distribution')
rutil.plotly_download_png(plot_url, output_dir + 'BotBotsDis.png')
def monthly_create_bar_charts(config):
"""
Create bar charts that use data from current month
[N.B. Bar charts that use data from multiple months are generated
in monthly_create_multi_bar]
Arguments:
config -- configuration dictionary
"""
print('Creating bar charts...')
print(' Downloading bar charts...')
bar_chart_dir = os.path.join(os.getcwd(), config["file_paths"][2])
bar_charts = [('ISPServerAll', 'Top 10 ISPs by server related event types'),
('ISPBotnets', 'Top 10 ISPs by non-server event type'),
('ISPAll', 'Top 10 ISPs for all events')]
for file, title in bar_charts:
shutil.copyfile(bar_chart_dir + file + '.csv', bar_chart_dir + file + 'Pie.csv')
header, data = rutil.read_csv(bar_chart_dir + file + '.csv', max_row=10)
plot_url = rutil.plotly_bar_chart(data[0][:10], zip(data[1:], header[1:]), title, 'stack')
rutil.plotly_download_png(plot_url, config['output_dir'] + file + '.png')
# Create bar charts that use data from multiple months
monthly_create_multi_bar(config)
def create_server_summary(file_paths, config):
"""
Creates the summary bar charts:
Defacement/Phishing/Malware Summary/(URL/IP)
Arguments:
file_paths -- folder paths for the three months of csv files
config -- configuration dictionary
"""
month = config['month']
year = config['year']
months = [month_string_format(year, month-2), month_string_format(year, month-1), month_string_format(year, month)]
output_dir = config['output_dir']
data = []
for file in file_paths:
_, csv_data = rutil.read_csv(file, [1,2,3])
data.append(csv_data)
server_dis_headers = ['Defacement','Phishing','Malware']
server_dis = [[],[],[]]
for i in range(3):
for j in range(3):
server_dis[i].append(data[j][i][1])
plot_url = rutil.plotly_bar_chart(months,
zip(server_dis, server_dis_headers),
'Server Related security events distribution',
bar_mode='stack')
rutil.plotly_download_png(plot_url, output_dir + 'ServerRelated.png')
gen = [(1,'Defacement',config['defce_color']),(2,'Phishing',config['phish_color']),(3,'Malware',config['malwr_color'])]
gen_headers = ['URL','Domain','IP']
gen_data = [[],[],[]]
for index, type, colors in gen:
for i in range(3):
gen_data[i] = []
for j in range(3):
gen_data[i].append(data[j][index-1][i+1])
plot_url = rutil.plotly_bar_chart(months,
zip(gen_data, gen_headers),
type + ' General Statistics', color=colors)
rutil.plotly_download_png(plot_url, output_dir + type + 'Gen.png')
url_ip_headers = ['URL/IP Ratio']
for index, type, colors in gen:
url_data = []
for j in range(3):
url_data.append(round(float(data[j][index-1][1]) / float(data[j][index-1][3]),2))
plot_url = rutil.plotly_bar_chart(months, [(url_data, 'URL/IP Ratio')], type + ' URL/IP Ratio', color=colors)
rutil.plotly_download_png(plot_url, output_dir + type + 'URLIP.png')
def quarterly_create_charts(config):
"""
Generates charts for the quarterly security watch report
using Plotly and Google Charts.
"""
yyqq, year, qrtr, qrtr_label, data_paths = config['params']
output = config['output']
print('Generating Security Watch Report for ' + qrtr_label[4])
print('Creating charts:')
# Lambda function to use qrtr_labels as the labels for the x-axis
qrtr_bar = lambda x,y: rutil.plotly_bar_chart(qrtr_label,x,y)
# Defacement, Phishing and Malware Trend and URL/IP
url_ip_col = [('Defacement', 1, u'網頁塗改',config['defce_color']),
('Phishing', 2, u'釣魚網站',config['phish_color']),
('Malware',3,u'惡意程式寄存',config['malwr_color'])]
for type, index, type_c, clr in url_ip_col:
url_ip_unique_data, url_ip_ratio_data = config[type + '_url_data']
plot_url = rutil.plotly_bar_chart(qrtr_label, zip(url_ip_unique_data, ['Unique URL', 'Unique IP']),
'Trend of ' + type + ' security events',color=clr)
rutil.plotly_download_png(plot_url, output + type + 'UniqueBar.png')
plot_url = rutil.plotly_bar_chart(qrtr_label, [(url_ip_ratio_data[0],'URL/IP ratio')],
'URL/IP ratio of ' + type + ' security events',color=clr)
rutil.plotly_download_png(plot_url, output + type + 'RatioBar.png')
plot_url = rutil.plotly_bar_chart(qrtr_label, zip(url_ip_unique_data, ['唯一網址', '唯一IP']),
type_c + u'安全事件趨勢',color=clr)
rutil.plotly_download_png(plot_url, output + type + 'UniqueBarChi.png')
plot_url = rutil.plotly_bar_chart(qrtr_label, [(url_ip_ratio_data[0],'唯一網址/IP比')],
type_c + u'安全事件唯一網址/IP比',color=clr)
rutil.plotly_download_png(plot_url, output + type + 'RatioBarChi.png')
# Botnet (C&C) Distribution and Trend
cc_data = config['cc_data']
plot_url = rutil.plotly_bar_chart(qrtr_label,
zip(cc_data[0:2], ['IRC','HTTP']),
'Trend and Distribution of Botnet (C&Cs) security events',
'stack')
rutil.plotly_download_png(plot_url, output + 'BotnetCCDisBar.png')
plot_url = rutil.plotly_bar_chart(qrtr_label,
zip(cc_data[0:2], ['IRC','HTTP']),
u'殭屍網絡控制中心安全事件的趨勢和分佈',
'stack')
rutil.plotly_download_png(plot_url, output + 'BotnetCCDisBarChi.png')
plot_url = qrtr_bar([(cc_data[2], 'Botnet C&Cs')],
'Trend of Botnet (C&C) security events')
rutil.plotly_download_png(plot_url, output + 'BotnetCCBar.png')
plot_url = qrtr_bar([(cc_data[2], u'殭屍網絡控制中心(C&C)')],
u'殭屍網絡控制中心(C&C)安全事件趨勢')
rutil.plotly_download_png(plot_url, output + 'BotnetCCBarChi.png')
# Unique Botnet (Bots) Trend
bn_data = config['bn_data']
plot_url = qrtr_bar([(bn_data,'Botnet (Bots)')],
'Trend of Botnet (Bots) security events')
rutil.plotly_download_png(plot_url, output + 'BotnetBotsBar.png')
plot_url = qrtr_bar([(bn_data,u'殭屍電腦')],
u'殭屍網絡(殭屍電腦)安全事件趨勢')
rutil.plotly_download_png(plot_url, output + 'BotnetBotsBarChi.png')
# Top 5 Botnets
top_bn_data, top_bn_name, top_bn_curr = config['top_bn']
plot_url = rutil.plotly_line_chart(qrtr_label,
zip(top_bn_data, top_bn_name),
'Trend of 5 Botnet Families in Hong Kong Network')
rutil.plotly_download_png(plot_url, output + 'BotnetFamTopLine.png')
plot_url = rutil.plotly_line_chart(qrtr_label,
zip(top_bn_data, top_bn_name),
u'五大主要殭屍網絡趨勢')
rutil.plotly_download_png(plot_url, output + 'BotnetFamTopLineChi.png')
# Server-related Events
url_data = config['url_data']
plot_url = rutil.plotly_bar_chart(qrtr_label,
zip(url_data, ['Defacement','Phishing','Malware hosting']),
'Trend and Distribution of server related security events',
'stack')
rutil.plotly_download_png(plot_url, output + 'ServerDisBar.png')
plot_url = rutil.plotly_bar_chart(qrtr_label,
zip(url_data, [u'網頁塗改',u'釣魚網站',u'惡意程式寄存']),
u'與伺服器有關的安全事件的趨勢和分佈',
'stack')
rutil.plotly_download_png(plot_url, output + 'ServerDisBarChi.png')
# Total Events
serv_events = config['serv_events']
plot_url = qrtr_bar([(serv_events, 'Unique security events')],
'Trend of Security events')
rutil.plotly_download_png(plot_url, output + 'TotalEventBar.png')
plot_url = qrtr_bar([(serv_events, u'唯一安全事件')],
u'安全事件趨勢')
rutil.plotly_download_png(plot_url, output + 'TotalEventBarChi.png')
# Botnet Family Pie Chart (Google Charts)
rutil.google_pie_chart([('botnetDailyMax','BotnetFamPie')],
data_paths[len(data_paths) - 1],
output)