def create_server_summary(file_paths, config):
"""
Creates the summary bar charts:
Defacement/Phishing/Malware Summary/(URL/IP)
Arguments:
file_paths -- folder paths for the three months of csv files
config -- configuration dictionary
"""
month = config['month']
year = config['year']
months = [month_string_format(year, month-2), month_string_format(year, month-1), month_string_format(year, month)]
output_dir = config['output_dir']
data = []
for file in file_paths:
_, csv_data = rutil.read_csv(file, [1,2,3])
data.append(csv_data)
server_dis_headers = ['Defacement','Phishing','Malware']
server_dis = [[],[],[]]
for i in range(3):
for j in range(3):
server_dis[i].append(data[j][i][1])
plot_url = rutil.plotly_bar_chart(months,
zip(server_dis, server_dis_headers),
'Server Related security events distribution',
bar_mode='stack')
rutil.plotly_download_png(plot_url, output_dir + 'ServerRelated.png')
gen = [(1,'Defacement',config['defce_color']),(2,'Phishing',config['phish_color']),(3,'Malware',config['malwr_color'])]
gen_headers = ['URL','Domain','IP']
gen_data = [[],[],[]]
for index, type, colors in gen:
for i in range(3):
gen_data[i] = []
for j in range(3):
gen_data[i].append(data[j][index-1][i+1])
plot_url = rutil.plotly_bar_chart(months,
zip(gen_data, gen_headers),
type + ' General Statistics', color=colors)
rutil.plotly_download_png(plot_url, output_dir + type + 'Gen.png')
url_ip_headers = ['URL/IP Ratio']
for index, type, colors in gen:
url_data = []
for j in range(3):
url_data.append(round(float(data[j][index-1][1]) / float(data[j][index-1][3]),2))
plot_url = rutil.plotly_bar_chart(months, [(url_data, 'URL/IP Ratio')], type + ' URL/IP Ratio', color=colors)
rutil.plotly_download_png(plot_url, output_dir + type + 'URLIP.png')
def monthly_create_bar_charts(config):
"""
Create bar charts that use data from current month
[N.B. Bar charts that use data from multiple months are generated
in monthly_create_multi_bar]
Arguments:
config -- configuration dictionary
"""
print('Creating bar charts...')
print(' Downloading bar charts...')
bar_chart_dir = os.path.join(os.getcwd(), config["file_paths"][2])
bar_charts = [('ISPServerAll', 'Top 10 ISPs by server related event types'),
('ISPBotnets', 'Top 10 ISPs by non-server event type'),
('ISPAll', 'Top 10 ISPs for all events')]
for file, title in bar_charts:
shutil.copyfile(bar_chart_dir + file + '.csv', bar_chart_dir + file + 'Pie.csv')
header, data = rutil.read_csv(bar_chart_dir + file + '.csv', max_row=10)
plot_url = rutil.plotly_bar_chart(data[0][:10], zip(data[1:], header[1:]), title, 'stack')
rutil.plotly_download_png(plot_url, config['output_dir'] + file + '.png')
# Create bar charts that use data from multiple months
monthly_create_multi_bar(config)
def quarterly_latex(config):
"""
Compiles the LaTeX report for the Quarterly Security
Watch Report. Generates the LaTeX tables from reading
CSV files; Uses template LaTeX report_qrtr_temp_chi
and report_quarterly_temp and inserts the data
appropriately.
"""
yyqq, year, qrtr, qrtr_label, data_paths = config['params']
output = config['output']
bar_chart_param, pie_chart_param = config['trim_config']
# Top 5 Botnets Table
top_bn_data, top_bn_name, _ = config['top_bn']
table_hdr = ['Name'] + qrtr_label
table_top_bot = ''
table_top_bot += '\\begin{table}[!htbp]\n\\centering\n'
table_top_bot += '\n\\begin{tabular}{llllll} \\hline\n'
table_top_bot += '&'.join(map(lambda x: '\\bf ' + x, table_hdr)) + '\\\\\\hline\n'
rows = map(lambda x,y:x+'&'+'&'.join(y)+'\\\\\n', top_bn_name, top_bn_data)
for row in rows:
table_top_bot += row
table_top_bot += '\\hline\n\\end{tabular}\n\\end{table}\n'
# Generate latex table for Major Botnet Families
headers, data = rutil.read_csv(data_paths[4] + 'botnetDailyMax.csv', [0,1])
_, prev_data = rutil.read_csv(data_paths[3] + 'botnetDailyMax.csv', [0,1])
rank_change = []
pct_change = ['NA'] * 10
for i in range(10):
if data[0][i] == prev_data[0][i]:
rank_change.append('$\\rightarrow$')
elif data[0][i] in prev_data[0][:i]:
rank_change.append('$\\Downarrow$')
elif data[0][i] in prev_data[0][i+1:]:
rank_change.append('$\\Uparrow$')
else:
rank_change.append('NEW')
for i in range(len(prev_data[0])):
for j in range(10):
if prev_data[0][i] == data[0][j]:
new = float(data[1][j])
old = float(prev_data[1][i])
pct_change[j] = str(round((new - old) * 100 / old, 1)) + '\%'
# Major Botnet Families table headers
headers = ['Rank', '$\\Uparrow\\Downarrow$',
'Concerned Bots', 'Number of Unique',
'Changes with']
table_ltx = ''
table_ltx += '\\begin{table}[!htbp]\n\\centering\n'
table_ltx += '\\caption{__CAPTION__}'
table_ltx += '\n\\begin{tabular}{lllll} \\hline\n__HEADERS__\\\\\\hline\n'
# Major Botnet Families table data
for i in range(len(data[0]) if len(data[0]) < 10 else 10):
table_ltx += '&'.join([str(i+1), rank_change[i], data[0][i], data[1][i], pct_change[i]]) + '\\\\\n'
table_ltx += '\\hline\n\\end{tabular}\n\\end{table}\n'
ltx_temp = ''
# Create Chinese and English version of Major Botnet Families
table_ltx_cap_eng = 'Major Botnet Families in Hong Kong Networks'
table_ltx_cap_chi = u'香港網絡內的主要殭屍網絡'
table_ltx_hdr_eng = '&'.join(map(lambda x:'\\bf ' + x,headers)) + '\\\\\n&&& \\bf IP addresses & \\bf previous period\n'
table_ltx_hdr_chi = u'\\bf 排名 & \\bf $\\Uparrow\\Downarrow$ & \\bf 殭屍網絡名稱 & \\bf 唯一IP地址 & \\bf 變化 \n'
table_eng = table_ltx.replace('__HEADERS__', table_ltx_hdr_eng)
table_eng = table_eng.replace('__CAPTION__', table_ltx_cap_eng)
table_chi = table_ltx.replace('__HEADERS__', table_ltx_hdr_chi)
table_chi = table_chi.replace('__CAPTION__', table_ltx_cap_chi)
# Compile Latex report
serv_events = config['serv_events']
with open(output + 'report_quarterly_temp.tex') as f:
ltx_temp = f.read()
fontcfg = ConfigParser.ConfigParser(allow_no_value=True)
fontcfg.read('config.cfg')
f = lambda x: fontcfg.get('font','font_' + x)
ltx_temp = ltx_temp.replace('__FONT_SIZE__',f('size'))
ltx_temp = ltx_temp.replace('__FONT__',f('family'))
ltx_temp = ltx_temp.replace('botnet\\_table', table_eng)
ltx_temp = ltx_temp.replace('QUARTER', qrtr_label[4])
ltx_temp = ltx_temp.replace('UNIQUEEVENTS', serv_events[4])
ltx_temp = ltx_temp.replace('table\\_top\\_bot', table_top_bot)
ltx_temp = ltx_temp.replace('__PIE_CHART__', pie_chart_param)
ltx_temp = ltx_temp.replace('__BAR_CHART__', bar_chart_param)
with open(output + 'SecurityWatchReport.tex', 'w+') as f:
f.write(ltx_temp)
with open(output + 'report_qrtr_temp_chi.tex') as f:
ltx_temp = f.read()
ltx_temp = ltx_temp.replace('UNIQUEEVENTS', serv_events[4])
ltx_temp = ltx_temp.replace('table\\_top\\_bot', table_top_bot)
ltx_temp = ltx_temp.replace('__PIE_CHART__', pie_chart_param)
ltx_temp = ltx_temp.replace('__BAR_CHART__', bar_chart_param)
with open(output + 'SecurityWatchReportChi.tex', 'w+') as f:
f.write(ltx_temp)
with codecs.open(output + 'chiqrtr.tex', mode='w+', encoding='utf-8-sig') as f:
f.write(u'20' + unicode(year) + u'第' + [u'一',u'二',u'三',u'四'][qrtr-1] + u'季度')
with codecs.open(output + 'botnetchitable.tex', mode='w+', encoding='utf-8-sig') as f:
f.write(table_chi)
print('Rendering PDF')
os.chdir(output)
os.system('pdflatex SecurityWatchReport.tex')
os.system('pdflatex SecurityWatchReport.tex') # Second time to replace references and ToC
os.rename('SecurityWatchReport.pdf',
'SecurityWatchReport' + qrtr_label[4] + '.pdf')
print('Report successfully compiled. Exiting now...')
os.system('xelatex SecurityWatchReportChi.tex')
os.system('xelatex SecurityWatchReportChi.tex') # Second time to replace references and ToC
os.rename('SecurityWatchReportChi.pdf',
'SecurityWatchReportChi' + qrtr_label[4] + '.pdf')
print('Report successfully compiled. Exiting now...')
def quarterly_compile_data(config):
"""
Reads CSV files and compiles the data.
"""
yyqq, year, qrtr, qrtr_label, data_paths = config['params']
# URL/IP data
url_data = [[],[],[]]
url_ip_col = [('Defacement', 1),
('Phishing', 2),
('Malware', 3)]
for type, index in url_ip_col:
url_ip_unique_data = [[],[]]
url_ip_ratio_data = [[]]
for d in data_paths:
_, data = rutil.read_csv(d + 'serverSummary.csv', columns=[index])
url_count = data[0][1]
ip_count = data[0][3]
url_ip_ratio = round(float(url_count) / float(ip_count),2)
url_ip_unique_data[0].append(url_count)
url_ip_unique_data[1].append(ip_count)
url_ip_ratio_data[0].append(str(url_ip_ratio))
url_data[index-1] = url_ip_unique_data[0]
config[type + '_url_data'] = (url_ip_unique_data, url_ip_ratio_data)
config['url_data'] = list(url_data)
# Botnet (C&C) Distribution and Trend data
cc_data = [[],[],[]]
for d in data_paths:
_, data = rutil.read_csv(d + 'C&CServers.csv', columns=[0,3])
ip_list = []
irc_count = 0
http_count = 0
for i in range(len(data[0])):
ip = data[0][i]
if ip not in ip_list:
ip_list.append(ip)
if data[1][i] == '-':
http_count += 1
else:
irc_count += 1
cc_data[0].append(str(irc_count))
cc_data[1].append(str(http_count))
cc_data[2].append(str(irc_count+http_count))
config['cc_data'] = cc_data
# Unique Botnet (Bots) Trend
bn_data = []
for d in data_paths:
_, data = rutil.read_csv(d + 'botnetDailyMax.csv', columns=[1])
total_count = 0
for i in range(len(data[0])):
if data[0][i] is not '':
total_count += int(data[0][i])
bn_data.append(total_count)
config['bn_data'] = bn_data
# Top Botnet data
top_bn_data = [[],[],[],[],[]]
top_bn_name = []
top_bn_curr = []
_, data = rutil.read_csv(data_paths[len(data_paths)-1] + 'botnetDailyMax.csv', [0,1])
for i in range(5):
top_bn_name.append(data[0][i])
top_bn_curr.append(data[1][i])
for j in range(4):
_, data = rutil.read_csv(data_paths[j] + 'botnetDailyMax.csv', [0,1])
for i in range(len(data[0])):
index = -1
try:
index = top_bn_name.index(data[0][i])
if index >= 0:
top_bn_data[index].append(data[1][i])
except:
index = -1
for i in range(5):
if len(top_bn_data[i]) <= j:
top_bn_data[i].append('0')
for i in range(5):
top_bn_data[i].append(top_bn_curr[i])
config['top_bn'] = (top_bn_data, top_bn_name, top_bn_curr)
url_data.append(bn_data)
url_data.append(cc_data[2])
config['serv_events'] = reduce(rutil.sum_array, url_data)