def test_degree_analysis_on_nodes(self, log_data, nicks, nick_same_list):
update_expected_output_directory(log_data)
message_number_graph = network.message_number_graph(
log_data, nicks, nick_same_list)
message_time_graph = network.message_time_graph(
log_data, nicks, nick_same_list)
nick_change_graph = user.nick_change_graph(log_data)
degree_anal_message_number = network.degree_analysis_on_graph(
message_number_graph)
degree_anal_message_time = network.degree_analysis_on_graph(
message_time_graph)
degree_anal_nick_change = network.degree_analysis_on_graph(
nick_change_graph)
expected_analysis_msg_number = []
expected_analysis_msg_time = []
expected_analysis_nick_change = []
unjson('degree_anal_message_number.json', expected_analysis_msg_number)
unjson('degree_anal_message_time.json', expected_analysis_msg_time)
unjson('degree_anal_nick_change.json', expected_analysis_nick_change)
self.assertDictEqual(degree_anal_message_number,
expected_analysis_msg_number[0],
msg=None)
self.assertDictEqual(degree_anal_message_time,
expected_analysis_msg_time[0],
msg=None)
self.assertDictEqual(degree_anal_nick_change,
expected_analysis_nick_change[0],
msg=None)
def test_degree_distribution_multi_channel(self):
log_data = reader.linux_input(self.log_data_dir, ["ALL"],
self.start_date, self.end_date)
expected_result_CC_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/CC_degree_curve_fit')
expected_result_CU_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/CU_degree_curve_fit')
expected_result_UU_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/UU_degree_curve_fit')
nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(
log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(
nicks, nick_same_list, channels_for_user, nick_channel_dict,
nicks_hash, channels_hash)
degree_anal_message_number_CC = network.degree_analysis_on_graph(
dict_out["CC"]["graph"], directed=False)
degree_anal_message_number_UU = network.degree_analysis_on_graph(
dict_out["UU"]["graph"], directed=False)
degree_anal_message_number_CU = network.degree_analysis_on_graph(
dict_out["CU"]["graph"], directed=False)
Y = degree_anal_message_number_CU["degree"]["raw_for_vis"][1:]
data = [(i, Y[i]) for i in range(len(Y))]
CU_truncated, cutoff = channel.truncate_table(data, 0.5)
CU_T = [data[1] for data in list(CU_truncated)]
expected_output_CC_degree_curve_fit = vis.generate_log_plots(
degree_anal_message_number_CC["degree"]["raw_for_vis"],
self.current_directory, "CC_degree_curve_fit")
expected_output_CU_degree_curve_fit = vis.generate_log_plots(
CU_T, self.current_directory, "CU_degree_curve_fit")
expected_output_UU_degree_curve_fit = vis.generate_log_plots(
degree_anal_message_number_UU["degree"]["raw_for_vis"],
self.current_directory, "UU_degree_curve_fit")
os.remove(self.current_directory + "/CC_degree_curve_fit" + ".png")
os.remove(self.current_directory + "/CU_degree_curve_fit" + ".png")
os.remove(self.current_directory + "/UU_degree_curve_fit" + ".png")
self.assertEqual(expected_result_CC_degree_curve_fit,
expected_output_CC_degree_curve_fit)
self.assertEqual(expected_result_CU_degree_curve_fit,
expected_output_CU_degree_curve_fit)
self.assertEqual(expected_result_UU_degree_curve_fit,
expected_output_UU_degree_curve_fit)
def box_plot_for_degree(log_directory, output_directory, channel_name):
cutoff = 0
for channel_name_iter in channel_name:
out_degree_fit_parameters = np.zeros((12, 4))
in_degree_fit_parameters = np.zeros((12, 4))
total_degree_fit_parameters = np.zeros((12, 4))
for month in range(1, 13):
log_data = reader.linux_input(log_directory, channel_name_iter,
"2013-" + str(month) + "-1",
"2013-" + str(month) + "-31")
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
message_number_graph = network.message_number_graph(
log_data, nicks, nick_same_list, False)
degree_anal_message_number = network.degree_analysis_on_graph(
message_number_graph)
out_degree_fit_parameters[month - 1] = vis.generate_log_plots(
degree_anal_message_number["out_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
in_degree_fit_parameters[month - 1] = vis.generate_log_plots(
degree_anal_message_number["in_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
total_degree_fit_parameters[month - 1] = vis.generate_log_plots(
degree_anal_message_number["total_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
parameters = ['slope', 'intercept', 'r_square']
for para_ind in range(len(parameters)):
vis.box_plot(
out_degree_fit_parameters[:, para_ind], output_directory,
"out_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
vis.box_plot(
in_degree_fit_parameters[:, para_ind], output_directory,
"in_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
vis.box_plot(
total_degree_fit_parameters[:, para_ind], output_directory,
"total_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([out_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "out_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([in_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "in_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([total_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "total_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
def test_message_exchange_network(self):
log_data = reader.linux_input(self.log_data_dir, ["#kubuntu-devel"],
self.start_date, self.end_date)
expected_result = util.load_from_disk(
self.current_directory +
'/data/output/degree_anal_message_number_graph_kubuntu-devel')
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
message_number_graph = network.message_number_graph(
log_data, nicks, nick_same_list, False)
expected_output = network.degree_analysis_on_graph(
message_number_graph)
self.assertEqual(expected_result, expected_output)
def test_degree_analysis_on_nodes(self, log_data, nicks, nick_same_list):
update_expected_output_directory(log_data)
message_number_graph = network.message_number_graph(log_data, nicks, nick_same_list)
message_time_graph = network.message_time_graph(log_data, nicks, nick_same_list)
nick_change_graph = user.nick_change_graph(log_data)
degree_anal_message_number = network.degree_analysis_on_graph(message_number_graph)
degree_anal_message_time = network.degree_analysis_on_graph(message_time_graph)
degree_anal_nick_change = network.degree_analysis_on_graph(nick_change_graph)
expected_analysis_msg_number = []
expected_analysis_msg_time = []
expected_analysis_nick_change = []
unjson('degree_anal_message_number.json', expected_analysis_msg_number)
unjson('degree_anal_message_time.json', expected_analysis_msg_time)
unjson('degree_anal_nick_change.json', expected_analysis_nick_change)
self.assertDictEqual(degree_anal_message_number, expected_analysis_msg_number[0], msg=None)
self.assertDictEqual(degree_anal_message_time, expected_analysis_msg_time[0], msg=None)
self.assertDictEqual(degree_anal_nick_change, expected_analysis_nick_change[0], msg=None)
def test_degree_analysis_on_graph(self):
directed_graph = util.load_from_disk(current_directory +
'/data/directed_graph')
undirected_graph = util.load_from_disk(current_directory +
'/data/undirected_graph')
directed_deg_analysis_expected = util.load_from_disk(
current_directory + '/data/directed_deg_analysis_result')
undirected_deg_analysis_expected = util.load_from_disk(
current_directory + '/data/undirected_deg_anlaysis_result')
capturedOutput = StringIO.StringIO()
sys.stdout = capturedOutput
directed_deg_analysis = network.degree_analysis_on_graph(
directed_graph, directed=True)
undirected_deg_analysis = network.degree_analysis_on_graph(
undirected_graph, directed=False)
sys.stdout = sys.__stdout__
capturedOutput.close()
self.assertEqual(directed_deg_analysis, directed_deg_analysis_expected)
self.assertEqual(undirected_deg_analysis,
undirected_deg_analysis_expected)
def test_degree_distribution_message_exchange_network(self):
degree_type = ["out_degree", "in_degree", "total_degree"]
log_data = reader.linux_input(self.log_data_dir, ["#kubuntu-devel"],
self.start_date, self.end_date)
expected_result = util.load_from_disk(
self.current_directory +
'/data/output/message_exchange_network_curve_fit')
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
message_number_graph = network.message_number_graph(
log_data, nicks, nick_same_list, False)
degree_anal_message_number = network.degree_analysis_on_graph(
message_number_graph)
expected_output = {}
for dtype in degree_type:
expected_output[dtype] = vis.generate_log_plots(
degree_anal_message_number[dtype]["raw_for_vis"],
self.current_directory, "#kubuntu-devel" + dtype)
os.remove(self.current_directory + "/#kubuntu-devel" + dtype +
".png")
self.assertEqual(expected_result, expected_output)
channel_name = config.CHANNEL_NAME
starting_date = config.STARTING_DATE
ending_date = config.ENDING_DATE
output_directory = config.OUTPUT_DIRECTORY
# ============== INPUT==================
log_data = reader.linux_input(log_directory, channel_name, starting_date,
ending_date)
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
# ============== ANALYSIS =============
message_number_graph = network.message_number_graph(log_data, nicks,
nick_same_list, False)
message_number_graph_day_list = network.message_number_graph(
log_data, nicks, nick_same_list, True)
degree_anal_message_numder = network.degree_analysis_on_graph(
message_number_graph)
message_time_graph_list = network.message_time_graph(log_data, nicks,
nick_same_list, True)
message_time_graph = network.message_time_graph(log_data, nicks,
nick_same_list, False)
out_degree_node_number, in_degree_node_number, total_degree_node_number = network.degree_node_number_csv(
log_data, nicks, nick_same_list)
nick_change_graph_list = user.nick_change_graph(log_data, True)
bin_matrix, total_messages = network.message_number_bins_csv(
log_data, nicks, nick_same_list)
conv_len, conv_ref_time = channel.conv_len_conv_refr_time(
log_data, nicks, nick_same_list)
resp_time = channel.response_time(log_data, nicks, nick_same_list)
user.keywords_clusters(log_data, nicks, nick_same_list)
network.degree_analysis_on_graph(message_number_graph)
channel_name = config.CHANNEL_NAME
starting_date = config.STARTING_DATE
ending_date = config.ENDING_DATE
output_directory = config.OUTPUT_DIRECTORY
degree_type = ["out_degree", "in_degree", "total_degree"]
presence_type = ["CC", "UU", "CU"]
# ============== INPUT==================
log_data = reader.linux_input(log_directory, channel_name, starting_date, ending_date)
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
# ============== ANALYSIS =============
message_number_graph = network.message_number_graph(log_data, nicks, nick_same_list, False)
degree_anal_message_number = network.degree_analysis_on_graph(message_number_graph)
bin_matrix, total_messages = network.message_number_bins_csv(log_data, nicks, nick_same_list)
data = [[i for i in range(len(bin_matrix[0]))]]
data.append([sum(i) for i in zip(*bin_matrix)])
default_cutoff = config.CUTOFF_PERCENTILE
percentiles = [0, 1, 5, 10, 20]
for cutoff in percentiles:
config.CUTOFF_PERCENTILE = cutoff
truncated_rt, rt_cutoff_time = channel.response_time(log_data, nicks, nick_same_list, config.CUTOFF_PERCENTILE)
conv_len, conv_ref_time = channel.conv_len_conv_refr_time(log_data, nicks, nick_same_list, rt_cutoff_time, config.CUTOFF_PERCENTILE)
saver.save_csv(conv_len, output_directory, "conv_len-cutoff-" + str(cutoff))
saver.save_csv(truncated_rt, output_directory, "resp_time-cutoff-" + str(cutoff))
saver.save_csv(conv_ref_time, output_directory, "conv_ref_time-cutoff-" + str(cutoff))
from lib.analysis import network, channel, user, community log_directory = config.LOG_DIRECTORY channel_name = config.CHANNEL_NAME starting_date = config.STARTING_DATE ending_date = config.ENDING_DATE output_directory = config.OUTPUT_DIRECTORY # ============== INPUT================== log_data = reader.linux_input(log_directory, channel_name, starting_date, ending_date) nicks, nick_same_list = nickTracker.nick_tracker(log_data) # ============== ANALYSIS ============= message_number_graph = network.message_number_graph(log_data, nicks, nick_same_list, False) message_number_graph_day_list = network.message_number_graph(log_data, nicks, nick_same_list, True) degree_anal_message_numder = network.degree_analysis_on_graph(message_number_graph) message_time_graph_list = network.message_time_graph(log_data, nicks, nick_same_list, True) message_time_graph = network.message_time_graph(log_data, nicks, nick_same_list, False) out_degree_node_number, in_degree_node_number, total_degree_node_number = network.degree_node_number_csv(log_data, nicks, nick_same_list) nick_change_graph_list = user.nick_change_graph(log_data, True) bin_matrix, total_messages = network.message_number_bins_csv(log_data, nicks, nick_same_list) conv_len, conv_ref_time = channel.conv_len_conv_refr_time(log_data, nicks, nick_same_list) resp_time = channel.response_time(log_data, nicks, nick_same_list) user.keywords_clusters(log_data, nicks, nick_same_list) network.degree_analysis_on_graph(message_number_graph) # adjCC_graph, adjCC_membership = community.infomap_igraph(ig_graph=None, net_file_location="/home/rohan/Desktop/adjCC.net") # ============== OUTPUT ================ saver.draw_nx_graph(message_number_graph, output_directory, "message_number_graph")
def box_plot_for_degree(log_directory, output_directory, channel_name,
start_date, end_date):
"""
Correlational : statistical distribution of curve fit parameters generated for degree distribution. The function
takes the given time duration and selects one month at a time for generation of a degree distribution sample. Each
degree distribution sample shall have 3 curve fit parameters namely slope, intercept & r_square. The function collects these parameters
for all the months of the given time duration. The function produces box plot separately for each parameter.
Args:
log_directory(str): path to the location of Logs
output_directory(str): path to the location where the results are to be stored
channel_name(list): channels for which the analysis is to be done.
start_date(datetime): starting date for the logs to be analysed. This has to be the beginning of the month.
end_date(datetime): ending date for which the logs are to be analysed. This has to be the end of the month.
Returns:
null
"""
start_date = start_date.strptime('%Y-%m-%d')
end_date = end_date.strptime('%Y-%m-%d')
cutoff = 0
for channel_name_iter in channel_name:
out_degree_fit_parameters = np.zeros((12, 4))
in_degree_fit_parameters = np.zeros((12, 4))
total_degree_fit_parameters = np.zeros((12, 4))
for dt in rrule(MONTHLY, dtstart=start_date, until=end_date):
last_day_of_the_month = dt + relativedelta(
months=1) - datetime.timedelta(days=1)
# for month in range(1, 13):
log_data = reader.linux_input(
log_directory, channel_name_iter, dt.strftime("%Y-%m-%d"),
last_day_of_the_month.strftime("%Y-%m-%d"))
nicks, nick_same_list = nickTracker.nick_tracker(log_data)
message_number_graph = network.message_number_graph(
log_data, nicks, nick_same_list, False)
degree_anal_message_number = network.degree_analysis_on_graph(
message_number_graph)
out_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots(
degree_anal_message_number["out_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
in_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots(
degree_anal_message_number["in_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
total_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots(
degree_anal_message_number["total_degree"]["raw_for_vis"],
output_directory, channel_name_iter[0])
parameters = ['slope', 'intercept', 'r_square']
for para_ind in range(len(parameters)):
vis.box_plot(
out_degree_fit_parameters[:, para_ind], output_directory,
"out_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
vis.box_plot(
in_degree_fit_parameters[:, para_ind], output_directory,
"in_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
vis.box_plot(
total_degree_fit_parameters[:, para_ind], output_directory,
"total_degree_" + str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([out_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "out_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([in_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "in_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))
saver.save_csv([total_degree_fit_parameters[:, para_ind].tolist()],
output_directory, "total_degree_" +
str(parameters[para_ind]) + "_2013_" +
channel_name_iter[0] + "_cut_" + str(cutoff))