def box_plot_for_degree(log_directory, output_directory, channel_name): cutoff = 0 for channel_name_iter in channel_name: out_degree_fit_parameters = np.zeros((12, 4)) in_degree_fit_parameters = np.zeros((12, 4)) total_degree_fit_parameters = np.zeros((12, 4)) for month in range(1, 13): log_data = reader.linux_input(log_directory, channel_name_iter, "2013-" + str(month) + "-1", "2013-" + str(month) + "-31") nicks, nick_same_list = nickTracker.nick_tracker(log_data) message_number_graph = network.message_number_graph( log_data, nicks, nick_same_list, False) degree_anal_message_number = network.degree_analysis_on_graph( message_number_graph) out_degree_fit_parameters[month - 1] = vis.generate_log_plots( degree_anal_message_number["out_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) in_degree_fit_parameters[month - 1] = vis.generate_log_plots( degree_anal_message_number["in_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) total_degree_fit_parameters[month - 1] = vis.generate_log_plots( degree_anal_message_number["total_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) parameters = ['slope', 'intercept', 'r_square'] for para_ind in range(len(parameters)): vis.box_plot( out_degree_fit_parameters[:, para_ind], output_directory, "out_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( in_degree_fit_parameters[:, para_ind], output_directory, "in_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( total_degree_fit_parameters[:, para_ind], output_directory, "total_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([out_degree_fit_parameters[:, para_ind].tolist()], output_directory, "out_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([in_degree_fit_parameters[:, para_ind].tolist()], output_directory, "in_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([total_degree_fit_parameters[:, para_ind].tolist()], output_directory, "total_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff))
def test_degree_distribution_multi_channel(self): log_data = reader.linux_input(self.log_data_dir, ["ALL"], self.start_date, self.end_date) expected_result_CC_degree_curve_fit = util.load_from_disk( self.current_directory + '/data/output/CC_degree_curve_fit') expected_result_CU_degree_curve_fit = util.load_from_disk( self.current_directory + '/data/output/CU_degree_curve_fit') expected_result_UU_degree_curve_fit = util.load_from_disk( self.current_directory + '/data/output/UU_degree_curve_fit') nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker( log_data, True) dict_out, graph = network.channel_user_presence_graph_and_csv( nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash) degree_anal_message_number_CC = network.degree_analysis_on_graph( dict_out["CC"]["graph"], directed=False) degree_anal_message_number_UU = network.degree_analysis_on_graph( dict_out["UU"]["graph"], directed=False) degree_anal_message_number_CU = network.degree_analysis_on_graph( dict_out["CU"]["graph"], directed=False) Y = degree_anal_message_number_CU["degree"]["raw_for_vis"][1:] data = [(i, Y[i]) for i in range(len(Y))] CU_truncated, cutoff = channel.truncate_table(data, 0.5) CU_T = [data[1] for data in list(CU_truncated)] expected_output_CC_degree_curve_fit = vis.generate_log_plots( degree_anal_message_number_CC["degree"]["raw_for_vis"], self.current_directory, "CC_degree_curve_fit") expected_output_CU_degree_curve_fit = vis.generate_log_plots( CU_T, self.current_directory, "CU_degree_curve_fit") expected_output_UU_degree_curve_fit = vis.generate_log_plots( degree_anal_message_number_UU["degree"]["raw_for_vis"], self.current_directory, "UU_degree_curve_fit") os.remove(self.current_directory + "/CC_degree_curve_fit" + ".png") os.remove(self.current_directory + "/CU_degree_curve_fit" + ".png") os.remove(self.current_directory + "/UU_degree_curve_fit" + ".png") self.assertEqual(expected_result_CC_degree_curve_fit, expected_output_CC_degree_curve_fit) self.assertEqual(expected_result_CU_degree_curve_fit, expected_output_CU_degree_curve_fit) self.assertEqual(expected_result_UU_degree_curve_fit, expected_output_UU_degree_curve_fit)
def test_generate_log_plots(self, data, expected_result): output = vis.generate_log_plots(data, current_dir, "log_plot_test") #delete the plot created os.remove(current_dir + '/log_plot_test.png') assert np.allclose(output, expected_result)
def test_generate_log_plots(self, mock_calc_plot): data = util.load_from_disk(self.test_data_dir + "/vis/degree_msg_number") expected_result = util.load_from_disk(self.test_data_dir + "/vis/out_degree_analysis") mock_calc_plot.return_value = util.load_from_disk(self.test_data_dir + "vis/calc_plot_data") expected_output = vis.generate_log_plots(data, self.test_data_dir, "log_plot_test") self.assertTrue(np.allclose(expected_output, expected_result))
def test_degree_distribution_message_exchange_network(self): degree_type = ["out_degree", "in_degree", "total_degree"] log_data = reader.linux_input(self.log_data_dir, ["#kubuntu-devel"], self.start_date, self.end_date) expected_result = util.load_from_disk( self.current_directory + '/data/output/message_exchange_network_curve_fit') nicks, nick_same_list = nickTracker.nick_tracker(log_data) message_number_graph = network.message_number_graph( log_data, nicks, nick_same_list, False) degree_anal_message_number = network.degree_analysis_on_graph( message_number_graph) expected_output = {} for dtype in degree_type: expected_output[dtype] = vis.generate_log_plots( degree_anal_message_number[dtype]["raw_for_vis"], self.current_directory, "#kubuntu-devel" + dtype) os.remove(self.current_directory + "/#kubuntu-devel" + dtype + ".png") self.assertEqual(expected_result, expected_output)
saver.save_csv([["response_time_cutoff"], [rt_cutoff_time]], output_directory, "rt_cutoff") saver.save_csv([["month", "users", "directed_messages"], ["Jan-2013", len(message_number_graph), int(message_number_graph.size('weight'))]], output_directory, "users_messages") for dtype in degree_type: saver.save_csv(degree_anal_message_number[dtype]["formatted_for_csv"], output_directory, dtype) saver.save_csv(bin_matrix, output_directory, "MessageNumber_binsize_"+str(config.BIN_LENGTH_MINS)) # =============== VIZ =================== message_graph, message_comm = community.infomap_igraph(ig_graph=None, net_file_location= output_directory + 'message_number_graph.net') vis.plot_infomap_igraph(message_graph, message_comm.membership, output_directory, "message") vis.plot_data (data, output_directory, "bins") for dtype in degree_type: slope,intercept,r_square,mse = vis.generate_log_plots(degree_anal_message_number[dtype]["raw_for_vis"], output_directory, channel_name[0] +dtype) saver.save_csv( [["Y","K","R^2", "MSE"], [slope,intercept,r_square,mse]], output_directory, dtype+"-curve-fit") # ============== PRESENCE ACROSS MULTIPLE CHANNELS ============== # Change analysis to all channels in config log_data = reader.linux_input(log_directory, ["ALL"], starting_date, ending_date) nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(log_data, True) dict_out, graph = network.channel_user_presence_graph_and_csv(nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash) saver.save_js_arc(dict_out["CC"]["reducedGraph"], channels_hash, config.OUTPUT_DIRECTORY + "protovis/", "cc.js") for ptype in presence_type: saver.save_csv(dict_out[ptype]["reducedMatrix"],output_directory, "r"+ptype) saver.save_net_nx_graph(dict_out[ptype]["graph"], output_directory, "adj"+ptype) saver.save_net_nx_graph(dict_out[ptype]["reducedGraph"], output_directory, "radj"+ptype) radj_graph, radj_comm = community.infomap_igraph(ig_graph=None, net_file_location= output_directory + 'radj'+ptype+'.net')
def box_plot_for_degree(log_directory, output_directory, channel_name, start_date, end_date): """ Correlational : statistical distribution of curve fit parameters generated for degree distribution. The function takes the given time duration and selects one month at a time for generation of a degree distribution sample. Each degree distribution sample shall have 3 curve fit parameters namely slope, intercept & r_square. The function collects these parameters for all the months of the given time duration. The function produces box plot separately for each parameter. Args: log_directory(str): path to the location of Logs output_directory(str): path to the location where the results are to be stored channel_name(list): channels for which the analysis is to be done. start_date(datetime): starting date for the logs to be analysed. This has to be the beginning of the month. end_date(datetime): ending date for which the logs are to be analysed. This has to be the end of the month. Returns: null """ start_date = start_date.strptime('%Y-%m-%d') end_date = end_date.strptime('%Y-%m-%d') cutoff = 0 for channel_name_iter in channel_name: out_degree_fit_parameters = np.zeros((12, 4)) in_degree_fit_parameters = np.zeros((12, 4)) total_degree_fit_parameters = np.zeros((12, 4)) for dt in rrule(MONTHLY, dtstart=start_date, until=end_date): last_day_of_the_month = dt + relativedelta( months=1) - datetime.timedelta(days=1) # for month in range(1, 13): log_data = reader.linux_input( log_directory, channel_name_iter, dt.strftime("%Y-%m-%d"), last_day_of_the_month.strftime("%Y-%m-%d")) nicks, nick_same_list = nickTracker.nick_tracker(log_data) message_number_graph = network.message_number_graph( log_data, nicks, nick_same_list, False) degree_anal_message_number = network.degree_analysis_on_graph( message_number_graph) out_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots( degree_anal_message_number["out_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) in_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots( degree_anal_message_number["in_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) total_degree_fit_parameters[dt.month - 1] = vis.generate_log_plots( degree_anal_message_number["total_degree"]["raw_for_vis"], output_directory, channel_name_iter[0]) parameters = ['slope', 'intercept', 'r_square'] for para_ind in range(len(parameters)): vis.box_plot( out_degree_fit_parameters[:, para_ind], output_directory, "out_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( in_degree_fit_parameters[:, para_ind], output_directory, "in_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( total_degree_fit_parameters[:, para_ind], output_directory, "total_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([out_degree_fit_parameters[:, para_ind].tolist()], output_directory, "out_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([in_degree_fit_parameters[:, para_ind].tolist()], output_directory, "in_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv([total_degree_fit_parameters[:, para_ind].tolist()], output_directory, "total_degree_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff))
len(message_number_graph), int(message_number_graph.size('weight')) ]], output_directory, "users_messages") degree_anal_message_number = network.degree_analysis_on_graph( message_number_graph) print("msg exchange graph node degree analysis completed at: ", datetime.datetime.now(), file=exec_times_file) exec_times_file.flush() for dtype in degree_type: saver.save_csv(degree_anal_message_number[dtype]["formatted_for_csv"], output_directory, dtype) slope, intercept, r_square, mse = vis.generate_log_plots( degree_anal_message_number[dtype]["raw_for_vis"], output_directory, "slackware-" + dtype) saver.save_csv( [["Y", "K", "R^2", "MSE"], [slope, intercept, r_square, mse]], output_directory, dtype + "-curve-fit") print("msg exchange graph node degree analysis saved at: ", datetime.datetime.now(), file=exec_times_file) exec_times_file.flush() del message_number_graph, degree_anal_message_number del slope, intercept, r_square, mse gc.collect() print("msg exchange with cutoff=0 gc completed at: ", datetime.datetime.now(),