def test_conversation_characteristics_cutoff_1(self, mock_savefig): cutoff = 1 expected_result_conv_len = [ 0.45678248067618998, 1.9431782685053713, 0.0030314547153581827, 3.3570362370587976e-05 ] expected_result_resp_time = [ 0.26876242441433712, 0.38822996056503406, 0.0001355301591146847, 1.4291783519203551e-05 ] expected_result_conv_ref = [ 0.0031066946048193583, 0.0089793356687177077, 2.3338045062882878e-05, 8.2373085916393017e-08, 66 ] truncated_rt, rt_cutoff_time = channel.response_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, cutoff) conv_len, conv_ref_time = channel.conv_len_conv_refr_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, rt_cutoff_time, cutoff) expected_output_conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot( conv_len, self.current_directory, "conv_len_cutoff" + str(cutoff)) expected_output_resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot( truncated_rt, self.current_directory, "resp_time_cutoff" + str(cutoff)) expected_output_conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted( conv_ref_time, self.current_directory, "conv_ref_time_cutoff" + str(cutoff)) self.assertTrue(expected_output_conv_len_curve_fit_parameters, expected_result_conv_len) self.assertTrue(expected_output_resp_time_curve_fit_parameters, expected_result_resp_time) self.assertTrue(expected_output_conv_ref_time_curve_fit_parameters, expected_result_conv_ref)
def test_conversation_characteristics_cutoff_0(self, mock_savefig): cutoff = 0 expected_result_conv_len = [ 0.46025248575487415, 1.8745480617100398, 0.0014084453709393393, 1.8113237256968182e-05 ] expected_result_resp_time = [ 0.26599443483759627, 0.38817554962605116, 0.00012042990450484642, 1.1831364434688785e-05 ] expected_result_conv_ref = [ 0.004067315269095536, 0.01296093837728012, 1.761952783942606e-05, 5.6259486000435723e-08, 60 ] truncated_rt, rt_cutoff_time = channel.response_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, cutoff) conv_len, conv_ref_time = channel.conv_len_conv_refr_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, rt_cutoff_time, cutoff) expected_output_conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot( conv_len, self.current_directory, "conv_len_cutoff" + str(cutoff)) expected_output_resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot( truncated_rt, self.current_directory, "resp_time_cutoff" + str(cutoff)) expected_output_conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted( conv_ref_time, self.current_directory, "conv_ref_time_cutoff" + str(cutoff)) self.assertTrue(expected_output_conv_len_curve_fit_parameters, expected_result_conv_len) self.assertTrue(expected_output_resp_time_curve_fit_parameters, expected_result_resp_time) self.assertTrue(expected_output_conv_ref_time_curve_fit_parameters, expected_result_conv_ref)
def test_conversation_characteristics_cutoff_5(self, mock_savefig): cutoff = 5 expected_result_conv_len = [ 0.44916983849233633, 1.9156349592761313, 0.0048790728866266418, 4.3411589194639429e-05 ] expected_result_resp_time = [ 0.28001731891457893, 0.38845839930487419, 0.00020016446653020847, 2.896185549800808e-05 ] expected_result_conv_ref = [ 0.0019379564807119043, 0.0048940078069499857, 4.2070926227686924e-05, 1.7671895171226243e-07, 90 ] truncated_rt, rt_cutoff_time = channel.response_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, cutoff) conv_len, conv_ref_time = channel.conv_len_conv_refr_time( self.log_data_kubuntu_devel, self.nicks, self.nick_same_list, rt_cutoff_time, cutoff) expected_output_conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot( conv_len, self.current_directory, "conv_len_cutoff" + str(cutoff)) expected_output_resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot( truncated_rt, self.current_directory, "resp_time_cutoff" + str(cutoff)) expected_output_conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted( conv_ref_time, self.current_directory, "conv_ref_time_cutoff" + str(cutoff)) self.assertTrue(expected_output_conv_len_curve_fit_parameters, expected_result_conv_len) self.assertTrue(expected_output_resp_time_curve_fit_parameters, expected_result_resp_time) self.assertTrue(expected_output_conv_ref_time_curve_fit_parameters, expected_result_conv_ref)
def test_exponential_curve_fit_and_plot_x_shifted(self, data, expected_result): output = vis.exponential_curve_fit_and_plot_x_shifted( data, current_dir, "exponential_plot_test_shifted") #delete the plot created os.remove(current_dir + '/exponential_plot_test_shifted.png') assert np.allclose(output, expected_result)
def test_exponential_curve_fit_and_plot_x_shifted( self, mock_curve_fit, mock_curve_func, mock_probability_distribution): data = util.load_from_disk(self.test_data_dir + "/vis/conv_ref_time") expected_result = util.load_from_disk(self.test_data_dir + "/vis/conv_ref_time_fit_params") mock_curve_fit.return_value = util.load_from_disk( self.test_data_dir + "/vis/curve_fit_x_shifted") mock_curve_func.side_effect = self.mock_exponential_curve_func mock_probability_distribution.side_effect = self.mock_generate_probability_distribution expected_output = vis.exponential_curve_fit_and_plot_x_shifted( data, self.test_data_dir, "exponential_plot_test_shifted") # delete the plot created os.remove(self.test_data_dir + '/exponential_plot_test_shifted.png') self.assertTrue(np.allclose(expected_output, expected_result))
for i in range(len(nick_change_graph_list)): saver.draw_nx_graph(nick_change_graph_list[i], output_directory, "ncg" + str(i + 1)) saver.draw_nx_graph(message_number_graph, output_directory, "mnagg") saver.draw_nx_graph(message_time_graph, output_directory, "mtgagg") saver.save_csv(conv_len, output_directory, "conv_len") saver.save_csv(resp_time, output_directory, "resp_time") saver.save_csv(conv_ref_time, output_directory, "conv_ref_time") # =============== VIZ =================== conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot( conv_len, 20, output_directory, "conv_len") resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot( resp_time, 20, output_directory, "resp_time") conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted( conv_ref_time, 30, output_directory, "conv_ref_time") # vis.plot_infomap_igraph(adjCC_graph, adjCC_membership, output_directory, "adjCC_infomaps") # vis.generate_log_plots(9, out_degree_node_number, output_directory, channel_name[0] +"OUT"+ starting_date + ending_date) # ============== VALIDATION ============== validate.validate_RT_RL_CRT( conv_len_curve_fit_parameters, [[10.5, 10.6], [2.12, 2.32], [0, 0.2], [0, 0.0002]], "conv_len") validate.validate_RT_RL_CRT( resp_time_curve_fit_parameters, [[0.3, 10.4], [10.3, 30.4], [-0.002, 0.002], [0, 0.002]], "resp_time") validate.validate_RT_RL_CRT( conv_ref_time_curve_fit_parameters, [[10.05, 10.1], [0.1, 0.2], [0.02, 0.04], [0, 0.0002], [9, 11]], "conv_ref_time")
data = [[i for i in range(len(bin_matrix[0]))]] data.append([sum(i) for i in zip(*bin_matrix)]) default_cutoff = config.CUTOFF_PERCENTILE percentiles = [0, 1, 5, 10, 20] for cutoff in percentiles: config.CUTOFF_PERCENTILE = cutoff truncated_rt, rt_cutoff_time = channel.response_time(log_data, nicks, nick_same_list, config.CUTOFF_PERCENTILE) conv_len, conv_ref_time = channel.conv_len_conv_refr_time(log_data, nicks, nick_same_list, rt_cutoff_time, config.CUTOFF_PERCENTILE) saver.save_csv(conv_len, output_directory, "conv_len-cutoff-" + str(cutoff)) saver.save_csv(truncated_rt, output_directory, "resp_time-cutoff-" + str(cutoff)) saver.save_csv(conv_ref_time, output_directory, "conv_ref_time-cutoff-" + str(cutoff)) conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot(conv_len, output_directory, "conv_len_cutoff" + str(cutoff)) resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot(truncated_rt, output_directory, "resp_time_cutoff" + str(cutoff)) conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted(conv_ref_time, output_directory, "conv_ref_time_cutoff" + str(cutoff)) saver.save_csv( [["a","b","c", "MSE"], [conv_len_curve_fit_parameters]], output_directory,"conv_len_curve_fit_parameters-cutoff-" + str(cutoff)) saver.save_csv( [["a","b","c", "MSE"], [resp_time_curve_fit_parameters]], output_directory,"resp_time_curve_fit_parameters-cutoff-" + str(cutoff)) saver.save_csv( [["a","b","c", "MSE"], [conv_ref_time_curve_fit_parameters]], output_directory,"conv_ref_time_curve_fit_parameters-cutoff-"+str(cutoff)) config.CUTOFF_PERCENTILE = default_cutoff #revert back to default user.keywords_clusters(log_data, nicks, nick_same_list, output_directory, "keywords") network.degree_analysis_on_graph(message_number_graph) threshold = config.THRESHOLD_MESSAGE_NUMBER_GRAPH #store original default config cutoffs = [0, 10, 20] for cutoff in cutoffs: config.THRESHOLD_MESSAGE_NUMBER_GRAPH = cutoff msg_graph_experts, top_hub, top_keyword_overlap, top_auth = network.identify_hubs_and_experts(log_data, nicks, nick_same_list)
saver.save_csv(bin_matrix, output_directory, "MessageNumber_binsize_"+str(config.BIN_LENGTH_MINS)) for i in range(len(message_number_graph_day_list)): saver.draw_nx_graph(message_number_graph_day_list[i][0], output_directory, "mng" + str(i+1)) for i in range(len(nick_change_graph_list)): saver.draw_nx_graph(nick_change_graph_list[i], output_directory, "ncg" + str(i+1)) saver.draw_nx_graph(message_number_graph, output_directory, "mnagg") saver.draw_nx_graph(message_time_graph, output_directory, "mtgagg") saver.save_csv(conv_len, output_directory, "conv_len") saver.save_csv(resp_time, output_directory, "resp_time") saver.save_csv(conv_ref_time, output_directory, "conv_ref_time") # =============== VIZ =================== conv_len_curve_fit_parameters = vis.exponential_curve_fit_and_plot(conv_len, 20, output_directory, "conv_len") resp_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot(resp_time, 20, output_directory, "resp_time") conv_ref_time_curve_fit_parameters = vis.exponential_curve_fit_and_plot_x_shifted(conv_ref_time, 30, output_directory, "conv_ref_time") # vis.plot_infomap_igraph(adjCC_graph, adjCC_membership, output_directory, "adjCC_infomaps") # vis.generate_log_plots(9, out_degree_node_number, output_directory, channel_name[0] +"OUT"+ starting_date + ending_date) # ============== VALIDATION ============== validate.validate_RT_RL_CRT(conv_len_curve_fit_parameters, [[10.5, 10.6], [2.12, 2.32], [0, 0.2], [0, 0.0002]], "conv_len") validate.validate_RT_RL_CRT(resp_time_curve_fit_parameters, [[0.3, 10.4], [10.3, 30.4], [-0.002, 0.002], [0, 0.002]], "resp_time") validate.validate_RT_RL_CRT(conv_ref_time_curve_fit_parameters, [[10.05, 10.1], [0.1, 0.2], [0.02, 0.04], [0, 0.0002], [9, 11]], "conv_ref_time") # ============== PRESENCE ACROSS MULTIPLE CHANNELS ============== # Change analysis to all channels in config #nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(log_data, True) #dict_out, graph = network.channel_user_presence_graph_and_csv(nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash) #saver.save_csv(dict_out["UU"]["reducedMatrix"],output_directory, "rUU")
def correlational_CL_RT_CRT(log_directory, output_directory, start_date, end_date): """ Correlational : statistical distribution as illustrated by box plot for RT, CL, CRT parameters. The function takes the given time duration and selects one month at a time for generation of a degree distribution sample. Each degree distribution sample shall have 3 curve fit parameters namely a,b & c. The function collects these parameters for all the months of the given time duration. The function produces box plot separately for each parameter. Args: log_directory(str): path to the location of Logs output_directory(str): path to the location where the results are to be stored channel_name(list): channels for which the analysis is to be done start_date(datetime): starting date for the logs to be analysed. This has to be the beginning of the month. end_date(datetime): ending date for which the logs are to be analysed. This has to be the end of the month. Returns: null """ start_date = start_date.strptime('%Y-%m-%d') end_date = end_date.strptime('%Y-%m-%d') percentiles = [0, 1, 5, 10, 20] for channel_name_iter in [["#kubuntu-devel"], ["#ubuntu-devel"], ["#kubuntu"]]: for cutoff in percentiles: conv_len_curve_fit_parameters = np.zeros((12, 4)) resp_time_curve_fit_parameters = np.zeros((12, 4)) conv_ref_time_curve_fit_parameters = np.zeros((12, 5)) for dt in rrule(MONTHLY, dtstart=start_date, until=end_date): last_day_of_the_month = dt + relativedelta( months=1) - datetime.timedelta(days=1) log_data = reader.linux_input( log_directory, channel_name_iter, dt.strftime("%Y-%m-%d"), last_day_of_the_month.strftime("%Y-%m-%d")) nicks, nick_same_list = nickTracker.nick_tracker(log_data) default_cutoff = config.CUTOFF_PERCENTILE config.CUTOFF_PERCENTILE = cutoff truncated_rt, rt_cutoff_time = channel.response_time( log_data, nicks, nick_same_list, config.CUTOFF_PERCENTILE) conv_len, conv_ref_time = channel.conv_len_conv_refr_time( log_data, nicks, nick_same_list, rt_cutoff_time, config.CUTOFF_PERCENTILE) conv_len_curve_fit_parameters[ dt.month - 1] = vis.exponential_curve_fit_and_plot( conv_len, output_directory, "conv_len_cutoff" + str(cutoff)) resp_time_curve_fit_parameters[ dt.month - 1] = vis.exponential_curve_fit_and_plot( truncated_rt, output_directory, "resp_time_cutoff" + str(cutoff)) conv_ref_time_curve_fit_parameters[ dt.month - 1] = vis.exponential_curve_fit_and_plot_x_shifted( conv_ref_time, output_directory, "conv_ref_time_cutoff" + str(cutoff)) parameters = ['a', 'b', 'c'] for para_ind in range(len(parameters)): vis.box_plot( conv_len_curve_fit_parameters[:, para_ind], output_directory, "conv_len_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( resp_time_curve_fit_parameters[:, para_ind], output_directory, "resp_time_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) vis.box_plot( conv_ref_time_curve_fit_parameters[:, para_ind], output_directory, "conv_refr_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv( [conv_len_curve_fit_parameters[:, para_ind].tolist()], output_directory, "conv_len_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv( [resp_time_curve_fit_parameters[:, para_ind].tolist()], output_directory, "resp_time_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) saver.save_csv( [conv_ref_time_curve_fit_parameters[:, para_ind].tolist()], output_directory, "conv_refr_" + str(parameters[para_ind]) + "_2013_" + channel_name_iter[0] + "_cut_" + str(cutoff)) config.CUTOFF_PERCENTILE = default_cutoff