def test_channel_user_presence_graph_and_csv(self):
nicks = util.load_from_disk(self.current_directory +
'/data/test_presence_nicks')
nick_same_list = util.load_from_disk(
self.current_directory + '/data/test_presence_nick_same_list')
channels_for_user = util.load_from_disk(
self.current_directory + '/data/test_presence_channels_for_user')
nick_channel_dict = util.load_from_disk(
self.current_directory + '/data/test_presence_nick_channel_dict')
nicks_hash = util.load_from_disk(self.current_directory +
'/data/test_presence_nicks_hash')
channels_hash = util.load_from_disk(
self.current_directory + '/data/test_presence_channels_hash')
expected_dict_out = util.load_from_disk(self.current_directory +
'/data/dict_out')
expected_graph = util.load_from_disk(self.current_directory +
'/data/graph')
capturedOutput = StringIO.StringIO()
sys.stdout = capturedOutput
dict_out, graph = network.channel_user_presence_graph_and_csv(
nicks, nick_same_list, channels_for_user, nick_channel_dict,
nicks_hash, channels_hash)
sys.stdout = sys.__stdout__
capturedOutput.close()
self.assertTrue(nx.is_isomorphic(expected_graph, graph))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['CC']['graph'],
dict_out['CC']['graph']))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['CC']['reducedGraph'],
dict_out['CC']['reducedGraph']))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['CU']['graph'],
dict_out['CU']['graph']))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['CU']['reducedGraph'],
dict_out['CU']['reducedGraph']))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['UU']['graph'],
dict_out['UU']['graph']))
self.assertTrue(
nx.is_isomorphic(expected_dict_out['UU']['reducedGraph'],
dict_out['UU']['reducedGraph']))
assert_array_equal(expected_dict_out['CC']['matrix'],
dict_out['CC']['matrix'])
assert_array_equal(expected_dict_out['CC']['reducedMatrix'],
dict_out['CC']['reducedMatrix'])
assert_array_equal(expected_dict_out['CU']['matrix'],
dict_out['CU']['matrix'])
assert_array_equal(expected_dict_out['CU']['reducedMatrix'],
dict_out['CU']['reducedMatrix'])
assert_array_equal(expected_dict_out['UU']['matrix'],
dict_out['UU']['matrix'])
assert_array_equal(expected_dict_out['UU']['reducedMatrix'],
dict_out['UU']['reducedMatrix'])
def test_full_presence_graph(self, log_data):
update_expected_output_directory(log_data)
nicks1, nick_same_list1, channels_for_user1, nick_channel_dict1, \
nicks_hash1, channels_hash1 = nick_tracker(log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(nicks1, nick_same_list1, \
channels_for_user1, nick_channel_dict1, \
nicks_hash1, channels_hash1)
self.assertTrue(compare_graph_outputs(graph, "full_presence_graph.gpickle"), msg=None)
def test_full_presence_graph(self, log_data):
update_expected_output_directory(log_data)
nicks1, nick_same_list1, channels_for_user1, nick_channel_dict1, \
nicks_hash1, channels_hash1 = nick_tracker(log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(nicks1, nick_same_list1, \
channels_for_user1, nick_channel_dict1, \
nicks_hash1, channels_hash1)
self.assertTrue(compare_graph_outputs(graph, "full_presence_graph.gpickle"), msg=None)
def test_degree_distribution_multi_channel(self):
log_data = reader.linux_input(self.log_data_dir, ["ALL"],
self.start_date, self.end_date)
expected_result_CC_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/CC_degree_curve_fit')
expected_result_CU_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/CU_degree_curve_fit')
expected_result_UU_degree_curve_fit = util.load_from_disk(
self.current_directory + '/data/output/UU_degree_curve_fit')
nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(
log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(
nicks, nick_same_list, channels_for_user, nick_channel_dict,
nicks_hash, channels_hash)
degree_anal_message_number_CC = network.degree_analysis_on_graph(
dict_out["CC"]["graph"], directed=False)
degree_anal_message_number_UU = network.degree_analysis_on_graph(
dict_out["UU"]["graph"], directed=False)
degree_anal_message_number_CU = network.degree_analysis_on_graph(
dict_out["CU"]["graph"], directed=False)
Y = degree_anal_message_number_CU["degree"]["raw_for_vis"][1:]
data = [(i, Y[i]) for i in range(len(Y))]
CU_truncated, cutoff = channel.truncate_table(data, 0.5)
CU_T = [data[1] for data in list(CU_truncated)]
expected_output_CC_degree_curve_fit = vis.generate_log_plots(
degree_anal_message_number_CC["degree"]["raw_for_vis"],
self.current_directory, "CC_degree_curve_fit")
expected_output_CU_degree_curve_fit = vis.generate_log_plots(
CU_T, self.current_directory, "CU_degree_curve_fit")
expected_output_UU_degree_curve_fit = vis.generate_log_plots(
degree_anal_message_number_UU["degree"]["raw_for_vis"],
self.current_directory, "UU_degree_curve_fit")
os.remove(self.current_directory + "/CC_degree_curve_fit" + ".png")
os.remove(self.current_directory + "/CU_degree_curve_fit" + ".png")
os.remove(self.current_directory + "/UU_degree_curve_fit" + ".png")
self.assertEqual(expected_result_CC_degree_curve_fit,
expected_output_CC_degree_curve_fit)
self.assertEqual(expected_result_CU_degree_curve_fit,
expected_output_CU_degree_curve_fit)
self.assertEqual(expected_result_UU_degree_curve_fit,
expected_output_UU_degree_curve_fit)
def test_community_analysis_multi_channel(self):
log_data = reader.linux_input(self.log_data_dir, ["ALL"],
self.start_date, self.end_date)
expected_result = util.load_from_disk(
self.current_directory +
'/data/output/community_analysis_multi_channel')
nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(
log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(
nicks, nick_same_list, channels_for_user, nick_channel_dict,
nicks_hash, channels_hash)
presence_type = ["CC", "UU", "CU"]
expected_output = {ptype: {} for ptype in presence_type}
for ptype in presence_type:
saver.save_net_nx_graph(dict_out[ptype]["graph"],
self.current_directory, "adj" + ptype)
saver.save_net_nx_graph(dict_out[ptype]["reducedGraph"],
self.current_directory, "radj" + ptype)
expected_output[ptype]['adj'] = community.infomap_igraph(
ig_graph=None,
net_file_location=self.current_directory + '/adj' + ptype +
'.net')
expected_output[ptype]['radj'] = community.infomap_igraph(
ig_graph=None,
net_file_location=self.current_directory + '/radj' + ptype +
'.net')
os.remove(self.current_directory + '/adj' + ptype + '.net')
os.remove(self.current_directory + '/radj' + ptype + '.net')
self.assertTrue(expected_result[ptype]['adj'][0].isomorphic(
expected_output[ptype]['adj'][0]))
self.assertEqual(
compare_communities(expected_output[ptype]['adj'][1],
expected_result[ptype]['adj'][1]), 0)
self.assertTrue(expected_result[ptype]['radj'][0].isomorphic(
expected_output[ptype]['radj'][0]))
self.assertEqual(
compare_communities(expected_output[ptype]['radj'][1],
expected_result[ptype]['radj'][1]), 0)
def test_presence_networks(self):
log_data = reader.linux_input(self.log_data_dir, ["ALL"],
self.start_date, self.end_date)
expected_result = util.load_from_disk(
self.current_directory + '/data/output/presence_graph_dict')
nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(
log_data, True)
expected_output, graph = network.channel_user_presence_graph_and_csv(
nicks, nick_same_list, channels_for_user, nick_channel_dict,
nicks_hash, channels_hash)
edge_types = ['CC', 'UU', 'CU']
for edge_type in edge_types:
self.assertTrue(
nx.is_isomorphic(expected_output[edge_type]['graph'],
expected_result[edge_type]['graph']))
self.assertTrue(
nx.is_isomorphic(expected_output[edge_type]['reducedGraph'],
expected_result[edge_type]['reducedGraph']))
expected_output[edge_type].pop('graph')
expected_output[edge_type].pop('reducedGraph')
expected_result[edge_type].pop('graph')
expected_result[edge_type].pop('reducedGraph')
np.testing.assert_equal(expected_output, expected_result)
saver.save_csv(bin_matrix, output_directory, "MessageNumber_binsize_"+str(config.BIN_LENGTH_MINS))
# =============== VIZ ===================
message_graph, message_comm = community.infomap_igraph(ig_graph=None, net_file_location= output_directory + 'message_number_graph.net')
vis.plot_infomap_igraph(message_graph, message_comm.membership, output_directory, "message")
vis.plot_data (data, output_directory, "bins")
for dtype in degree_type:
slope,intercept,r_square,mse = vis.generate_log_plots(degree_anal_message_number[dtype]["raw_for_vis"], output_directory, channel_name[0] +dtype)
saver.save_csv( [["Y","K","R^2", "MSE"], [slope,intercept,r_square,mse]], output_directory, dtype+"-curve-fit")
# ============== PRESENCE ACROSS MULTIPLE CHANNELS ==============
# Change analysis to all channels in config
log_data = reader.linux_input(log_directory, ["ALL"], starting_date, ending_date)
nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash = nickTracker.nick_tracker(log_data, True)
dict_out, graph = network.channel_user_presence_graph_and_csv(nicks, nick_same_list, channels_for_user, nick_channel_dict, nicks_hash, channels_hash)
saver.save_js_arc(dict_out["CC"]["reducedGraph"], channels_hash, config.OUTPUT_DIRECTORY + "protovis/", "cc.js")
for ptype in presence_type:
saver.save_csv(dict_out[ptype]["reducedMatrix"],output_directory, "r"+ptype)
saver.save_net_nx_graph(dict_out[ptype]["graph"], output_directory, "adj"+ptype)
saver.save_net_nx_graph(dict_out[ptype]["reducedGraph"], output_directory, "radj"+ptype)
radj_graph, radj_comm = community.infomap_igraph(ig_graph=None, net_file_location= output_directory + 'radj'+ptype+'.net')
vis.plot_infomap_igraph(radj_graph, radj_comm.membership, output_directory, "radj"+ptype+"_infomaps-reduced")
adj_graph, adj_comm = community.infomap_igraph(ig_graph=None, net_file_location= output_directory + 'adj'+ptype+'.net')
vis.plot_infomap_igraph(adj_graph, adj_comm.membership, output_directory, "adj"+ptype+"_infomaps-full")
degree_anal_message_number_CC = network.degree_analysis_on_graph(dict_out["CC"]["graph"], directed=False)
saver.save_csv(degree_anal_message_number_CC["degree"]["formatted_for_csv"], output_directory, "CC_degree")
degree_anal_message_number_UU = network.degree_analysis_on_graph(dict_out["UU"]["graph"], directed = False)