def test_get_nick_sen_rec(self, iter_range, nick_to_search, conn_comp_list,
nick_sen_rec, expected_result):
self.assertEqual(
util.get_nick_sen_rec(iter_range, nick_to_search, conn_comp_list,
nick_sen_rec), expected_result)
self.assertEqual(util.get_nick_sen_rec(3, "krish", conn_comp_list, ""),
"krishna")
def parse_log_lines_for_conv(log_dict, nicks, conn_comp_list,
conversations):
dateadd = -1 #Variable used for response time calculation. Varies from 0-365.
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
dateadd = dateadd + 1
send_time = [
] #list of all the times a user sends a message to another user
#code for making relation map between clients
for line in day_log:
flag_comma = 0
if (util.check_if_msg_line(line)):
nick_sender = ""
nick_receiver = ""
m = re.search(r"\<(.*?)\>", line)
nick_to_search = util.correctLastCharCR(
m.group(0)[1:-1])
nick_sender = util.get_nick_sen_rec(
len(nicks), nick_to_search, conn_comp_list,
nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
conversations, nick_receiver, send_time = \
build_conversation(rec_list, nick, send_time,
nick_to_search, nick_receiver, nick_sender,
dateadd, conversations, conn_comp_list, line)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [
e.strip() for e in rec_list[1].split(',')
]
rec_list_2 = util.correct_last_char_list(
rec_list_2)
conversations, nick_receiver, send_time = \
build_conversation(rec_list_2, nick, send_time,
nick_to_search, nick_receiver,
nick_sender, dateadd, conversations,
conn_comp_list, line)
if (flag_comma == 0):
rec = util.splice_find(line, ">", ", ", 1)
conversations, nick_receiver, send_time = \
conv_helper(rec, nick, send_time, nick_to_search,
nick_receiver, nick_sender, dateadd,
conversations, conn_comp_list, line)
return conversations, nick_receiver, send_time
def msg_no_analysis_helper(rec_list, corrected_nick, nick, conn_comp_list,conversations,today_conversation):
for receiver in rec_list:
if(receiver == nick):
if(corrected_nick != nick):
nick_receiver = ''
nick_receiver = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, nick, conn_comp_list, nick_receiver)
if DAY_BY_DAY_ANALYSIS:
today_conversation = util.extend_conversation_list(nick_sender, nick_receiver, today_conversation)
else:
conversations = util.extend_conversation_list(nick_sender, nick_receiver, conversations)
def msg_no_analysis_helper(rec_list, corrected_nick, nick, conn_comp_list,conversations,today_conversation):
for receiver in rec_list:
if(receiver == nick):
if(corrected_nick != nick):
nick_receiver = ''
nick_receiver = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, nick, conn_comp_list, nick_receiver)
if DAY_BY_DAY_ANALYSIS:
today_conversation = util.extend_conversation_list(nick_sender, nick_receiver, today_conversation)
else:
conversations = util.extend_conversation_list(nick_sender, nick_receiver, conversations)
def conv_helper(rec, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list): if(rec == nick): send_time.append(line[1:6]) if(nick_to_search != nick): nick_receiver = util.get_nick_sen_rec(len(nicks), nick, conn_comp_list, nick_receiver) for i in range(config.MAX_CONVERSATIONS): if (nick_sender in conversations[i] and nick_receiver in conversations[i]): conversations = conv_append(conversations, i, dateadd, line) break if(len(conversations[i]) == 0): conversations[i].append(nick_sender) conversations[i].append(nick_receiver) conversations = conv_append(conversations, i, dateadd, line) break return conversations, nick_receiver, send_time
def message_time_graph(log_dict, nicks, nick_same_list, DAY_BY_DAY_ANALYSIS=False):
""" creates a directed graph where each edge denotes a message sent from a user to another user
with the stamp denoting the time at which the message was sent
Args:
log_dict (dictionary): Dictionary of logs data created using reader.py
nicks(List) : List of nickname created using nickTracker.py
nick_same_list(List) :List of same_nick names created using nickTracker.py
Returns:
msg_time_graph_list(List): List of message time graphs for different days
msg_time_aggr_graph: aggregate message time graph where edges are date + time when sender sends a message to receiver
"""
msg_time_graph_list = []
msg_time_aggr_graph = nx.MultiDiGraph()
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
def compare_spliced_nick(nick_to_compare, spliced_nick, nick_name, line):
if(nick_to_compare == nick_name):
if(spliced_nick != nick_name):
nick_receiver = nick_receiver_from_conn_comp(nick_name, conn_comp_list)
util.build_graphs(nick_sender, nick_receiver, line[1:6], year, month, day, graph_conversation, msg_time_aggr_graph)
util.create_connected_nick_list(conn_comp_list)
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
year, month, day = util.get_year_month_day(day_content)
graph_conversation = nx.MultiDiGraph() #graph with multiple directed edges between clients used
for line in day_log:
flag_comma = 0
if(util.check_if_msg_line (line)):
m = re.search(r"\<(.*?)\>", line)
spliced_nick = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = ""
nick_sender = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, spliced_nick, conn_comp_list, nick_sender)
for nick_name in nicks:
rec_list = [e.strip() for e in line.split(':')] #receiver list splited about :
util.rec_list_splice(rec_list)
if not rec_list[1]: #index 0 will contain time 14:02
break
rec_list = util.correct_last_char_list(rec_list)
for nick_to_search in rec_list:
if(nick_to_search == nick_name):
if(spliced_nick != nick_name):
nick_receiver = ""
nick_receiver = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, nick_name, conn_comp_list, nick_receiver)
util.build_graphs(nick_sender, nick_receiver, line[1:6], year, month, day, graph_conversation, msg_time_aggr_graph)
if "," in rec_list[1]: #receiver list may of the form <Dhruv> Rohan, Ram :
flag_comma = 1
rec_list_2 = [e.strip() for e in rec_list[1].split(',')]
rec_list_2 = util.correct_last_char_list(rec_list_2)
for nick_to_search in rec_list_2:
compare_spliced_nick(nick_to_search, spliced_nick, nick_name, line)
if(flag_comma == 0): #receiver list can be <Dhruv> Rohan, Hi!
rec = line[line.find(">") + 1:line.find(", ")]
rec = util.correctLastCharCR(rec[1:])
compare_spliced_nick(rec, spliced_nick, nick_name, line)
msg_time_graph_list.append(graph_conversation)
if DAY_BY_DAY_ANALYSIS:
return msg_time_graph_list
else:
return msg_time_aggr_graph
def message_number_graph(log_dict, nicks, nick_same_list, DAY_BY_DAY_ANALYSIS=False):
""" Creates a directed graph
with each node representing an IRC user
and each directed edge has a weight which
mentions the number messages sent and recieved by that user
in the selected time frame.
Args:
log_dict (dict): with key as dateTime.date object and value as {"data":datalist,"channel_name":channels name}
nicks(list): list of all the nicks
nick_same_list(list): list of lists mentioning nicks which belong to same users
Returns:
message_number_graph (nx graph object)
"""
message_number_day_list = []
conversations=[[0] for i in range(config.MAX_EXPECTED_DIFF_NICKS)]
aggregate_message_number_graph = nx.DiGraph() #graph with multiple directed edges between clients used
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
def msg_no_analysis_helper(rec_list, corrected_nick, nick, conn_comp_list,conversations,today_conversation):
for receiver in rec_list:
if(receiver == nick):
if(corrected_nick != nick):
nick_receiver = ''
nick_receiver = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, nick, conn_comp_list, nick_receiver)
if DAY_BY_DAY_ANALYSIS:
today_conversation = util.extend_conversation_list(nick_sender, nick_receiver, today_conversation)
else:
conversations = util.extend_conversation_list(nick_sender, nick_receiver, conversations)
def message_no_add_egde(message_graph, conversation):
for index in xrange(config.MAX_EXPECTED_DIFF_NICKS):
if(len(conversation[index]) == 3 and conversation[index][0] >= config.THRESHOLD_MESSAGE_NUMBER_GRAPH):
if len(conversation[index][1]) >= config.MINIMUM_NICK_LENGTH and len(conversation[index][2]) >= config.MINIMUM_NICK_LENGTH:
message_graph.add_edge(conversation[index][1], conversation[index][2], weight=conversation[index][0])
return message_graph
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
today_conversation = [[0] for i in range(config.MAX_EXPECTED_DIFF_NICKS)]
for line in day_log:
flag_comma = 0
if(util.check_if_msg_line (line)):
parsed_nick = re.search(r"\<(.*?)\>", line)
corrected_nick = util.correctLastCharCR(parsed_nick.group(0)[1:-1])
nick_sender = ""
nick_receiver = ""
nick_sender = util.get_nick_sen_rec(config.MAX_EXPECTED_DIFF_NICKS, corrected_nick, conn_comp_list, nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
msg_no_analysis_helper(rec_list, corrected_nick, nick, conn_comp_list, conversations,today_conversation)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2=[e.strip() for e in rec_list[1].split(',')]
for i in xrange(0,len(rec_list_2)):
if(rec_list_2[i]):
rec_list_2[i] = util.correctLastCharCR(rec_list_2[i])
msg_no_analysis_helper(rec_list_2, corrected_nick, nick, conn_comp_list, conversations, today_conversation)
if(flag_comma == 0):
rec = line[line.find(">")+1:line.find(", ")]
rec = rec[1:]
rec = util.correctLastCharCR(rec)
if(rec == nick):
if(corrected_nick != nick):
nick_receiver = nick_receiver_from_conn_comp(nick, conn_comp_list)
if DAY_BY_DAY_ANALYSIS:
today_message_number_graph = nx.DiGraph()
today_message_number_graph = message_no_add_egde(today_message_number_graph, today_conversation)
year, month, day = util.get_year_month_day(day_content)
message_number_day_list.append([today_message_number_graph, year+'-'+month+'-'+day])
print "\nBuilding graph object with EDGE WEIGHT THRESHOLD:", config.THRESHOLD_MESSAGE_NUMBER_GRAPH
if not DAY_BY_DAY_ANALYSIS:
aggregate_message_number_graph = message_no_add_egde(aggregate_message_number_graph, conversations)
if config.DEBUGGER:
print "========> 30 on " + str(len(conversations)) + " conversations"
print conversations[:30]
if DAY_BY_DAY_ANALYSIS:
return message_number_day_list
else:
return aggregate_message_number_graph
def response_time(log_dict, nicks, nick_same_list, cutoff_percentile):
""" finds the response time of a message
i.e. the best guess for the time at which one can expect a reply for his/her message.
Args:
log_dict (str): Dictionary of logs data created using reader.py
nicks(List) : List of nickname created using nickTracker.py
nick_same_list :List of same_nick names created using nickTracker.py
cutoff_percentile (int): Cutoff percentile indicating statistical significance
Returns:
rows_RT(zip List): Response Time (This refers to the response
time of a message i.e. the best guess for the time at
which one can expect a reply for his/her message)
"""
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
graph_cumulative = []
graph_x_axis = []
graph_y_axis = []
def build_mean_list(conversations, index, mean_list):
for j in range(2, len(conversations[index])):
mean_list.append(conversations[index][j])
return mean_list
def resp_helper(rec, nick, send_time, nick_to_search, nick_receiver,
nick_sender, conversations, conn_comp_list):
if (rec == nick):
send_time.append(line[1:6])
if (nick_to_search != nick):
nick_receiver = util.get_nick_sen_rec(len(nicks), nick,
conn_comp_list,
nick_receiver)
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (nick_sender in conversations[i]
and nick_receiver in conversations[i]):
conversations[i].append(line[1:6])
break
if (len(conversations[i]) == 0):
conversations[i].append(nick_sender)
conversations[i].append(nick_receiver)
conversations[i].append(line[1:6])
break
return conversations, nick_receiver, send_time
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
send_time = [
] #list of all the times a user sends a message to another user
meanstd_list = []
totalmeanstd_list = []
x_axis = []
y_axis = []
real_y_axis = []
conversations = [[]
for i in range(config.MAX_RESPONSE_CONVERSATIONS)]
#code for making relation map between clients
for line in day_log:
flag_comma = 0
if (util.check_if_msg_line(line)):
nick_sender = ""
nick_receiver = ""
m = re.search(r"\<(.*?)\>", line)
nick_to_search = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = util.get_nick_sen_rec(
len(nicks), nick_to_search, conn_comp_list,
nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
for name in rec_list:
conversations, nick_receiver, send_time = resp_helper(
name, nick, send_time, nick_to_search,
nick_receiver, nick_sender, conversations,
conn_comp_list)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [
e.strip() for e in rec_list[1].split(',')
]
rec_list_2 = util.correct_last_char_list(
rec_list_2)
for name in rec_list_2:
conversations, nick_receiver, send_time = resp_helper(
name, nick, send_time, nick_to_search,
nick_receiver, nick_sender, conversations,
conn_comp_list)
if (flag_comma == 0):
rec = util.splice_find(line, ">", ", ", 1)
conversations, nick_receiver, send_time = resp_helper(
rec, nick, send_time, nick_to_search,
nick_receiver, nick_sender, conversations,
conn_comp_list)
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (len(conversations[i]) != 0):
for j in range(2, len(conversations[i]) - 1):
conversations[i][j] = (
int(conversations[i][j + 1][0:2]) *
config.MINS_PER_HOUR +
int(conversations[i][j + 1][3:5])) - (
int(conversations[i][j][0:2]) *
config.MINS_PER_HOUR +
int(conversations[i][j][3:5]))
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (len(conversations[i]) != 0):
if (len(conversations[i]) == 3):
conversations[i][2] = int(conversations[i][2][
0:2]) * config.MINS_PER_HOUR + int(
conversations[i][2][3:5])
else:
del conversations[i][-1]
#Explanation provided in parser-CL+CRT.py
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (len(conversations[i]) != 0):
totalmeanstd_list = build_mean_list(
conversations, i, totalmeanstd_list)
if (len(totalmeanstd_list) != 0):
for i in range(max(totalmeanstd_list) + 1):
x_axis.append(i)
for i in x_axis:
y_axis.append(
float(totalmeanstd_list.count(i)) /
float(len(totalmeanstd_list)))
#finding the probability of each RT to occur=No. of occurence/total occurences.
real_y_axis.append(y_axis[0])
for i in range(len(y_axis)):
real_y_axis.append(
float(real_y_axis[i - 1]) + float(y_axis[i]))
#to find cumulative just go on adding the current value to previously cumulated value till sum becomes 1 for last entry.
for i in range(len(totalmeanstd_list)):
graph_cumulative.append(totalmeanstd_list[i])
if len(totalmeanstd_list) > 0:
totalmeanstd_list.append(numpy.mean(totalmeanstd_list))
totalmeanstd_list.append(
numpy.mean(totalmeanstd_list) +
2 * numpy.std(totalmeanstd_list))
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (len(conversations[i]) != 0):
meanstd_list = build_mean_list(conversations, i,
meanstd_list)
conversations[i].append(numpy.mean(meanstd_list))
conversations[i].append(
numpy.mean(meanstd_list) +
(2 * numpy.std(meanstd_list)))
meanstd_list[:] = []
graph_cumulative.sort()
truncated_rt = None
rt_cutoff_time = None
if graph_cumulative:
for i in range(graph_cumulative[len(graph_cumulative) - 1] + 1):
graph_y_axis.append(graph_cumulative.count(
i)) # problem when ti=0 count is unexpectedly large
graph_x_axis.append(i)
#Finally storing the RT values along with their frequencies in a csv file; no need to invoke build_stat_dist() function
rows_rt = zip(graph_x_axis, graph_y_axis)
truncated_rt, rt_cutoff_time = truncate_table(rows_rt,
cutoff_percentile)
if config.CUTOFF_TIME_STRATEGY == "TWO_SIGMA":
resp_time, resp_frequency_tuple = zip(*truncated_rt)
resp_frequency = list(resp_frequency_tuple)
rt_cutoff_time_frac = numpy.mean(
resp_frequency) + 2 * numpy.std(resp_frequency)
rt_cutoff_time = int(numpy.ceil(rt_cutoff_time_frac))
return truncated_rt, rt_cutoff_time
def message_time_graph(log_dict,
nicks,
nick_same_list,
DAY_BY_DAY_ANALYSIS=False):
""" creates a directed graph where each edge denotes a message sent from a user to another user
with the stamp denoting the time at which the message was sent
Args:
log_dict (dictionary): Dictionary of logs data created using reader.py
nicks(List) : List of nickname created using nickTracker.py
nick_same_list(List) :List of same_nick names created using nickTracker.py
Returns:
msg_time_graph_list(List): List of message time graphs for different days
msg_time_aggr_graph: aggregate message time graph where edges are date + time when sender sends a message to receiver
"""
msg_time_graph_list = []
msg_time_aggr_graph = nx.MultiDiGraph()
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
def compare_spliced_nick(nick_to_compare, spliced_nick, nick_name, line):
if (nick_to_compare == nick_name):
if (spliced_nick != nick_name):
nick_receiver = nick_receiver_from_conn_comp(
nick_name, conn_comp_list)
util.build_graphs(nick_sender, nick_receiver, line[1:6], year,
month, day, graph_conversation,
msg_time_aggr_graph)
util.create_connected_nick_list(conn_comp_list)
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
year, month, day = util.get_year_month_day(day_content)
graph_conversation = nx.MultiDiGraph(
) #graph with multiple directed edges between clients used
for line in day_log:
flag_comma = 0
if (util.check_if_msg_line(line)):
m = re.search(r"\<(.*?)\>", line)
spliced_nick = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = ""
nick_sender = util.get_nick_sen_rec(
config.MAX_EXPECTED_DIFF_NICKS, spliced_nick,
conn_comp_list, nick_sender)
for nick_name in nicks:
rec_list = [e.strip() for e in line.split(':')
] #receiver list splited about :
util.rec_list_splice(rec_list)
if not rec_list[1]: #index 0 will contain time 14:02
break
rec_list = util.correct_last_char_list(rec_list)
for nick_to_search in rec_list:
if (nick_to_search == nick_name):
if (spliced_nick != nick_name):
nick_receiver = ""
nick_receiver = util.get_nick_sen_rec(
config.MAX_EXPECTED_DIFF_NICKS,
nick_name, conn_comp_list,
nick_receiver)
util.build_graphs(nick_sender,
nick_receiver, line[1:6],
year, month, day,
graph_conversation,
msg_time_aggr_graph)
if "," in rec_list[
1]: #receiver list may of the form <Dhruv> Rohan, Ram :
flag_comma = 1
rec_list_2 = [
e.strip() for e in rec_list[1].split(',')
]
rec_list_2 = util.correct_last_char_list(
rec_list_2)
for nick_to_search in rec_list_2:
compare_spliced_nick(nick_to_search,
spliced_nick, nick_name,
line)
if (flag_comma == 0
): #receiver list can be <Dhruv> Rohan, Hi!
rec = line[line.find(">") + 1:line.find(", ")]
rec = util.correctLastCharCR(rec[1:])
compare_spliced_nick(rec, spliced_nick, nick_name,
line)
msg_time_graph_list.append(graph_conversation)
if DAY_BY_DAY_ANALYSIS:
return msg_time_graph_list
else:
return msg_time_aggr_graph
def message_number_graph(log_dict,
nicks,
nick_same_list,
DAY_BY_DAY_ANALYSIS=False):
""" Creates a directed graph
with each node representing an IRC user
and each directed edge has a weight which
mentions the number messages sent and recieved by that user
in the selected time frame.
Args:
log_dict (dict): with key as dateTime.date object and value as {"data":datalist,"channel_name":channels name}
nicks(list): list of all the nicks
nick_same_list(list): list of lists mentioning nicks which belong to same users
Returns:
message_number_graph (nx graph object)
"""
message_number_day_list = []
conversations = [[0] for i in range(config.MAX_EXPECTED_DIFF_NICKS)]
aggregate_message_number_graph = nx.DiGraph(
) #graph with multiple directed edges between clients used
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
def msg_no_analysis_helper(rec_list, corrected_nick, nick, conn_comp_list,
conversations, today_conversation):
for receiver in rec_list:
if (receiver == nick):
if (corrected_nick != nick):
nick_receiver = ''
nick_receiver = util.get_nick_sen_rec(
config.MAX_EXPECTED_DIFF_NICKS, nick, conn_comp_list,
nick_receiver)
if DAY_BY_DAY_ANALYSIS:
today_conversation = util.extend_conversation_list(
nick_sender, nick_receiver, today_conversation)
else:
conversations = util.extend_conversation_list(
nick_sender, nick_receiver, conversations)
def message_no_add_egde(message_graph, conversation):
for index in xrange(config.MAX_EXPECTED_DIFF_NICKS):
if (len(conversation[index]) == 3 and conversation[index][0] >=
config.THRESHOLD_MESSAGE_NUMBER_GRAPH):
if len(
conversation[index]
[1]) >= config.MINIMUM_NICK_LENGTH and len(
conversation[index][2]) >= config.MINIMUM_NICK_LENGTH:
message_graph.add_edge(conversation[index][1],
conversation[index][2],
weight=conversation[index][0])
return message_graph
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
today_conversation = [
[0] for i in range(config.MAX_EXPECTED_DIFF_NICKS)
]
for line in day_log:
flag_comma = 0
if (util.check_if_msg_line(line)):
parsed_nick = re.search(r"\<(.*?)\>", line)
corrected_nick = util.correctLastCharCR(
parsed_nick.group(0)[1:-1])
nick_sender = ""
nick_receiver = ""
nick_sender = util.get_nick_sen_rec(
config.MAX_EXPECTED_DIFF_NICKS, corrected_nick,
conn_comp_list, nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
msg_no_analysis_helper(rec_list, corrected_nick, nick,
conn_comp_list, conversations,
today_conversation)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [
e.strip() for e in rec_list[1].split(',')
]
for i in xrange(0, len(rec_list_2)):
if (rec_list_2[i]):
rec_list_2[i] = util.correctLastCharCR(
rec_list_2[i])
msg_no_analysis_helper(rec_list_2, corrected_nick,
nick, conn_comp_list,
conversations,
today_conversation)
if (flag_comma == 0):
rec = line[line.find(">") + 1:line.find(", ")]
rec = rec[1:]
rec = util.correctLastCharCR(rec)
if (rec == nick):
if (corrected_nick != nick):
nick_receiver = nick_receiver_from_conn_comp(
nick, conn_comp_list)
if DAY_BY_DAY_ANALYSIS:
today_message_number_graph = nx.DiGraph()
today_message_number_graph = message_no_add_egde(
today_message_number_graph, today_conversation)
year, month, day = util.get_year_month_day(day_content)
message_number_day_list.append([
today_message_number_graph, year + '-' + month + '-' + day
])
print "\nBuilding graph object with EDGE WEIGHT THRESHOLD:", config.THRESHOLD_MESSAGE_NUMBER_GRAPH
if not DAY_BY_DAY_ANALYSIS:
aggregate_message_number_graph = message_no_add_egde(
aggregate_message_number_graph, conversations)
if config.DEBUGGER:
print "========> 30 on " + str(len(conversations)) + " conversations"
print conversations[:30]
if DAY_BY_DAY_ANALYSIS:
return message_number_day_list
else:
return aggregate_message_number_graph
def conv_len_conv_refr_time(log_dict, nicks, nick_same_list):
""" Calculates the conversation length (CL) that is the length of time for which two users communicate
i.e. if a message is not replied to within Response Time(RT),
then it is considered as a part of another conversation.
This function also calculates the conversation refresh time(CRT)
For a pair of users, this is the time when one conversation ends and another one starts.
Args:
log_dict (str): Dictionary of logs data created using reader.py
nicks(List) : list of nickname created using nickTracker.py
nick_same_list :List of same_nick names created using nickTracker.py
Returns:
row_cl(zip List): Conversation Length
row_crt(zip List) :Conversation Refresh time
"""
conv = []
conv_diff = []
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
# We use connected components algorithm to group all those nick clusters that have atleast one nick common in their clusters. So e.g.
#Cluster 1- nick1,nick2,nick3,nick4(some nicks of a user) #Cluster 2 -nick5,nick6,nick2,nick7. Then we would get - nick1,nick2,nick3,nick4,nick5,nick6,nick7 and we can safely assume they belong to the same user.
conversations = [
[] for i in range(config.MAX_CONVERSATIONS)
] #This might need to be incremented from 10000 if we have more users. Same logic as the above 7000 one. Applies to all the other codes too.
## I would advice on using a different data structure which does not have an upper bound like we do in arrays.
graphx1 = []
graphy1 = []
graphx2 = []
graphy2 = []
dateadd = -1 #Variable used for response time calculation. Varies from 0-365.
def build_conversation(rec_list, nick, send_time, nick_to_search,
nick_receiver, nick_sender, dateadd, conversations,
conn_comp_list):
for names in rec_list:
conversations, nick_receiver, send_time = conv_helper(
names, nick, send_time, nick_to_search, nick_receiver,
nick_sender, dateadd, conversations, conn_comp_list)
return conversations, nick_receiver, send_time
def conv_helper(rec, nick, send_time, nick_to_search, nick_receiver,
nick_sender, dateadd, conversations, conn_comp_list):
if (rec == nick):
send_time.append(line[1:6])
if (nick_to_search != nick):
nick_receiver = util.get_nick_sen_rec(len(nicks), nick,
conn_comp_list,
nick_receiver)
for i in range(config.MAX_CONVERSATIONS):
if (nick_sender in conversations[i]
and nick_receiver in conversations[i]):
conversations = conv_append(conversations, i, dateadd,
line)
break
if (len(conversations[i]) == 0):
conversations[i].append(nick_sender)
conversations[i].append(nick_receiver)
conversations = conv_append(conversations, i, dateadd,
line)
break
return conversations, nick_receiver, send_time
def conv_mat_diff(i, j, conversations):
"""
i(int): matrix index for row
j(int): matrix index for column
"""
return (conversations[i][j] - conversations[i][j - 1])
def conv_append(conversations, index, dateadd, line):
conversations[index].append(
config.HOURS_PER_DAY * config.MINS_PER_HOUR * dateadd +
int(line[1:6][0:2]) * config.MINS_PER_HOUR + int(line[1:6][3:5]))
return conversations
for day_content_all_channels in list(log_dict.values()):
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
dateadd = dateadd + 1
send_time = [
] #list of all the times a user sends a message to another user
#code for making relation map between clients
for line in day_log:
flag_comma = 0
if (util.check_if_msg_line(line)):
nick_sender = ""
nick_receiver = ""
m = re.search(r"\<(.*?)\>", line)
nick_to_search = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = util.get_nick_sen_rec(
len(nicks), nick_to_search, conn_comp_list,
nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
conversations, nick_receiver, send_time = build_conversation(
rec_list, nick, send_time, nick_to_search,
nick_receiver, nick_sender, dateadd, conversations,
conn_comp_list)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [
e.strip() for e in rec_list[1].split(',')
]
rec_list_2 = util.correct_last_char_list(
rec_list_2)
conversations, nick_receiver, send_time = build_conversation(
rec_list_2, nick, send_time, nick_to_search,
nick_receiver, nick_sender, dateadd,
conversations, conn_comp_list)
if (flag_comma == 0):
rec = util.splice_find(line, ">", ", ", 1)
conversations, nick_receiver, send_time = conv_helper(
rec, nick, send_time, nick_to_search,
nick_receiver, nick_sender, dateadd,
conversations, conn_comp_list)
#Lines 212-290 consider all cases in which messages are addressed as - (nick1:nick2 or nick1,nick2 or nick1,nick2:) and stores their response times in conversations. conversations[i] contains all the response times between userA and userB throughout an entire year.
for i in range(
len(conversations)
): #Lines 295-297 remove the first two elements from every conversations[i] as they are the UIDS of sender and receiver respectively(and not RTs)
if (
len(conversations[i]) != 0
): # response times are calculated starting from index 2. So now we have all the response times in conversations.
del conversations[i][0:2]
for i in range(len(conversations)):
if (len(conversations[i]) != 0):
first = conversations[i][0]
for j in range(1, len(conversations[i])):
if (conv_mat_diff(i, j, conversations) > 9):
conv.append(
conversations[i][j - 1] - first
) #We are recording the conversation length in conv and CRT in conv_diff. Here 9 is the average response
#time we have already found before(see parser-RT.py). For every channel this value differs and would have to be changed in the code.
conv_diff.append(conv_mat_diff(i, j, conversations))
first = conversations[i][j]
if (j == (len(conversations[i]) - 1)):
conv.append(conversations[i][j] - first)
break
def build_conv_csv(conv_list, graph_x, graph_y):
for i in range(max(conv_list)):
graph_x.append(i)
graph_y.append(conv_list.count(i))
return graph_x, graph_y
graphx1, graphy1 = build_conv_csv(conv, graphx1, graphy1)
graphx2, graphy2 = build_conv_csv(conv_diff, graphx2, graphy2)
#To plot CDF we store the CL and CRT values and their number of occurences as shown above.
row_cl = list(zip(graphx1, graphy1))
row_crt = list(zip(graphx2, graphy2))
return row_cl, row_crt
def conv_len_conv_refr_time(log_dict, nicks, nick_same_list):
""" Calculates the conversation length (CL) that is the length of time for which two users communicate
i.e. if a message is not replied to within Response Time(RT),
then it is considered as a part of another conversation.
This function also calculates the conversation refresh time(CRT)
For a pair of users, this is the time when one conversation ends and another one starts.
Args:
log_dict (str): Dictionary of logs data created using reader.py
nicks(List) : list of nickname created using nickTracker.py
nick_same_list :List of same_nick names created using nickTracker.py
Returns:
row_cl(zip List): Conversation Length
row_crt(zip List) :Conversation Refresh time
"""
conv = []
conv_diff = []
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
# We use connected components algorithm to group all those nick clusters that have atleast one nick common in their clusters. So e.g.
#Cluster 1- nick1,nick2,nick3,nick4(some nicks of a user) #Cluster 2 -nick5,nick6,nick2,nick7. Then we would get - nick1,nick2,nick3,nick4,nick5,nick6,nick7 and we can safely assume they belong to the same user.
conversations=[[] for i in range(config.MAX_CONVERSATIONS)] #This might need to be incremented from 10000 if we have more users. Same logic as the above 7000 one. Applies to all the other codes too.
## I would advice on using a different data structure which does not have an upper bound like we do in arrays.
graphx1 =[]
graphy1 =[]
graphx2 =[]
graphy2 =[]
dateadd = -1 #Variable used for response time calculation. Varies from 0-365.
def build_conversation(rec_list, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list):
for names in rec_list:
conversations, nick_receiver, send_time = conv_helper(names, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list)
return conversations, nick_receiver, send_time
def conv_helper(rec, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list):
if(rec == nick):
send_time.append(line[1:6])
if(nick_to_search != nick):
nick_receiver = util.get_nick_sen_rec(len(nicks), nick, conn_comp_list, nick_receiver)
for i in range(config.MAX_CONVERSATIONS):
if (nick_sender in conversations[i] and nick_receiver in conversations[i]):
conversations = conv_append(conversations, i, dateadd, line)
break
if(len(conversations[i]) == 0):
conversations[i].append(nick_sender)
conversations[i].append(nick_receiver)
conversations = conv_append(conversations, i, dateadd, line)
break
return conversations, nick_receiver, send_time
def conv_mat_diff(i,j,conversations):
"""
i(int): matrix index for row
j(int): matrix index for column
"""
return (conversations[i][j]-conversations[i][j-1])
def conv_append(conversations, index, dateadd, line):
conversations[index].append(config.HOURS_PER_DAY*config.MINS_PER_HOUR*dateadd + int(line[1:6][0:2])*config.MINS_PER_HOUR + int(line[1:6][3:5]))
return conversations
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
dateadd = dateadd + 1
send_time = [] #list of all the times a user sends a message to another user
#code for making relation map between clients
for line in day_log:
flag_comma = 0
if(util.check_if_msg_line (line)):
nick_sender = ""
nick_receiver = ""
m = re.search(r"\<(.*?)\>", line)
nick_to_search = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = util.get_nick_sen_rec(len(nicks), nick_to_search, conn_comp_list, nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
conversations, nick_receiver, send_time = build_conversation(rec_list, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [e.strip() for e in rec_list[1].split(',')]
rec_list_2 = util.correct_last_char_list(rec_list_2)
conversations, nick_receiver, send_time = build_conversation(rec_list_2, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list)
if(flag_comma == 0):
rec = util.splice_find(line, ">", ", ", 1)
conversations, nick_receiver, send_time = conv_helper(rec, nick, send_time, nick_to_search, nick_receiver, nick_sender, dateadd, conversations, conn_comp_list)
#Lines 212-290 consider all cases in which messages are addressed as - (nick1:nick2 or nick1,nick2 or nick1,nick2:) and stores their response times in conversations. conversations[i] contains all the response times between userA and userB throughout an entire year.
for i in range(len(conversations)): #Lines 295-297 remove the first two elements from every conversations[i] as they are the UIDS of sender and receiver respectively(and not RTs)
if(len(conversations[i]) != 0): # response times are calculated starting from index 2. So now we have all the response times in conversations.
del conversations[i][0:2]
for i in range(len(conversations)):
if(len(conversations[i]) != 0):
first = conversations[i][0]
for j in range(1, len(conversations[i])):
if(conv_mat_diff(i, j, conversations) > 9):
conv.append(conversations[i][j-1] - first) #We are recording the conversation length in conv and CRT in conv_diff. Here 9 is the average response
#time we have already found before(see parser-RT.py). For every channel this value differs and would have to be changed in the code.
conv_diff.append(conv_mat_diff(i, j, conversations))
first = conversations[i][j]
if(j == (len(conversations[i]) - 1)):
conv.append(conversations[i][j] - first)
break
def build_conv_csv(conv_list, graph_x, graph_y):
for i in range(max(conv_list)):
graph_x.append(i)
graph_y.append(conv_list.count(i))
return graph_x, graph_y
graphx1, graphy1 = build_conv_csv(conv, graphx1, graphy1)
graphx2, graphy2 = build_conv_csv(conv_diff, graphx2, graphy2)
#To plot CDF we store the CL and CRT values and their number of occurences as shown above.
row_cl = zip(graphx1, graphy1)
row_crt = zip(graphx2, graphy2)
return row_cl, row_crt
def response_time(log_dict, nicks, nick_same_list):
""" finds the response time of a message
i.e. the best guess for the time at which one can expect a reply for his/her message.
Args:
log_dict (str): Dictionary of logs data created using reader.py
nicks(List) : List of nickname created using nickTracker.py
nick_same_list :List of same_nick names created using nickTracker.py
output_directory (str): Location of output directory
Returns:
rows_RT(zip List): Response Time (This refers to the response
time of a message i.e. the best guess for the time at
which one can expect a reply for his/her message)
"""
G = util.to_graph(nick_same_list)
conn_comp_list = list(connected_components(G))
util.create_connected_nick_list(conn_comp_list)
graph_cumulative = []
graph_x_axis = []
graph_y_axis = []
def build_mean_list(conversations, index, mean_list):
for j in range(2, len(conversations[index])):
mean_list.append(conversations[index][j])
return mean_list
def resp_helper(rec, nick, send_time, nick_to_search, nick_receiver, nick_sender, conversations, conn_comp_list):
if(rec == nick):
send_time.append(line[1:6])
if(nick_to_search != nick):
nick_receiver = util.get_nick_sen_rec(len(nicks), nick, conn_comp_list, nick_receiver)
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if (nick_sender in conversations[i] and nick_receiver in conversations[i]):
conversations[i].append(line[1:6])
break
if(len(conversations[i]) == 0):
conversations[i].append(nick_sender)
conversations[i].append(nick_receiver)
conversations[i].append(line[1:6])
break
return conversations, nick_receiver, send_time
for day_content_all_channels in log_dict.values():
for day_content in day_content_all_channels:
day_log = day_content["log_data"]
send_time = [] #list of all the times a user sends a message to another user
meanstd_list = []
totalmeanstd_list = []
x_axis = []
y_axis = []
real_y_axis = []
conversations = [[] for i in range(config.MAX_RESPONSE_CONVERSATIONS)]
#code for making relation map between clients
for line in day_log:
flag_comma = 0
if(util.check_if_msg_line (line)):
nick_sender = ""
nick_receiver = ""
m = re.search(r"\<(.*?)\>", line)
nick_to_search = util.correctLastCharCR(m.group(0)[1:-1])
nick_sender = util.get_nick_sen_rec(len(nicks), nick_to_search, conn_comp_list, nick_sender)
for nick in nicks:
rec_list = [e.strip() for e in line.split(':')]
util.rec_list_splice(rec_list)
if not rec_list[1]:
break
rec_list = util.correct_last_char_list(rec_list)
for name in rec_list:
conversations, nick_receiver, send_time = resp_helper(name, nick, send_time, nick_to_search, nick_receiver, nick_sender, conversations, conn_comp_list)
if "," in rec_list[1]:
flag_comma = 1
rec_list_2 = [e.strip() for e in rec_list[1].split(',')]
rec_list_2 = util.correct_last_char_list(rec_list_2)
for name in rec_list_2:
conversations, nick_receiver, send_time = resp_helper(name, nick, send_time, nick_to_search, nick_receiver, nick_sender, conversations, conn_comp_list)
if(flag_comma == 0):
rec = util.splice_find(line, ">", ", ",1)
conversations, nick_receiver, send_time = resp_helper(rec, nick, send_time, nick_to_search, nick_receiver, nick_sender, conversations, conn_comp_list)
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if(len(conversations[i]) != 0):
for j in range(2, len(conversations[i]) - 1):
conversations[i][j]=(int(conversations[i][j+1][0:2])*config.MINS_PER_HOUR+int(conversations[i][j+1][3:5])) - (int(conversations[i][j][0:2])*config.MINS_PER_HOUR+int(conversations[i][j][3:5]))
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if(len(conversations[i]) != 0):
if(len(conversations[i]) == 3):
conversations[i][2] = int(conversations[i][2][0:2])*config.MINS_PER_HOUR+int(conversations[i][2][3:5])
else:
del conversations[i][-1]
#Explanation provided in parser-CL+CRT.py
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if(len(conversations[i]) != 0):
totalmeanstd_list = build_mean_list(conversations, i, totalmeanstd_list)
if(len(totalmeanstd_list) != 0):
for i in range(max(totalmeanstd_list) + 1):
x_axis.append(i)
for i in x_axis:
y_axis.append(float(totalmeanstd_list.count(i)) / float(len(totalmeanstd_list)))
#finding the probability of each RT to occur=No. of occurence/total occurences.
real_y_axis.append(y_axis[0])
for i in range(len(y_axis)):
real_y_axis.append(float(real_y_axis[i-1]) + float(y_axis[i]))
#to find cumulative just go on adding the current value to previously cumulated value till sum becomes 1 for last entry.
for i in range(len(totalmeanstd_list)):
graph_cumulative.append(totalmeanstd_list[i])
if len(totalmeanstd_list) > 0:
totalmeanstd_list.append(numpy.mean(totalmeanstd_list))
totalmeanstd_list.append(numpy.mean(totalmeanstd_list)+2*numpy.std(totalmeanstd_list))
for i in range(config.MAX_RESPONSE_CONVERSATIONS):
if(len(conversations[i]) != 0):
meanstd_list = build_mean_list(conversations, i, meanstd_list)
conversations[i].append(numpy.mean(meanstd_list))
conversations[i].append(numpy.mean(meanstd_list)+(2*numpy.std(meanstd_list)))
meanstd_list[:] = []
graph_cumulative.sort()
for i in range(graph_cumulative[len(graph_cumulative)-1] + 1):
graph_y_axis.append(graph_cumulative.count(i)) # problem when ti=0 count is unexpectedly large
graph_x_axis.append(i)
#Finally storing the RT values along with their frequencies in a csv file.
rows_rt = zip(graph_x_axis, graph_y_axis)
return rows_rt
