def get_numbers_by_date(filename, destnum=False, log="Start call", date_start=False, date_end=False, quiet=False, legacy_log=False):
calls = {}
phone_nums = ''
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
#print("dest num not = " + destnum)
continue
if line.find(log) != -1:
if phone_num not in calls.keys():
calls[phone_num] = current_date
phone_nums += phone_num + ','
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
if not quiet:
print("Phone numbers by date")
calls_sorted = sorted(calls.iteritems(), key=lambda(k,v): (v,k))
calls_sorted.reverse()
total = 0
for num, date in calls_sorted:
print(num +": "+otalo_utils.date_str(date))
print('numbers are ' + phone_nums)
return calls
def get_online_time(filename, destnum=False, phone_num_filter=False, date_start=False, date_end=False, quiet=False, daily_data=False, transfer_calls=False):
online_time = {}
current_day = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_day:
current_day = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_day
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#Don't flush, assume it's rarely needed
#flush_open_calls(online_time, open_calls, current_day)
open_calls = {}
current_day += timedelta(days=days+1)
if not current_day in online_time:
online_time[current_day] = 0
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys() and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {'start':current_time, 'last':current_time }
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
#flush_open_calls(online_time, open_calls, current_day)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Total online time, by time period (s):")
dates = online_time.keys()
dates.sort()
tot_secs = 0
for date in dates:
online_secs = online_time[date]
tot_secs += online_secs
print(otalo_utils.date_str(date) +"\t"+ str(online_secs))
print('Average online time per period: ' + str(tot_secs/len(online_time)))
return online_time
def get_online_time(filename,
destnum=False,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
daily_data=False,
transfer_calls=False):
online_time = {}
current_day = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_day:
current_day = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_day
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#Don't flush, assume it's rarely needed
#flush_open_calls(online_time, open_calls, current_day)
open_calls = {}
current_day += timedelta(days=days + 1)
if not current_day in online_time:
online_time[current_day] = 0
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys(
) and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {
'start': current_time,
'last': current_time
}
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
#flush_open_calls(online_time, open_calls, current_day)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Total online time, by time period (s):")
dates = online_time.keys()
dates.sort()
tot_secs = 0
for date in dates:
online_secs = online_time[date]
tot_secs += online_secs
print(otalo_utils.date_str(date) + "\t" + str(online_secs))
print('Average online time per period: ' +
str(tot_secs / len(online_time)))
return online_time
def new_and_repeat_callers(filename, destnum=False, log="Start call", date_start=False, date_end=False, quiet=False, legacy_log=False, transfer_calls=False):
calls = {}
phone_nums = ''
already_called = []
current_week_start = False
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
if phone_num not in already_called:
already_called.append(phone_num)
calls[current_week_start]['new'] += 1
else:
calls[current_week_start]['repeat'] += 1
else:
if phone_num not in already_called:
already_called.append(phone_num)
calls[current_week_start] = {'new':1,'repeat':0}
else:
calls[current_week_start] = {'new':0,'repeat':1}
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
if not quiet:
print("Number of "+ log + "'s by new and repeat callers:")
print("Date\tNew Caller Calls\tRepeat Caller Calls\tNew Calls Rate")
dates = calls.keys()
dates.sort()
total = 0
for date in dates:
total += calls[date]['new'] + calls[date]['repeat']
print(otalo_utils.date_str(date) +"\t"+str(calls[date]['new'])+"\t"+str(calls[date]['repeat'])+"\t"+str(float(calls[date]['new']) / float(calls[date]['repeat'])))
print("total is " + str(total))
print ("number of unique callers is " + str(len(already_called)))
return calls
def get_calls(filename, destnum=False, log="Start call", phone_num_filter=False, date_start=False, date_end=False, quiet=False, legacy_log=False, transfer_calls=False, daily_data=False):
calls = {}
nums = []
current_week_start = 0
total = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not legacy_log:
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
current_week_start += timedelta(days=days+1)
calls[current_week_start] = 0
#print('found3 ' + phone_num)
if phone_num not in nums:
nums.append(phone_num)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of "+ log + "'s, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
total += calls[date]
print(otalo_utils.date_str(date) +"\t"+str(calls[date]))
print("total is " + str(total))
print ("number of unique callers is " + str(len(nums)))
return calls
def get_calls_by_feature(filename, destnum, phone_num_filter=0, date_start=False, date_end=False, quiet=False, legacy_log=False):
features = {}
feature_names = []
current_week_start = 0
feature_chosen = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if not legacy_log and destnum and destnum.find(dest) == -1:
continue
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if not current_week_start in features:
features[current_week_start] = {}
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
del open_calls[phone_num]
# add new call with no feature access yet
open_calls[phone_num] = False
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
del open_calls[phone_num]
elif phone_num in open_calls:
feature_chosen = open_calls[phone_num]
feature = line[line.rfind('/')+1:line.find('.wav')]
if feature == "okyourreplies" or feature == "okplay_all" or feature == "okplay" or feature == "okrecord":
if feature not in feature_names:
feature_names.append(feature)
if feature in features[current_week_start]:
features[current_week_start][feature] += 1
else:
features[current_week_start][feature] = 1
elif feature == "okyouwant_pre" or feature == "okplaytag_pre":
# on the next go-around, look for the feature
open_calls[phone_num] = True
elif feature_chosen:
if feature not in feature_names:
feature_names.append(feature)
if feature in features[current_week_start]:
features[current_week_start][feature] += 1
else:
features[current_week_start][feature] = 1
open_calls[phone_num] = False
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of calls by feature, by week:")
header = "\t"
for name in feature_names:
header += name+"\t"
print(header)
dates = features.keys()
dates.sort()
for date in dates:
row = otalo_utils.date_str(date) +"\t"
for name in feature_names:
if name in features[date]:
row += str(features[date][name]) + "\t"
else:
row += "0\t"
print(row)
def get_num_qna(filename, line, forum=False, date_start=False, date_end=False, phone_num_filter=False, quiet=False):
qna = {}
oneweek = timedelta(days=7)
if not date_start:
if forum:
date_start = Message_forum.objects.filter(forum=forum).aggregate(Min('message__date'))
else:
date_start = Message_forum.objects.filter(forum__line=line).aggregate(Min('message__date'))
date_start = date_start[date_start.keys()[0]]
if not date_end:
if forum:
date_end = Message_forum.objects.filter(forum=forum).aggregate(Max('message__date'))
else:
date_end = Message_forum.objects.filter(forum__line=line).aggregate(Max('message__date'))
date_end = date_end[date_end.keys()[0]]
transfer_recordings = get_recordings(filename, destnum=line.number, phone_num_filter=phone_num_filter, date_start=date_start, date_end=date_end, transfer_calls=True)
tot_turn_around = 0
num_app_responses = 0
while(date_start < date_end):
if forum:
this_weeks_msgs = Message_forum.objects.filter(forum=forum, message__date__gte=date_start, message__date__lt=date_start+oneweek)
else:
this_weeks_msgs = Message_forum.objects.filter(forum__line=line, message__date__gte=date_start, message__date__lt=date_start+oneweek)
if phone_num_filter:
this_weeks_msgs = this_weeks_msgs.filter(message__user__number__in=phone_num_filter)
this_weeks_msgs = this_weeks_msgs.exclude(message__file__in=transfer_recordings)
# for msg in this_weeks_msgs:
# print(str(msg))
questions = this_weeks_msgs.filter(message__lft=1)
n_questions = questions.count()
n_qs_unique = questions.values('message__user').distinct().count()
n_qs_approved = questions.filter(status=Message_forum.STATUS_APPROVED).count()
responses = this_weeks_msgs.filter(message__lft__gt=1)
n_responses = responses.count()
n_rs_unique = responses.values('message__user').distinct().count()
approved_responses = responses.filter(status=Message_forum.STATUS_APPROVED)
n_rs_approved = approved_responses.count()
response_files = responses.values('message__file')
response_files = [pair.values()[0] for pair in response_files]
n_rs_bcast = Input.objects.filter(input__in=response_files).count()
turn_around_time = 0
for resp in approved_responses:
#print('resp is '+str(resp))
fullthread = Message.objects.filter(Q(thread=resp.message.thread) | Q(pk=resp.message.thread.pk))
ancestors = fullthread.filter(lft__lt=resp.message.lft, rgt__gt=resp.message.rgt).order_by('-lft')
# ignore buggy threads
if ancestors:
parent = ancestors[0]
turn_around_time += (resp.message.date-parent.date).seconds/3600
avg_turn_around_time = 'N/A'
if turn_around_time > 0:
avg_turn_around_time = float(turn_around_time)/float(approved_responses.count())
tot_turn_around += turn_around_time
num_app_responses += approved_responses.count()
responders = User.objects.filter(forum__line=line).distinct()
responder_responses = responses.filter(message__user__in=responders)
n_responders = responder_responses.count()
n_responders_unique = responder_responses.values('message__user').distinct().count()
n_responders_approved = responder_responses.filter(status=Message_forum.STATUS_APPROVED).count()
responder_rate=0
if n_responses>0:
responder_rate = float(n_responders)/float(n_responses)
qna[date_start] = [n_questions, n_qs_approved, n_qs_unique, n_responses, n_rs_approved,n_rs_unique,n_rs_bcast,avg_turn_around_time, n_responders,n_responders_approved,n_responders_unique,responder_rate]
#print("adding to "+otalo_utils.date_str(date_start)+": " +str(qna[date_start]))
date_start += oneweek
if not quiet:
print("Number of questions and responses, by week:")
print("Date\ttot questions\ttot approved\tunique posters\ttotal responses\tapproved\tunique\tbcast response\tavg turn-around-time (h)\tresponder msgs\tapproved\tunique\trate")
dates = qna.keys()
dates.sort()
for date in dates:
row = otalo_utils.date_str(date) +"\t"
stats = qna[date]
for stat in stats:
row += str(stat) + "\t"
print(row)
avg_turn_around = float(tot_turn_around)/float(num_app_responses)
print('overall turn-around: '+str(avg_turn_around))
return qna
def main():
if len(sys.argv) < 1:
print("Wrong")
else:
lineid = sys.argv[1]
f = settings.INBOUND_LOG_ROOT + lineid + '.log'
now = datetime.now()
# reset to beginning of day
today = datetime(year=now.year, month=now.month, day=now.day)
oneday = timedelta(days=1)
#today = today - oneday
line = Line.objects.get(pk=int(lineid))
print("<html>")
print("<div> Below are basic usage statistics for " + str(line.name) + " over the last four days, starting with today. </div>")
# calls
print("<div><h4>Number of Incoming Calls</h4></div>")
print("<table>")
for i in range (0,4):
calls = num_calls.get_calls(filename=f, destnum=str(line.number), date_start=today-oneday*i, date_end=today-oneday*(i-1), quiet=True,transfer_calls='INBOUND_ONLY')
ncalls = calls[calls.keys()[0]] if calls else 0
print("<tr>")
print("<td width='100px'>"+otalo_utils.date_str(today-oneday*i)+"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>"+str(ncalls)+"</td>")
print("</tr>")
print("</table>")
# calls by caller
print("<div><h4>Who called today?</h4></div>")
print("<table>")
calls = stats_by_phone_num.get_calls_by_number(filename=f, destnum=str(line.number), date_start=today, date_end=today+oneday, quiet=True,transfer_calls='INBOUND_ONLY')
for num, tot in calls:
print("<tr>")
print("<td width='100px'>"+num+"</td>")
print("<td>"+str(tot)+"</td>")
print("</tr>")
print("</table>")
# feature access
print("<div><h4>Today's calls by number of feature accesses</h4></div>")
print("<table>")
calls = num_calls.get_features_within_call(filename=f, destnum=str(line.number), date_start=today, date_end=today+oneday, quiet=True, transfer_calls='INBOUND_ONLY')
feature_calls = calls[calls.keys()[0]] if calls else {}
features_hist = {}
for call in feature_calls:
features_tot = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
features_tot += call[feature]
if features_tot in features_hist:
features_hist[features_tot] += 1
else:
features_hist[features_tot] = 1
sorted_items = features_hist.items()
sorted_items.sort()
for accesses, ncalls in sorted_items:
print("<tr>")
print("<td width='80px'>"+str(accesses)+" accesses</td>")
print("<td>"+str(ncalls)+"</td>")
print("</tr>")
print("</table>")
# call duration
durations = call_duration.get_call_durations(filename=f, destnum=str(line.number), date_start=today, date_end=today+oneday, quiet=True, transfer_calls='INBOUND_ONLY')
durs_by_call = durations[durations.keys()[0]] if durations else {}
durs = [dur[1].seconds for dur in durs_by_call]
avg_dur = str(sum(durs)/len(durs)) if durs else "n/a"
print("<br/><div>")
print("<b>Average call duration (secs):</b> ")
print(avg_dur)
print("</div>")
# questions
print("<div><h4>Number of Original Messages</h4></div>")
print("<table>")
for i in range (0,4):
msgs = Message_forum.objects.filter(message__date__gte=today-oneday*i, message__date__lt=oneday+today-oneday*i, forum__line=line, message__lft=1)
n_approved = msgs.filter(status = Message_forum.STATUS_APPROVED).count()
print("<tr>")
print("<td width='100px'>"+otalo_utils.date_str(today-oneday*i)+"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>"+str(msgs.count())+" (" + str(n_approved) + " approved) </td>")
print("</tr>")
print("</table>")
# answers
print("<div><h4>Number of Responses</h4></div>")
print("<table>")
for i in range (0,4):
msgs = Message_forum.objects.filter(message__date__gte=today-oneday*i, message__date__lt=oneday+today-oneday*i, forum__line=line, message__lft__gt=1)
n_approved = msgs.filter(status = Message_forum.STATUS_APPROVED).count()
print("<tr>")
print("<td width='100px'>"+otalo_utils.date_str(today-oneday*i)+"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>"+str(msgs.count())+" (" + str(n_approved) + " approved) </td>")
print("</tr>")
print("</table>")
# answers by responder
print("<div><h4>Number of Responses by Responder (last 7 days)</h4></div>")
print("<table>")
print("<tr>")
print("<td width='100px'><u>Responder</u></td><td width='100px'><u>Assigned</u></td><td width='100px'><u>Responses</u></td>")
print("</tr>")
oneweek = timedelta(days=7)
# get responders ordered by number of responses
responders = User.objects.filter(forum__line=line).distinct()
responder_counts = {}
for responder in responders:
# get active assignments only
nassigned = Message_responder.objects.filter(user=responder, assign_date__gte=today-oneweek, passed_date__isnull=True, listens__lte=LISTEN_THRESH).count()
# count the answers whether they have been approved or not
nresponses = Message_forum.objects.filter(message__date__gte=today-oneweek, message__user=responder, message__lft__gt=1).count()
responder_counts[responder] = [nassigned, nresponses]
# sort by number of responses
responder_counts = sorted(responder_counts.iteritems(), key=lambda(k,v): v[1], reverse=True)
for responder,counts in responder_counts:
print("<tr>")
print("<td>"+responder.name+"</td>")
print("<td>"+str(counts[0])+"</td>")
print("<td>"+str(counts[1])+"</td>")
print("</tr>")
print("</table>")
# Answer Calls
print("<div><h4>Answer Calls</h4></div>")
print("<table>")
print("<tr>")
print("<td width='250px'><u>Recipient</u></td>")
print("<td width='150px'><u>Attempted</u></td>")
print("<td width='50px'><u>Pickup?</u></td>")
print("</tr>")
answercalls = Call.objects.filter(survey__name__contains=Survey.ANSWER_CALL_DESIGNATOR, survey__number__in=[line.number, line.outbound_number], date__gte=today, date__lt=today+oneday, priority=1)
for call in answercalls:
u = User.objects.filter(number=call.subject.number)
name = call.subject.number
if bool(u):
u = u[0]
if u.name:
name = u.name + ' (' + u.number + ')'
othercalls = Call.objects.filter(survey=call.survey, priority__gt=1).order_by('date')
times = call.date.strftime("%H:%M")
for c in othercalls:
times += ','+c.date.strftime("%H:%M")
complete = call.complete or bool(othercalls.filter(complete=True))
complete = 'Yes' if complete else 'No'
print("<tr>")
print("<td>"+name+"</td>")
print("<td>"+times+"</td>")
print("<td>"+complete+"</td>")
print("</tr>")
print("</table>")
print("<div><h4>Today's Broadcasts</h4></div>")
print("<table>")
print("<tr>")
print("<td width='500px'><u>Broadcasts</u></td><td width='150px'><u>Recipients</u></td><td width='100px'><u>Matured</u></td>")
print("</tr>")
# Announcements
actives = Survey.objects.filter(broadcast=True, number__in=[line.number, line.outbound_number], call__date__gt=today, call__date__lt=today+oneday).exclude(name__contains=Survey.ANSWER_CALL_DESIGNATOR).order_by('-id').distinct()
# For each survey, get the number of subjects and calls
for survey in actives:
calls = Call.objects.filter(survey=survey)
n_subjects = survey.subjects.all().count()
calls_attempted = calls.filter(date__gte=today, date__lt=today+oneday).count()
calls_completed = calls.filter(date__gte=today, date__lt=today+oneday, complete=True).count()
print("<tr>")
print("<td>"+survey.name+"</td>")
print("<td>"+str(n_subjects)+" (" + str(calls_attempted) +" attempts)</td>")
print("<td>"+str(calls_completed)+"</td>")
print("</tr>")
print("</table>")
print("</html>")
def get_calls(filename,
destnum=False,
log="Start call",
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
legacy_log=False,
transfer_calls=False,
daily_data=False):
calls = {}
nums = []
current_week_start = 0
total = 0
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not legacy_log:
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
current_week_start += timedelta(days=days + 1)
calls[current_week_start] = 0
#print('found3 ' + phone_num)
if phone_num not in nums:
nums.append(phone_num)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of " + log + "'s, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
total += calls[date]
print(otalo_utils.date_str(date) + "\t" + str(calls[date]))
print("total is " + str(total))
print("number of unique callers is " + str(len(nums)))
return calls
def get_num_qna(filename,
line,
forum=False,
date_start=False,
date_end=False,
phone_num_filter=False,
quiet=False):
qna = {}
oneweek = timedelta(days=7)
if not date_start:
if forum:
date_start = Message_forum.objects.filter(forum=forum).aggregate(
Min('message__date'))
else:
date_start = Message_forum.objects.filter(
forum__line=line).aggregate(Min('message__date'))
date_start = date_start[date_start.keys()[0]]
if not date_end:
if forum:
date_end = Message_forum.objects.filter(forum=forum).aggregate(
Max('message__date'))
else:
date_end = Message_forum.objects.filter(
forum__line=line).aggregate(Max('message__date'))
date_end = date_end[date_end.keys()[0]]
transfer_recordings = get_recordings(filename,
destnum=line.number,
phone_num_filter=phone_num_filter,
date_start=date_start,
date_end=date_end,
transfer_calls=True)
tot_turn_around = 0
num_app_responses = 0
while (date_start < date_end):
if forum:
this_weeks_msgs = Message_forum.objects.filter(
forum=forum,
message__date__gte=date_start,
message__date__lt=date_start + oneweek)
else:
this_weeks_msgs = Message_forum.objects.filter(
forum__line=line,
message__date__gte=date_start,
message__date__lt=date_start + oneweek)
if phone_num_filter:
this_weeks_msgs = this_weeks_msgs.filter(
message__user__number__in=phone_num_filter)
this_weeks_msgs = this_weeks_msgs.exclude(
message__file__in=transfer_recordings)
# for msg in this_weeks_msgs:
# print(str(msg))
questions = this_weeks_msgs.filter(message__lft=1)
n_questions = questions.count()
n_qs_unique = questions.values('message__user').distinct().count()
n_qs_approved = questions.filter(
status=Message_forum.STATUS_APPROVED).count()
responses = this_weeks_msgs.filter(message__lft__gt=1)
n_responses = responses.count()
n_rs_unique = responses.values('message__user').distinct().count()
approved_responses = responses.filter(
status=Message_forum.STATUS_APPROVED)
n_rs_approved = approved_responses.count()
response_files = responses.values('message__file')
response_files = [pair.values()[0] for pair in response_files]
n_rs_bcast = Input.objects.filter(input__in=response_files).count()
turn_around_time = 0
for resp in approved_responses:
#print('resp is '+str(resp))
fullthread = Message.objects.filter(
Q(thread=resp.message.thread) | Q(pk=resp.message.thread.pk))
ancestors = fullthread.filter(
lft__lt=resp.message.lft,
rgt__gt=resp.message.rgt).order_by('-lft')
# ignore buggy threads
if ancestors:
parent = ancestors[0]
turn_around_time += (resp.message.date -
parent.date).seconds / 3600
avg_turn_around_time = 'N/A'
if turn_around_time > 0:
avg_turn_around_time = float(turn_around_time) / float(
approved_responses.count())
tot_turn_around += turn_around_time
num_app_responses += approved_responses.count()
responders = User.objects.filter(forum__line=line).distinct()
responder_responses = responses.filter(message__user__in=responders)
n_responders = responder_responses.count()
n_responders_unique = responder_responses.values(
'message__user').distinct().count()
n_responders_approved = responder_responses.filter(
status=Message_forum.STATUS_APPROVED).count()
responder_rate = 0
if n_responses > 0:
responder_rate = float(n_responders) / float(n_responses)
qna[date_start] = [
n_questions, n_qs_approved, n_qs_unique, n_responses,
n_rs_approved, n_rs_unique, n_rs_bcast, avg_turn_around_time,
n_responders, n_responders_approved, n_responders_unique,
responder_rate
]
#print("adding to "+otalo_utils.date_str(date_start)+": " +str(qna[date_start]))
date_start += oneweek
if not quiet:
print("Number of questions and responses, by week:")
print(
"Date\ttot questions\ttot approved\tunique posters\ttotal responses\tapproved\tunique\tbcast response\tavg turn-around-time (h)\tresponder msgs\tapproved\tunique\trate"
)
dates = qna.keys()
dates.sort()
for date in dates:
row = otalo_utils.date_str(date) + "\t"
stats = qna[date]
for stat in stats:
row += str(stat) + "\t"
print(row)
avg_turn_around = float(tot_turn_around) / float(num_app_responses)
print('overall turn-around: ' + str(avg_turn_around))
return qna
def get_calls_by_feature(filename,
destnum,
phone_num_filter=0,
date_start=False,
date_end=False,
quiet=False,
legacy_log=False):
features = {}
feature_names = []
current_week_start = 0
feature_chosen = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
if not legacy_log and destnum and destnum.find(dest) == -1:
continue
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
if not current_week_start in features:
features[current_week_start] = {}
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
del open_calls[phone_num]
# add new call with no feature access yet
open_calls[phone_num] = False
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
del open_calls[phone_num]
elif phone_num in open_calls:
feature_chosen = open_calls[phone_num]
feature = line[line.rfind('/') + 1:line.find('.wav')]
if feature == "okyourreplies" or feature == "okplay_all" or feature == "okplay" or feature == "okrecord":
if feature not in feature_names:
feature_names.append(feature)
if feature in features[current_week_start]:
features[current_week_start][feature] += 1
else:
features[current_week_start][feature] = 1
elif feature == "okyouwant_pre" or feature == "okplaytag_pre":
# on the next go-around, look for the feature
open_calls[phone_num] = True
elif feature_chosen:
if feature not in feature_names:
feature_names.append(feature)
if feature in features[current_week_start]:
features[current_week_start][feature] += 1
else:
features[current_week_start][feature] = 1
open_calls[phone_num] = False
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of calls by feature, by week:")
header = "\t"
for name in feature_names:
header += name + "\t"
print(header)
dates = features.keys()
dates.sort()
for date in dates:
row = otalo_utils.date_str(date) + "\t"
for name in feature_names:
if name in features[date]:
row += str(features[date][name]) + "\t"
else:
row += "0\t"
print(row)
def main():
if len(sys.argv) < 1:
print("Wrong")
else:
lineid = sys.argv[1]
f = settings.INBOUND_LOG_ROOT + lineid + '.log'
now = datetime.now()
# reset to beginning of day
today = datetime(year=now.year, month=now.month, day=now.day)
oneday = timedelta(days=1)
#today = today - oneday
line = Line.objects.get(pk=int(lineid))
print("<html>")
print("<div> Below are basic usage statistics for " + str(line.name) +
" over the last four days, starting with today. </div>")
# calls
print("<div><h4>Number of Incoming Calls</h4></div>")
print("<table>")
for i in range(0, 4):
calls = num_calls.get_calls(filename=f,
destnum=str(line.number),
date_start=today - oneday * i,
date_end=today - oneday * (i - 1),
quiet=True,
transfer_calls='INBOUND_ONLY')
ncalls = calls[calls.keys()[0]] if calls else 0
print("<tr>")
print("<td width='100px'>" + otalo_utils.date_str(today - oneday * i) +
"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>" + str(ncalls) + "</td>")
print("</tr>")
print("</table>")
# calls by caller
print("<div><h4>Who called today?</h4></div>")
print("<table>")
calls = stats_by_phone_num.get_calls_by_number(
filename=f,
destnum=str(line.number),
date_start=today,
date_end=today + oneday,
quiet=True,
transfer_calls='INBOUND_ONLY')
for num, tot in calls:
print("<tr>")
print("<td width='100px'>" + num + "</td>")
print("<td>" + str(tot) + "</td>")
print("</tr>")
print("</table>")
# feature access
print("<div><h4>Today's calls by number of feature accesses</h4></div>")
print("<table>")
calls = num_calls.get_features_within_call(filename=f,
destnum=str(line.number),
date_start=today,
date_end=today + oneday,
quiet=True,
transfer_calls='INBOUND_ONLY')
feature_calls = calls[calls.keys()[0]] if calls else {}
features_hist = {}
for call in feature_calls:
features_tot = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
features_tot += call[feature]
if features_tot in features_hist:
features_hist[features_tot] += 1
else:
features_hist[features_tot] = 1
sorted_items = features_hist.items()
sorted_items.sort()
for accesses, ncalls in sorted_items:
print("<tr>")
print("<td width='80px'>" + str(accesses) + " accesses</td>")
print("<td>" + str(ncalls) + "</td>")
print("</tr>")
print("</table>")
# call duration
durations = call_duration.get_call_durations(filename=f,
destnum=str(line.number),
date_start=today,
date_end=today + oneday,
quiet=True,
transfer_calls='INBOUND_ONLY')
durs_by_call = durations[durations.keys()[0]] if durations else {}
durs = [dur[1].seconds for dur in durs_by_call]
avg_dur = str(sum(durs) / len(durs)) if durs else "n/a"
print("<br/><div>")
print("<b>Average call duration (secs):</b> ")
print(avg_dur)
print("</div>")
# questions
print("<div><h4>Number of Original Messages</h4></div>")
print("<table>")
for i in range(0, 4):
msgs = Message_forum.objects.filter(
message__date__gte=today - oneday * i,
message__date__lt=oneday + today - oneday * i,
forum__line=line,
message__lft=1)
n_approved = msgs.filter(status=Message_forum.STATUS_APPROVED).count()
print("<tr>")
print("<td width='100px'>" + otalo_utils.date_str(today - oneday * i) +
"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>" + str(msgs.count()) + " (" + str(n_approved) +
" approved) </td>")
print("</tr>")
print("</table>")
# answers
print("<div><h4>Number of Responses</h4></div>")
print("<table>")
for i in range(0, 4):
msgs = Message_forum.objects.filter(
message__date__gte=today - oneday * i,
message__date__lt=oneday + today - oneday * i,
forum__line=line,
message__lft__gt=1)
n_approved = msgs.filter(status=Message_forum.STATUS_APPROVED).count()
print("<tr>")
print("<td width='100px'>" + otalo_utils.date_str(today - oneday * i) +
"</td>")
# since a single day's calls can only be bucketed into a single week
print("<td>" + str(msgs.count()) + " (" + str(n_approved) +
" approved) </td>")
print("</tr>")
print("</table>")
# answers by responder
print("<div><h4>Number of Responses by Responder (last 7 days)</h4></div>")
print("<table>")
print("<tr>")
print(
"<td width='100px'><u>Responder</u></td><td width='100px'><u>Assigned</u></td><td width='100px'><u>Responses</u></td>"
)
print("</tr>")
oneweek = timedelta(days=7)
# get responders ordered by number of responses
responders = User.objects.filter(forum__line=line).distinct()
responder_counts = {}
for responder in responders:
# get active assignments only
nassigned = Message_responder.objects.filter(
user=responder,
assign_date__gte=today - oneweek,
passed_date__isnull=True,
listens__lte=LISTEN_THRESH).count()
# count the answers whether they have been approved or not
nresponses = Message_forum.objects.filter(message__date__gte=today -
oneweek,
message__user=responder,
message__lft__gt=1).count()
responder_counts[responder] = [nassigned, nresponses]
# sort by number of responses
responder_counts = sorted(responder_counts.iteritems(),
key=lambda (k, v): v[1],
reverse=True)
for responder, counts in responder_counts:
print("<tr>")
print("<td>" + responder.name + "</td>")
print("<td>" + str(counts[0]) + "</td>")
print("<td>" + str(counts[1]) + "</td>")
print("</tr>")
print("</table>")
# Answer Calls
print("<div><h4>Answer Calls</h4></div>")
print("<table>")
print("<tr>")
print("<td width='250px'><u>Recipient</u></td>")
print("<td width='150px'><u>Attempted</u></td>")
print("<td width='50px'><u>Pickup?</u></td>")
print("</tr>")
answercalls = Call.objects.filter(
survey__name__contains=Survey.ANSWER_CALL_DESIGNATOR,
survey__number__in=[line.number, line.outbound_number],
date__gte=today,
date__lt=today + oneday,
priority=1)
for call in answercalls:
u = User.objects.filter(number=call.subject.number)
name = call.subject.number
if bool(u):
u = u[0]
if u.name:
name = u.name + ' (' + u.number + ')'
othercalls = Call.objects.filter(survey=call.survey,
priority__gt=1).order_by('date')
times = call.date.strftime("%H:%M")
for c in othercalls:
times += ',' + c.date.strftime("%H:%M")
complete = call.complete or bool(othercalls.filter(complete=True))
complete = 'Yes' if complete else 'No'
print("<tr>")
print("<td>" + name + "</td>")
print("<td>" + times + "</td>")
print("<td>" + complete + "</td>")
print("</tr>")
print("</table>")
print("<div><h4>Today's Broadcasts</h4></div>")
print("<table>")
print("<tr>")
print(
"<td width='500px'><u>Broadcasts</u></td><td width='150px'><u>Recipients</u></td><td width='100px'><u>Matured</u></td>"
)
print("</tr>")
# Announcements
actives = Survey.objects.filter(
broadcast=True,
number__in=[line.number, line.outbound_number],
call__date__gt=today,
call__date__lt=today +
oneday).exclude(name__contains=Survey.ANSWER_CALL_DESIGNATOR).order_by(
'-id').distinct()
# For each survey, get the number of subjects and calls
for survey in actives:
calls = Call.objects.filter(survey=survey)
n_subjects = survey.subjects.all().count()
calls_attempted = calls.filter(date__gte=today,
date__lt=today + oneday).count()
calls_completed = calls.filter(date__gte=today,
date__lt=today + oneday,
complete=True).count()
print("<tr>")
print("<td>" + survey.name + "</td>")
print("<td>" + str(n_subjects) + " (" + str(calls_attempted) +
" attempts)</td>")
print("<td>" + str(calls_completed) + "</td>")
print("</tr>")
print("</table>")
print("</html>")