def get_numbers_by_date(filename, destnum=False, log="Start call", date_start=False, date_end=False, quiet=False, legacy_log=False):
calls = {}
phone_nums = ''
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
#print("dest num not = " + destnum)
continue
if line.find(log) != -1:
if phone_num not in calls.keys():
calls[phone_num] = current_date
phone_nums += phone_num + ','
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
if not quiet:
print("Phone numbers by date")
calls_sorted = sorted(calls.iteritems(), key=lambda(k,v): (v,k))
calls_sorted.reverse()
total = 0
for num, date in calls_sorted:
print(num +": "+otalo_utils.date_str(date))
print('numbers are ' + phone_nums)
return calls
def get_guj_nums_only(filename, log="welcome", quiet=False, legacy_log=False):
calls = {}
phone_nums = ''
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if line.find('guj') == -1:
continue
if line.find(log) != -1:
if phone_num in calls.keys():
calls[phone_num] += 1
else:
calls[phone_num] = 1
phone_nums += phone_num + ','
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
if not quiet:
print("Number of "+ log + "'s by phone number:")
calls_sorted = sorted(calls.iteritems(), key=lambda(k,v): (v,k))
calls_sorted.reverse()
total = 0
for num, tot in calls_sorted:
total += tot
print(num +": "+str(tot))
print('total is ' + str(total))
print('numbers are ' + phone_nums)
return calls
def new_and_repeat_callers(filename, destnum=False, log="Start call", date_start=False, date_end=False, quiet=False, legacy_log=False, transfer_calls=False):
calls = {}
phone_nums = ''
already_called = []
current_week_start = False
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
if phone_num not in already_called:
already_called.append(phone_num)
calls[current_week_start]['new'] += 1
else:
calls[current_week_start]['repeat'] += 1
else:
if phone_num not in already_called:
already_called.append(phone_num)
calls[current_week_start] = {'new':1,'repeat':0}
else:
calls[current_week_start] = {'new':0,'repeat':1}
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
if not quiet:
print("Number of "+ log + "'s by new and repeat callers:")
print("Date\tNew Caller Calls\tRepeat Caller Calls\tNew Calls Rate")
dates = calls.keys()
dates.sort()
total = 0
for date in dates:
total += calls[date]['new'] + calls[date]['repeat']
print(otalo_utils.date_str(date) +"\t"+str(calls[date]['new'])+"\t"+str(calls[date]['repeat'])+"\t"+str(float(calls[date]['new']) / float(calls[date]['repeat'])))
print("total is " + str(total))
print ("number of unique callers is " + str(len(already_called)))
return calls
def get_features_within_call(filename,
destnum,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
transfer_calls=False):
features = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
if not current_week_start in features:
features[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
features[current_week_start].append(call)
dur = call['last'] - call['start']
n_features = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
n_features += call[feature]
if n_features in durations:
durations[n_features].append(dur)
else:
durations[n_features] = [dur]
del open_calls[phone_num]
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {
'order': '',
'feature_chosen': False,
'start': current_date,
'last': current_date
}
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
features[current_week_start].append(call)
dur = current_date - call['start']
n_features = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
n_features += call[feature]
if n_features in durations:
durations[n_features].append(dur)
else:
durations[n_features] = [dur]
del open_calls[phone_num]
elif phone_num in open_calls:
call = open_calls[phone_num]
feature = line[line.rfind('/') + 1:line.find('.wav')]
if feature == "okyourreplies" or feature == "okplay_all" or feature == "okplay" or feature == "okrecord":
if feature in call:
call[feature] += 1
else:
call[feature] = 1
call['order'] += feature + ','
elif feature == "okyouwant_pre" or feature == "okplaytag_pre":
# on the next go-around, look for the feature
call['feature_chosen'] = True
elif call['feature_chosen']:
if feature in call:
call[feature] += 1
else:
call[feature] = 1
call['order'] += feature + ','
call['feature_chosen'] = False
call['last'] = current_date
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Histogram of number of features accessed within call:")
print("Features\tCalls\tAvg Duration")
dates = features.keys()
dates.sort()
features_hist = {}
for date in dates:
for call in features[date]:
features_tot = 0
for feature in call:
if feature != 'feature_chosen' and feature != 'order' and feature != 'start' and feature != 'last':
features_tot += call[feature]
if features_tot in features_hist:
features_hist[features_tot] += 1
else:
features_hist[features_tot] = 1
sorted_items = features_hist.items()
sorted_items.sort()
for num, tot in sorted_items:
durs_by_call = durations[num]
durs = [dur.seconds for dur in durs_by_call]
print(
str(num) + "\t" + str(tot) + "\t" + str(sum(durs) / len(durs)))
print("Average features accessed within call, by week:")
for date in dates:
total_features = 0
num_calls = len(features[date])
for call in features[date]:
for feature in call:
if feature != 'feature_chosen' and feature != 'order' and feature != 'start' and feature != 'last':
total_features += call[feature]
print(
date.strftime('%Y-%m-%d') + "\t" +
str(float(total_features) / float(num_calls)))
return features
def get_online_time(filename,
destnum=False,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
daily_data=False,
transfer_calls=False):
online_time = {}
current_day = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_day:
current_day = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_day
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#Don't flush, assume it's rarely needed
#flush_open_calls(online_time, open_calls, current_day)
open_calls = {}
current_day += timedelta(days=days + 1)
if not current_day in online_time:
online_time[current_day] = 0
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys(
) and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {
'start': current_time,
'last': current_time
}
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
#flush_open_calls(online_time, open_calls, current_day)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Total online time, by time period (s):")
dates = online_time.keys()
dates.sort()
tot_secs = 0
for date in dates:
online_secs = online_time[date]
tot_secs += online_secs
print(otalo_utils.date_str(date) + "\t" + str(online_secs))
print('Average online time per period: ' +
str(tot_secs / len(online_time)))
return online_time
def get_online_time(filename, destnum=False, phone_num_filter=False, date_start=False, date_end=False, quiet=False, daily_data=False, transfer_calls=False):
online_time = {}
current_day = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_day:
current_day = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_day
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#Don't flush, assume it's rarely needed
#flush_open_calls(online_time, open_calls, current_day)
open_calls = {}
current_day += timedelta(days=days+1)
if not current_day in online_time:
online_time[current_day] = 0
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys() and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {'start':current_time, 'last':current_time }
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
online_time[current_day] += dur.seconds
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
#flush_open_calls(online_time, open_calls, current_day)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Total online time, by time period (s):")
dates = online_time.keys()
dates.sort()
tot_secs = 0
for date in dates:
online_secs = online_time[date]
tot_secs += online_secs
print(otalo_utils.date_str(date) +"\t"+ str(online_secs))
print('Average online time per period: ' + str(tot_secs/len(online_time)))
return online_time
def get_listens_within_call(filename, destnum, phone_num_filter=False, date_start=False, date_end=False, quiet=False, transfer_calls=False, daily_data=False):
listens = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#flush all open calls
open_calls = {}
current_week_start += timedelta(days=days+1)
if not current_week_start in listens:
listens[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
n_listens = call['listens']
listens[current_week_start].append(n_listens)
dur = call['last'] - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
open_calls[phone_num] = {'start':current_date, 'last':current_date, 'listens':0 }
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
n_listens = call['listens']
listens[current_week_start].append(n_listens)
dur = current_date - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
else:
if line.find("Stream") != -1:
if phone_num in open_calls:
open_calls[phone_num]['listens'] += 1
if phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_date
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
#print("IndexError: " + line)
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Histogram of number of listens within call")
print("Listens\tCalls\tAvg Duration")
dates = listens.keys()
dates.sort()
listens_hist = {}
for date in dates:
for tot in listens[date]:
if tot in listens_hist:
listens_hist[tot] += 1
else:
listens_hist[tot] = 1
sorted_items = listens_hist.items()
sorted_items.sort()
for num, tot in sorted_items:
durs_by_call = durations[num]
durs = [dur.seconds for dur in durs_by_call]
print(str(num)+"\t"+str(tot)+"\t"+str(sum(durs)/len(durs)))
print("Average and total listens within call, by week:")
for date in dates:
total_listens = 0
num_calls = len(listens[date])
for tot in listens[date]:
total_listens += tot
if total_listens == 0:
avg = 0
else:
avg = float(total_listens)/float(num_calls)
print(date.strftime('%Y-%m-%d') + "\t" + str(avg) + "\t" + str(total_listens))
return listens
def get_num_listens(filename, destnum, phone_num_filter=False, date_start=False, date_end=False, quiet=False, transfer_calls=False):
listens = {}
durations = {}
current_week_start = 0
open_calls = {}
if phone_num_filter:
# every phone number has an entry
for num in phone_num_filter:
listens[num] = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
n_listens = call['listens']
if phone_num in listens:
listens[phone_num] += n_listens
else:
listens[phone_num] = n_listens
dur = call['last'] - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
open_calls[phone_num] = {'start':current_date, 'last':current_date, 'listens':0 }
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
n_listens = call['listens']
if phone_num in listens:
listens[phone_num] += n_listens
else:
listens[phone_num] = n_listens
dur = current_date - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
else:
if line.find("Stream") != -1:
if phone_num in open_calls:
open_calls[phone_num]['listens'] += 1
if phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_date
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
#print("IndexError: " + line)
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of listens, by number:")
for num, tot in listens.items():
print(str(num) + "\t" + str(tot))
return listens
def get_calls(filename,
destnum=False,
log="Start call",
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
legacy_log=False,
transfer_calls=False,
daily_data=False):
calls = {}
nums = []
current_week_start = 0
total = 0
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not legacy_log:
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
current_week_start += timedelta(days=days + 1)
calls[current_week_start] = 0
#print('found3 ' + phone_num)
if phone_num not in nums:
nums.append(phone_num)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of " + log + "'s, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
total += calls[date]
print(otalo_utils.date_str(date) + "\t" + str(calls[date]))
print("total is " + str(total))
print("number of unique callers is " + str(len(nums)))
return calls
def get_features_within_call(filename, destnum, phone_num_filter=False, date_start=False, date_end=False, quiet=False, transfer_calls=False):
features = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if not current_week_start in features:
features[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
features[current_week_start].append(call)
dur = call['last'] - call['start']
n_features = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
n_features += call[feature]
if n_features in durations:
durations[n_features].append(dur)
else:
durations[n_features] = [dur]
del open_calls[phone_num]
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {'order':'','feature_chosen':False,'start':current_date,'last':current_date}
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
features[current_week_start].append(call)
dur = current_date - call['start']
n_features = 0
for feature in call:
if feature != 'order' and feature != 'feature_chosen' and feature != 'start' and feature != 'last':
n_features += call[feature]
if n_features in durations:
durations[n_features].append(dur)
else:
durations[n_features] = [dur]
del open_calls[phone_num]
elif phone_num in open_calls:
call = open_calls[phone_num]
feature = line[line.rfind('/')+1:line.find('.wav')]
if feature == "okyourreplies" or feature == "okplay_all" or feature == "okplay" or feature == "okrecord":
if feature in call:
call[feature] += 1
else:
call[feature] = 1
call['order'] += feature+','
elif feature == "okyouwant_pre" or feature == "okplaytag_pre":
# on the next go-around, look for the feature
call['feature_chosen'] = True
elif call['feature_chosen']:
if feature in call:
call[feature] += 1
else:
call[feature] = 1
call['order'] += feature+','
call['feature_chosen'] = False
call['last'] = current_date
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Histogram of number of features accessed within call:")
print("Features\tCalls\tAvg Duration")
dates = features.keys()
dates.sort()
features_hist = {}
for date in dates:
for call in features[date]:
features_tot = 0
for feature in call:
if feature != 'feature_chosen' and feature != 'order' and feature != 'start' and feature != 'last':
features_tot += call[feature]
if features_tot in features_hist:
features_hist[features_tot] += 1
else:
features_hist[features_tot] = 1
sorted_items = features_hist.items()
sorted_items.sort()
for num, tot in sorted_items:
durs_by_call = durations[num]
durs = [dur.seconds for dur in durs_by_call]
print(str(num)+"\t"+str(tot)+"\t"+str(sum(durs)/len(durs)))
print("Average features accessed within call, by week:")
for date in dates:
total_features = 0
num_calls = len(features[date])
for call in features[date]:
for feature in call:
if feature != 'feature_chosen' and feature != 'order' and feature != 'start' and feature != 'last':
total_features += call[feature]
print(date.strftime('%Y-%m-%d') + "\t" + str(float(total_features)/float(num_calls)))
return features
def get_lurking_and_posting(filename,
destnum,
forums,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
transfer_calls=False):
listens = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
forum_names = forums.values('name_file')
forum_names = [pair.values()[0] for pair in forum_names]
forumids = forums.values('pk')
forumids = [pair.values()[0] for pair in forumids]
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start += timedelta(days=7)
if not current_week_start in listens:
listens[current_week_start] = {
'lurks': 0,
'posts': 0,
'listens': []
}
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
listens[current_week_start]['lurks'] += call['lurks']
listens[current_week_start]['posts'] += call['posts']
listens[current_week_start]['listens'].append(
call['listens'])
del open_calls[phone_num]
open_calls[phone_num] = {
'lurks': 0,
'posts': 0,
'listens': 0,
'forum': False
}
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
listens[current_week_start]['lurks'] += call['lurks']
listens[current_week_start]['posts'] += call['posts']
listens[current_week_start]['listens'].append(
call['listens'])
del open_calls[phone_num]
elif phone_num in open_calls:
call = open_calls[phone_num]
for name in forum_names:
if line.find(name) != -1:
call['forum'] = True
# for record need the . to seperate from okrecorded and okrecordresponse
if call['forum'] and otalo_utils.is_prompt(
line) and line.find("okrecord.") != -1:
call['posts'] += 1
call['forum'] = False
elif call['forum'] and otalo_utils.is_prompt(
line) and line.find("okplay") != -1:
call['lurks'] += 1
call['forum'] = False
elif line.find("Stream") != -1:
fname = line[line.rfind('/') + 1:]
if bool(
Message_forum.objects.filter(
forum__in=forumids,
message__file__contains=fname.strip())):
#print("counting " + filename)
# check to make sure it's in the forums of interest
call['listens'] += 1
#else:
#print("rejecting" + filename)
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
#print("IndexError: " + line)
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Lurks, Posts, and Average listens within call, by week:")
print("Date\tLurks\tPosts\tLurking Rate\tListens per call")
dates = listens.keys()
dates.sort()
for date in dates:
lurks = listens[date]['lurks']
posts = listens[date]['posts']
lurk_rate = "n/a"
if lurks + posts > 0:
lurk_rate = float(lurks) / float(lurks + posts)
n_listens = listens[date]['listens']
avg_listens = "n/a"
if len(n_listens) > 0:
avg_listens = float(sum(n_listens)) / float(len(n_listens))
print(
date.strftime('%Y-%m-%d') + "\t" + str(lurks) + "\t" +
str(posts) + "\t" + str(lurk_rate) + "\t" + str(avg_listens))
return listens
def get_log_as_percent(filename, log, phone_num_filter=0):
calls = {}
current_week_start = 0
current_week_calls = 0
total = 0
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if not current_week_start:
current_week_calls = 0
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
call_tot = calls[current_week_start]
calls[current_week_start] = float(call_tot) / float(
current_week_calls)
current_week_calls = 0
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
if line.find("Start call") != -1:
current_week_calls += 1
if line.find(log) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Percent of " + log + "'s as a percentage of total calls, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
print(date.strftime('%Y-%m-%d') + "\t" + str(calls[date]))
def get_listens_within_call(filename,
destnum,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
transfer_calls=False,
daily_data=False):
listens = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
#flush all open calls
open_calls = {}
current_week_start += timedelta(days=days + 1)
if not current_week_start in listens:
listens[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
n_listens = call['listens']
listens[current_week_start].append(n_listens)
dur = call['last'] - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
open_calls[phone_num] = {
'start': current_date,
'last': current_date,
'listens': 0
}
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
n_listens = call['listens']
listens[current_week_start].append(n_listens)
dur = current_date - call['start']
if n_listens in durations:
durations[n_listens].append(dur)
else:
durations[n_listens] = [dur]
del open_calls[phone_num]
else:
if line.find("Stream") != -1:
if phone_num in open_calls:
open_calls[phone_num]['listens'] += 1
if phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_date
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
#print("IndexError: " + line)
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Histogram of number of listens within call")
print("Listens\tCalls\tAvg Duration")
dates = listens.keys()
dates.sort()
listens_hist = {}
for date in dates:
for tot in listens[date]:
if tot in listens_hist:
listens_hist[tot] += 1
else:
listens_hist[tot] = 1
sorted_items = listens_hist.items()
sorted_items.sort()
for num, tot in sorted_items:
durs_by_call = durations[num]
durs = [dur.seconds for dur in durs_by_call]
print(
str(num) + "\t" + str(tot) + "\t" + str(sum(durs) / len(durs)))
print("Average and total listens within call, by week:")
for date in dates:
total_listens = 0
num_calls = len(listens[date])
for tot in listens[date]:
total_listens += tot
if total_listens == 0:
avg = 0
else:
avg = float(total_listens) / float(num_calls)
print(
date.strftime('%Y-%m-%d') + "\t" + str(avg) + "\t" +
str(total_listens))
return listens
def get_online_time(filename, destnum=False, phone_num_filter=False, date_start=False, date_end=False, quiet=False):
online_time = {}
current_day = 0
open_calls = {}
f = open(filename)
if phone_num_filter:
# every phone number has an entry
for num in phone_num_filter:
online_time[num] = 0
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys() and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
if phone_num in online_time:
online_time[phone_num] += dur.seconds
else:
online_time[phone_num] = dur.seconds
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {'start':current_time, 'last':current_time }
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
if phone_num in online_time:
online_time[phone_num] += dur.seconds
else:
online_time[phone_num] = dur.seconds
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
#flush_open_calls(online_time, open_calls, current_day)
if not quiet:
print("Total online time, by phone number (s):")
for num, tot in online_time.items():
print(num +"\t"+str(tot))
return online_time
def get_log_as_percent(filename, log, phone_num_filter=0):
calls = {}
current_week_start = 0
current_week_calls = 0
total = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if not current_week_start:
current_week_calls = 0
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
call_tot = calls[current_week_start]
calls[current_week_start] = float(call_tot) / float(current_week_calls)
current_week_calls = 0
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if line.find("Start call") != -1:
current_week_calls += 1
if line.find(log) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Percent of "+ log + "'s as a percentage of total calls, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
print(date.strftime('%Y-%m-%d') +"\t"+str(calls[date]))
def get_calls_by_number(filename, destnum=False, log="Start call", phone_num_filter=False, date_start=False, date_end=False, quiet=False, legacy_log=False, transfer_calls=False):
calls = {}
phone_nums = ''
if phone_num_filter:
for num in phone_num_filter:
calls[num] = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
#print("dest num not = " + destnum)
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if line.find(log) != -1:
if phone_num in calls.keys():
calls[phone_num] += 1
else:
calls[phone_num] = 1
phone_nums += phone_num + ','
except ValueError as err:
#print("ValueError: " + str(err.args))
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
continue
calls_sorted = sorted(calls.iteritems(), key=lambda(k,v): (v,k))
calls_sorted.reverse()
if not quiet:
print("Number of "+ log + "'s by phone number:")
total = 0
for num, tot in calls_sorted:
#for num, tot in calls.items():
total += tot
print(num +"\t"+str(tot))
print('total is ' + str(total));
print('numbers are ' + phone_nums)
return calls_sorted
def get_lurking_and_posting(filename, destnum, forums, phone_num_filter=False, date_start=False, date_end=False, quiet=False, transfer_calls=False):
listens = {}
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
forum_names = forums.values('name_file')
forum_names = [pair.values()[0] for pair in forum_names]
forumids = forums.values('pk')
forumids = [pair.values()[0] for pair in forumids]
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
open_calls = {}
current_week_start += timedelta(days=7)
if not current_week_start in listens:
listens[current_week_start] = {'lurks':0,'posts':0,'listens':[]}
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
call = open_calls[phone_num]
listens[current_week_start]['lurks'] += call['lurks']
listens[current_week_start]['posts'] += call['posts']
listens[current_week_start]['listens'].append(call['listens'])
del open_calls[phone_num]
open_calls[phone_num] = {'lurks':0, 'posts':0, 'listens':0, 'forum':False }
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
call = open_calls[phone_num]
listens[current_week_start]['lurks'] += call['lurks']
listens[current_week_start]['posts'] += call['posts']
listens[current_week_start]['listens'].append(call['listens'])
del open_calls[phone_num]
elif phone_num in open_calls:
call = open_calls[phone_num]
for name in forum_names:
if line.find(name) != -1:
call['forum'] = True
# for record need the . to seperate from okrecorded and okrecordresponse
if call['forum'] and otalo_utils.is_prompt(line) and line.find("okrecord.") != -1:
call['posts'] += 1
call['forum'] = False
elif call['forum'] and otalo_utils.is_prompt(line) and line.find("okplay") != -1:
call['lurks'] += 1
call['forum'] = False
elif line.find("Stream") != -1:
fname = line[line.rfind('/')+1:]
if bool(Message_forum.objects.filter(forum__in=forumids, message__file__contains=fname.strip())):
#print("counting " + filename)
# check to make sure it's in the forums of interest
call['listens'] += 1
#else:
#print("rejecting" + filename)
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
#print("IndexError: " + line)
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Lurks, Posts, and Average listens within call, by week:")
print("Date\tLurks\tPosts\tLurking Rate\tListens per call")
dates = listens.keys()
dates.sort()
for date in dates:
lurks = listens[date]['lurks']
posts = listens[date]['posts']
lurk_rate = "n/a"
if lurks+posts > 0:
lurk_rate = float(lurks)/float(lurks + posts)
n_listens = listens[date]['listens']
avg_listens = "n/a"
if len(n_listens) > 0:
avg_listens = float(sum(n_listens))/float(len(n_listens))
print(date.strftime('%Y-%m-%d') + "\t" + str(lurks) + "\t" + str(posts) + "\t" + str(lurk_rate) + "\t" + str(avg_listens))
return listens
def get_calls_by_feature(filename, destnum, phone_num_filter=0, date_start=False, date_end=False, quiet=False, legacy_log=False):
features = {}
feature_names = []
feature_chosen = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not legacy_log and destnum and destnum.find(dest) == -1:
continue
if phone_num not in features:
features[phone_num] = {}
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls:
# close out current call
del open_calls[phone_num]
# add new call with no feature access yet
open_calls[phone_num] = False
elif line.find("End call") != -1:
if phone_num in open_calls:
# close out call
del open_calls[phone_num]
elif phone_num in open_calls:
feature_chosen = open_calls[phone_num]
feature = line[line.rfind('/')+1:line.find('.wav')]
curr_features = features[phone_num]
if feature == "okyourreplies" or feature == "okplay_all" or feature == "okplay" or feature == "okrecord":
if feature not in feature_names:
feature_names.append(feature)
if feature in curr_features.keys():
curr_features[feature] += 1
else:
curr_features[feature] = 1
elif feature == "okyouwant_pre" or feature == "okplaytag_pre":
# on the next go-around, look for the feature
open_calls[phone_num] = True
elif feature_chosen:
if feature not in feature_names:
feature_names.append(feature)
if feature in curr_features.keys():
curr_features[feature] += 1
else:
curr_features[feature] = 1
open_calls[phone_num] = False
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of calls by phone number, by feature:")
header = "\t"
for name in feature_names:
header += name+"\t"
print(header)
numbers = features.keys()
for num in numbers:
row = num +"\t"
for name in feature_names:
if name in features[num]:
row += str(features[num][name]) + "\t"
else:
row += "0\t"
print(row)
def get_calls(filename, destnum=False, log="Start call", phone_num_filter=False, date_start=False, date_end=False, quiet=False, legacy_log=False, transfer_calls=False, daily_data=False):
calls = {}
nums = []
current_week_start = 0
total = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if not legacy_log:
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if daily_data:
days = 0
else:
days = 6
if delta.days > days:
current_week_start += timedelta(days=days+1)
calls[current_week_start] = 0
#print('found3 ' + phone_num)
if phone_num not in nums:
nums.append(phone_num)
if line.lower().find(log.lower()) != -1:
if current_week_start in calls:
calls[current_week_start] += 1
else:
calls[current_week_start] = 1
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Number of "+ log + "'s, by week:")
dates = calls.keys()
dates.sort()
for date in dates:
total += calls[date]
print(otalo_utils.date_str(date) +"\t"+str(calls[date]))
print("total is " + str(total))
print ("number of unique callers is " + str(len(nums)))
return calls
def get_call_durations(filename,
destnum,
phone_num_filter=False,
date_start=False,
date_end=False,
quiet=False,
transfer_calls=False):
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
flush_open_calls(durations, open_calls, current_week_start)
open_calls = {}
current_week_start = datetime(year=current_date.year,
month=current_date.month,
day=current_date.day)
if not current_week_start in durations:
durations[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys(
) and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
durations[current_week_start].append([phone_num, dur])
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {
'start': current_time,
'last': current_time
}
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
durations[current_week_start].append([phone_num, dur])
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
flush_open_calls(durations, open_calls, current_week_start)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Average call duration, by week (s):")
dates = durations.keys()
dates.sort()
secs = 0
calls = 0
for date in dates:
durs_by_call = durations[date]
durs = [dur[1].seconds for dur in durs_by_call]
secs += sum(durs)
calls += len(durs)
print(
date.strftime('%Y-%m-%d') + "\t" + str(sum(durs) / len(durs)))
if calls > 0:
print('Overall Average: ' + str(secs / calls))
return durations
def get_recordings(filename, destnum=False, phone_num_filter=False, date_start=False, date_end=False, legacy_log=False, transfer_calls=False):
files = []
current_week_start = 0
total = 0
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if otalo_utils.is_record(line):
filename = otalo_utils.get_prompt(line)
filename = filename[filename.find(settings.MEDIA_ROOT)+len(settings.MEDIA_ROOT)+1:]
files.append(filename.strip())
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
return files
def get_call_durations(filename, destnum, phone_num_filter=False, date_start=False, date_end=False, quiet=False, transfer_calls=False):
durations = {}
current_week_start = 0
open_calls = {}
f = open(filename)
while(True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line)
dest = otalo_utils.get_destination(line)
##
################################################
current_time = otalo_utils.get_time(line)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if transfer_calls == "INBOUND_ONLY" and len(dest) == 10:
continue
elif transfer_calls == "TRANSFER_ONLY" and len(dest) < 10:
continue
if not current_week_start:
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
delta = current_date - current_week_start
if delta.days > 6:
#flush all open calls
flush_open_calls(durations, open_calls, current_week_start)
open_calls = {}
current_week_start = datetime(year=current_date.year, month=current_date.month, day=current_date.day)
if not current_week_start in durations:
durations[current_week_start] = []
if line.find("Start call") != -1:
# check to see if this caller already has one open
if phone_num in open_calls.keys() and current_time > open_calls[phone_num]['last']:
# close out current call
call = open_calls[phone_num]
del open_calls[phone_num]
dur = call['last'] - call['start']
#print("closing out call pre-emptively: " + phone_num + ", "+otalo_utils.date_str(current_date) + ", "+otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
durations[current_week_start].append([phone_num,dur])
# add new call
#print("adding new call: " + phone_num)
open_calls[phone_num] = {'start':current_time, 'last':current_time }
elif line.find("End call") != -1:
if phone_num in open_calls.keys():
# close out call
call = open_calls[phone_num]
dur = current_time - call['start']
#print("closing out call: "+phone_num + ", "+otalo_utils.date_str(current_date) + ", "+ otalo_utils.get_sessid(line) + ", duration: " + str(dur.seconds))
durations[current_week_start].append([phone_num,dur])
del open_calls[phone_num]
elif phone_num in open_calls:
#print("updating call dur: " + phone_num)
# this makes things conservative. A call is only officially counted if
# it starts with a call_started
open_calls[phone_num]['last'] = current_time
#print("open_calls: " + str(open_calls))
except KeyError as err:
#print("KeyError: " + phone_num + "-" + otalo.date_str(current_date) + " " + otalo.time_str(current_time))
raise
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
#flush the last week
flush_open_calls(durations, open_calls, current_week_start)
if not quiet:
if phone_num_filter:
print("Data for phone numbers: " + str(phone_num_filter))
print("Average call duration, by week (s):")
dates = durations.keys()
dates.sort()
secs = 0
calls = 0
for date in dates:
durs_by_call = durations[date]
durs = [dur[1].seconds for dur in durs_by_call]
secs += sum(durs)
calls += len(durs)
print(date.strftime('%Y-%m-%d') +"\t"+ str(sum(durs)/len(durs)))
if calls > 0:
print('Overall Average: ' + str(secs/calls))
return durations
def get_recordings(filename,
destnum=False,
phone_num_filter=False,
date_start=False,
date_end=False,
legacy_log=False,
transfer_calls=False):
files = []
current_week_start = 0
total = 0
f = open(filename)
while (True):
line = f.readline()
if not line:
break
try:
#################################################
## Use the calls here to determine what pieces
## of data must exist for the line to be valid.
## All of those below should probably always be.
phone_num = otalo_utils.get_phone_num(line)
current_date = otalo_utils.get_date(line, legacy_log)
dest = otalo_utils.get_destination(line, legacy_log)
##
################################################
#print(phone_num + ': dest = ' + dest)
if phone_num_filter and not phone_num in phone_num_filter:
continue
if date_start:
if date_end:
if not (current_date >= date_start
and current_date < date_end):
continue
if current_date > date_end:
break
else:
if not current_date >= date_start:
continue
if destnum and destnum.find(dest) == -1:
continue
# A hacky way to test for transfer call
# In the future you want to compare this call's
# start time to a time window related to the end
# of the survey call (in which you can keep the flag
# false and give a more targeted start and end date)
if transfer_calls:
if len(dest) < 10:
continue
elif len(dest) == 10:
continue
if otalo_utils.is_record(line):
filename = otalo_utils.get_prompt(line)
filename = filename[filename.find(settings.MEDIA_ROOT) +
len(settings.MEDIA_ROOT) + 1:]
files.append(filename.strip())
except ValueError as err:
#print("ValueError: " + line)
continue
except IndexError as err:
continue
except otalo_utils.PhoneNumException:
#print("PhoneNumException: " + line)
continue
return files