def reduce_location(l):
val = l.lower(
val = val.strip()
val = val.replace('usa','')
val = val.replace('united state','')
val = re.sub(r'\d*$','',val)
m = re.search('([\w ]*)\s*[\s,/]\s*(\w+\s*\w*)(\W*)',val)
if m:
val = (m.group(1).strip(), m.group(2).strip())
#print 'compressing 1 : \'%s\' FROM \'%s\''%(val,l)
else:
m = re.search('(.*?)(\W)+(\w+)(\W*)',val)
if m:
val = (m.group(1).strip(), m.group(3).strip())
#print 'compressing 2 : \'%s\' FROM \'%s\''%(val,l)
else:
val = ('', val)
if val[1] in STATES_NORMALIZED:
val = (val[0], STATES_NORMALIZED[val[1]].lower())
#print ' compressing 3 : \'%s\' FROM \'%s\''%(val,l)
test = '%s, %s'%(val[0], val[1])
if val[1] in STATES_NORMALIZED:
return val
if test == 'san, francisco':
return ('san francisco', 'ca')
if test == 'bronx, ny':
return ('new york', 'ny')
if test == 'new, york':
return ('', 'ny')
if test == 'houston, ':
return ('houston', 'tx')
if test == 'chicago, ':
return ('chicago', 'il')
if test == 'seattle, ':
return ('seattle', 'wa')
if test == 'brooklyn, ':
return ('brooklyn', 'ny')
if test == 'philadelphia, ':
return ('philadelphia', 'pa')
if test == 'dallas, ':
return ('dallas', 'tx')
if test == 'arlington, ':
return ('arlington', 'va')
if test == 'nashville, ':
return ('nashville', 'tn')
if test == 'chicago, ':
return ('chicago', 'il')
if test == 'chicago, ':
return ('chicago', 'il')
if test == 'washington, district of':
return ('washington', 'dc')
if test == 'los, angeles' or test == 'la, ':
return ('los angeles, ca')
if test == 'san, diego':
return ('san diego', 'ca')
if val[0] in STATES_NORMALIZED and val[1] == '':
return ('', val[0])
if val[0] == '':
return (val[1], '')
return val
unique_locations = {}
location_results = {}
for key in users.keys():
u = users[key]
orig_loc = u['details'][4]
loc = reduce_location(orig_loc)
if orig_loc not in unique_locations:
unique_locations[orig_loc] = loc
if len(u['sub']) == 1:
if loc in location_results:
location_results[loc]['sub'] += 1
else:
location_results[loc] = {'sub': 1, 'free': 0}
else:
if loc in location_results:
location_results[loc]['free'] += 1
else:
location_results[loc] = {'sub': 0, 'free': 1}
len(unique_locations.keys())
len(location_results.keys())
for l in location_results.keys():
if location_results[l]['sub'] > 2:
print '%i\t%i\t%s'%(location_results[l]['free'], location_results[l]['sub'], l)
filename = 'location_stats.csv'
f = open(filename, 'w')
for l in location_results.keys():
if location_results[l]['sub'] > 0 or location_results[l]['free'] > 100:
print('%i\t%i\t%s'%(location_results[l]['free'], location_results[l]['sub'], l))
f.write('%i\t%i\t%s\n'%(location_results[l]['free'], location_results[l]['sub'], l))
f.close()
# TODO - collect emails of users, see if they were shared to BEFORE their join date
# TODO - collect shares SENT to the users
user_ids = [long(i) for i in users.keys()]
shares = {}
query = ShareActions.objects.filter(user_id__in=user_ids, app=settings.APPS.Rdio).values_list('user_id', 'recipient_id', 'type','method','date_shared')
print ' .. chunking user info'
for share_chunk in chunk_query(query, 500):
print ' .... chunk'
for s in share_chunk:
if s.user_id in shares:
shares[s.user_id].append(s)
else:
shares[s.user_id] = [s]
shared_to = {}
query = ShareActions.objects.filter(recipient_id__in=user_ids, app=settings.APPS.Rdio).values_list('user_id', 'recipient_id', 'type','method','date_shared')
print ' .. chunking user info'
for share_chunk in chunk_query(query, 500):
print ' .... chunk'
for s in share_chunk:
if s.recipient_id in shared_to:
shared_to[s.recipient_id].append(s)
else:
shared_to[s.recipient_id] = [s]
for u in shares.keys():
users[str(u)]['shared'] = shares[u]
for u in shared_to.keys():
users[str(u)]['shared_to'] = shared_to[u]
share_summary = {
'avg_free_shared': [],
'avg_free_shared_to': [],
'avg_shared_before': [],
'avg_shared_after': [],
'avg_shared_to_before': [],
'avg_shared_to_after': []
}
filename = 'share_info.csv'
f = open(filename, 'w')
for u in set(shares.keys()).union(set(shared_to.keys())):
u_sharer = users[str(u)]
date_joined = '%s %s'%(u_sharer['details'][1],u_sharer['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
if len(u_sharer['sub']) == 1:
u_sharer['share_summary'] = {'send_before_sub': 0, 'send_after_sub': 0, 'shared_to_before_sub': 0, 'shared_to_after_sub': 0}
date_subscribed = '%s %s'%(u_sharer['sub'][0][3],u_sharer['sub'][0][4])
date_subscribed = datetime.datetime.strptime(date_subscribed, '%Y-%m-%d %H:%M:%S')
if 'shared' in u_sharer:
for s in u_sharer['shared']:
recipient = s.recipient_id
if recipient is None:
recipient = 'n/a'
if str(recipient) in users:
recipient = 'new_user %i'%len(users[str(recipient)]['sub'])
else:
recipient = 'other'
f.write('shared,%i,%s,%s,%s,%s,%i,%s,%s,%i,%i\n'%(
u, u_sharer['sub'][0][3], u_sharer['sub'][0][4],
recipient, s.type, s.method,
s.date_shared.date(), s.date_shared.time(),
(s.date_shared - date_subscribed).days,
(s.date_shared - date_joined).days
))
if (s.date_shared - date_subscribed).days > 0:
u_sharer['share_summary']['send_after_sub'] += 1
else:
u_sharer['share_summary']['send_before_sub'] += 1
if u_sharer['share_summary']['send_after_sub']:
share_summary['avg_shared_after'].append(u_sharer['share_summary']['send_after_sub'])
if u_sharer['share_summary']['send_before_sub']:
share_summary['avg_shared_before'].append(u_sharer['share_summary']['send_before_sub'])
if 'shared_to' in u_sharer:
for s in u_sharer['shared_to']:
sender = s.user_id
if sender is None:
sender = 'n/a'
if str(recipient) in users:
sender = 'new_user %i'%len(users[str(recipient)]['sub'])
else:
sender = 'other'
f.write('shared_to,%i,%s,%s,%s,%s,%i,%s,%s,%i,%i\n'%(
u, u_sharer['sub'][0][3], u_sharer['sub'][0][4],
sender, s.type, s.method,
s.date_shared.date(), s.date_shared.time(),
(s.date_shared - date_subscribed).days,
(s.date_shared - date_joined).days
))
if (s.date_shared - date_joined).days > 0:
u_sharer['share_summary']['shared_to_after_sub'] += 1
else:
u_sharer['share_summary']['shared_to_before_sub'] += 1
if u_sharer['share_summary']['shared_to_after_sub']:
share_summary['avg_shared_to_after'].append(u_sharer['share_summary']['shared_to_after_sub'])
if u_sharer['share_summary']['shared_to_before_sub']:
share_summary['avg_shared_to_before'].append(u_sharer['share_summary']['shared_to_before_sub'])
else:
u_sharer['share_summary'] = {'shared': 0, 'shared_to': 0}
if 'shared' in u_sharer:
u_sharer['share_summary']['shared'] += len(u_sharer['shared'])
share_summary['avg_free_shared'].append(u_sharer['share_summary']['shared'])
for s in u_sharer['shared']:
recipient = s.recipient_id
if recipient is None:
recipient = 'n/a'
if str(recipient) in users:
recipient = 'new_user %i'%len(users[str(recipient)]['sub'])
else:
recipient = 'other'
f.write('shared,%i,%s,%s,%s,%s,%i,%s,%s,%i,%i\n'%(
u, '', '',
recipient, s.type, s.method,
s.date_shared.date(), s.date_shared.time(),
0,
(s.date_shared - date_joined).days
))
if 'shared_to' in u_sharer:
u_sharer['share_summary']['shared_to'] += len(u_sharer['shared_to'])
share_summary['avg_free_shared_to'].append(u_sharer['share_summary']['shared_to'])
for s in u_sharer['shared_to']:
sender = s.user_id
if sender is None:
sender = 'n/a'
if str(recipient) in users:
sender = 'new_user %i'%len(users[str(recipient)]['sub'])
else:
sender = 'other'
f.write('shared_to,%i,%s,%s,%s,%s,%i,%s,%s,%i,%i\n'%(
u, '','',
sender, s.type, s.method,
s.date_shared.date(), s.date_shared.time(),
0,
(s.date_shared - date_joined).days
))
sub_shared = share_summary['avg_shared_before'] + share_summary['avg_shared_after']
print 'SHARED:\n\tfree: %i (avg %.2f) (mean %i) (pop %.4f)\n\tsub: %i (avg %.2f) (mean %i) (pop %.4f)\n\t\tbefore sub: %i (avg %.2f) (mean %i) (pop %.4f)\n\t\tafter sub:%i (avg %.2f) (mean %i) (pop %.4f)'%(
sum(share_summary['avg_free_shared']),
sum(share_summary['avg_free_shared'])/float(len(share_summary['avg_free_shared'])),
sorted(share_summary['avg_free_shared'])[len(share_summary['avg_free_shared'])/2],
len(share_summary['avg_free_shared']) / float(stayed_free_user_count),
sum(sub_shared),
sum(sub_shared)/float(len(sub_shared)),
sorted(sub_shared)[len(sub_shared)/2],
len(sub_shared) / float(subscribed_user_count),
sum(share_summary['avg_shared_before']),
sum(share_summary['avg_shared_before'])/float(len(share_summary['avg_shared_before'])),
sorted(share_summary['avg_shared_before'])[len(share_summary['avg_shared_before'])/2],
len(share_summary['avg_shared_before']) / float(subscribed_user_count),
sum(share_summary['avg_shared_after']),
sum(share_summary['avg_shared_after'])/float(len(share_summary['avg_shared_after'])),
sorted(share_summary['avg_shared_after'])[len(share_summary['avg_shared_after'])/2],
len(share_summary['avg_shared_after']) / float(subscribed_user_count)
)
sub_shared_to = share_summary['avg_shared_to_before'] + share_summary['avg_shared_to_after']
print 'SHARED TO:\n\tfree: %i (avg %.2f) (mean %i) (pop %.4f)\n\tsub: %i (avg %.2f) (mean %i) (pop %.4f)\n\t\tbefore sub: %i (avg %.2f) (mean %i) (pop %.4f)\n\t\tafter sub:%i (avg %.2f) (mean %i) (pop %.4f)'%(
sum(share_summary['avg_free_shared_to']),
sum(share_summary['avg_free_shared_to'])/float(len(share_summary['avg_free_shared_to'])),
sorted(share_summary['avg_free_shared_to'])[len(share_summary['avg_free_shared_to'])/2],
len(share_summary['avg_free_shared_to']) / float(stayed_free_user_count),
sum(sub_shared_to),
sum(sub_shared_to)/float(len(sub_shared_to)),
sorted(sub_shared_to)[len(sub_shared_to)/2],
len(sub_shared_to) / float(subscribed_user_count),
sum(share_summary['avg_shared_to_before']),
sum(share_summary['avg_shared_to_before'])/float(len(share_summary['avg_shared_to_before'])),
sorted(share_summary['avg_shared_to_before'])[len(share_summary['avg_shared_to_before'])/2],
len(share_summary['avg_shared_to_before']) / float(subscribed_user_count),
sum(share_summary['avg_shared_to_after']),
sum(share_summary['avg_shared_to_after'])/float(len(share_summary['avg_shared_to_after'])),
sorted(share_summary['avg_shared_to_after'])[len(share_summary['avg_shared_to_after'])/2],
len(share_summary['avg_shared_to_after']) / float(subscribed_user_count)
)
stayed_free_user_count = 0
subscribed_user_count = 0
odd_user_count = 0
for key in users.keys():
if len(users[key]['sub']) == 0:
stayed_free_user_count += 1
elif len(users[key]['sub']) == 1:
subscribed_user_count += 1
else:
odd_user_count += 1
query = UserFollowers.objects.filter(follower__in=user_ids)
print ' .. chunking follower info'
for follow_chunk in chunk_query(query, 5000):
print ' .... chunk'
for f in follow_chunk:
key = str(f.follower.id)
if 'is_follower' in users[key]:
users[key]['is_follower'].append({ 'followee': f.followee.id, 'date': f.date_added})
else:
users[key]['is_follower'] = [{ 'followee': f.followee.id, 'date': f.date_added}]
filename = 'is_follower_info.csv'
file = open(filename, 'w')
for key in users.keys():
if 'is_follower' in users[key]:
u = users[key]
date_joined = '%s %s'%(u['details'][1], u['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
date_subscribed = False
account = 'f'
if len(u['sub']) == 1:
account = 's'
date_subscribed = '%s %s'%(u['sub'][0][3],u['sub'][0][4])
date_subscribed = datetime.datetime.strptime(date_subscribed, '%Y-%m-%d %H:%M:%S')
days = {}
for f in u['is_follower']:
dayKey = f['date'].date()
if dayKey in days:
days[dayKey]['count'] += 1
else:
days_since_join = (f['date'] - date_joined).days
days_since_sub = 0
if date_subscribed:
days_since_sub = (f['date'] - date_subscribed).days
days[dayKey] = { 'count': 1, 'since_sub': days_since_sub, 'date': f['date'].date(),'since_join': days_since_join, 'total': len(u['is_follower']) }
#print '%s,%s,%s,%s,%i,%i'%(key, followee, f['date'].date(), f['date'].time(), days_since_join, days_since_sub)
#file.write('%s,%s,%s,%s,%i,%i,%i,%i\n'%(key, followee, f['date'].date(), f['date'].time(), days_since_join, days_since_sub, index, len(u['is_follower'])))
#index += 1
for d in days:
d = days[d]
file.write('%s,%s,%i,%i,%s,%i,%i\n'%(key, account,d['count'],d['total'],d['date'],d['since_join'],d['since_sub']))
query = UserFollowers.objects.filter(followee__in=user_ids)
print ' .. chunking followee info'
for follow_chunk in chunk_query(query, 5000):
print ' .... chunk'
for f in follow_chunk:
key = str(f.followee.id)
if 'is_followee' in users[key]:
users[key]['is_followee'].append({ 'follower': f.follower.id, 'date': f.date_added})
else:
users[key]['is_followee'] = [{ 'follower': f.follower.id, 'date': f.date_added}]
filename = 'is_followee_info.csv'
file = open(filename, 'w')
for key in users.keys():
if 'is_followee' in users[key]:
u = users[key]
date_joined = '%s %s'%(u['details'][1], u['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
date_subscribed = False
account = 'f'
if len(u['sub']) == 1:
account = 's'
date_subscribed = '%s %s'%(u['sub'][0][3],u['sub'][0][4])
date_subscribed = datetime.datetime.strptime(date_subscribed, '%Y-%m-%d %H:%M:%S')
days = {}
for f in u['is_followee']:
dayKey = f['date'].date()
if dayKey in days:
days[dayKey]['count'] += 1
else:
days_since_join = (f['date'] - date_joined).days
days_since_sub = 0
if date_subscribed:
days_since_sub = (f['date'] - date_subscribed).days
days[dayKey] = { 'count': 1, 'since_sub': days_since_sub, 'date': f['date'].date(),'since_join': days_since_join, 'total': len(u['is_followee']) }
#print '%s,%s,%s,%s,%i,%i'%(key, followee, f['date'].date(), f['date'].time(), days_since_join, days_since_sub)
#file.write('%s,%s,%s,%s,%i,%i,%i,%i\n'%(key, followee, f['date'].date(), f['date'].time(), days_since_join, days_since_sub, index, len(u['is_followee'])))
#index += 1
for d in days:
d = days[d]
file.write('%s,%s,%i,%i,%s,%i,%i\n'%(key, account,d['count'],d['total'],d['date'],d['since_join'],d['since_sub']))
filename = 'user_keys.csv'
file = open(filename, 'w')
for key in users.keys():
u = users[key]
if len(u['sub']) == 0:
file.write('%s,%s\n'%(key,'f'))
elif len(u['sub']) == 1:
file.write('%s,%s\n'%(key,'t'))
else:
file.write('%s,%s\n'%(key,'x'))
user_ids = [long(i) for i in users.keys()]
more_info_stuff = {}
query = User.objects.filter(id__in=user_ids)
print ' .. chunking misc info'
for misc_info_chunk in chunk_query(query, 5000):
print ' .... chunk'
for u in misc_info_chunk:
key = str(u.id)
more_info_stuff[key] = {
'email': u.email,
'login': u.last_login,
'join': u.date_joined,
'reviews': u.review_count,
'last_update': u.last_updated,
'device_count': len(u.offlinedevice_set.all())
}
filename = 'misc_user_info.csv'
file = open(filename, 'w')
for key in more_info_stuff.keys():
if str(key) not in users:
print "WHAT THE F**K %s"%key
else:
user = users[key]
data = more_info_stuff[key]
email = re.sub(r'.*@(.*)',r'\1',data['email'])
days_active = (data['last_update'] - data['join']).days
file.write('%s,%s,%i,%i,%i\n'%(key,email,days_active,data['reviews'],data['device_count']))
# user['misc'] = {
# 'email_domain': email,
# 'reviews': data['reviews'],
# 'days_active': days_active,
# 'device_count': data['device_count']
# }
playlists = {}
query = Playlist.objects.filter(owner__in=user_ids)
print ' .. chunking playlist info'
for playlist_chunk in chunk_query(query, 5000):
print ' .... chunk'
for p in playlist_chunk:
key = str(p.owner.id)
if key not in playlists:
playlists[key] = []
playlists[key].append({ 'created': p.created, 'updates': p.counts_updated, 'size': len(p.playlist_info.entries) })
filename = 'playlists.csv'
file = open(filename, 'w')
for key in playlists:
u = users[key]
u['playlists'] = []
date_joined = '%s %s'%(u['details'][1], u['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
for p in playlists[key]:
day_made = (p['created'] - date_joined).days
users[key]['playlists'].append({ 'day_made': day_made, 'size': p['size']})
file.write('%s,%i,%i,%s,%s\n'%(key, p['size'], day_made, p['created'].date(), p['created'].time()))
# 0 1 2 3 1= device, 2= day, 3 = songs played
#15089702,2,0.0,43
f_plays = open('/Users/rebecca/Work/hackday/novemberHack/plays_october.csv', 'r')
for l in f_plays:
l = l.split(',')
key = l[0]
u = users[key]
if 'plays' not in u:
u['plays'] = []
u['plays'].append({ 'device': l[1], 'day': l[2], 'started_songs': l[3].strip()})
f_plays.close()
f_tracked_plays = open('/Users/rebecca/Work/hackday/novemberHack/plays_tracked_october.csv', 'r')
for l in f_tracked_plays:
l = l.split(',')
key = l[0]
u = users[key]
if 'plays' not in u:
print 'WHAT THE HELL!?!? %s'%key
else:
for p in u['plays']:
if p['device'] == l[1] and p['day'] == l[2]:
p['tracked_songs'] = l[3].strip()
else:
print "\t .. how can I have a tracked count without a play count? %s : %s %s"%(key,l[1],l[2])
f_tracked_plays.close()
plays_summary = {
'no_plays': 0
}
for day in range(0,11):
plays_summary['free_players_%s'%day] = 0
plays_summary['free_ratio_%s'%day] = 0
plays_summary['free_skips_%s'%day] = 0
plays_summary['free_plays_%s'%day] = 0
plays_summary['sub_players_%s'%day] = 0
plays_summary['sub_ratio_%s'%day] = 0
plays_summary['sub_skips_%s'%day] = 0
plays_summary['sub_plays_%s'%day] = 0
for key in users.keys():
u = users[key]
sub = len(users[key]['sub']) == 1
if 'plays' not in u:
plays_summary['no_plays'] += 1
elif sub:
for p in u['plays']:
i = int(float(p['day']))
tracked_songs = int(p['tracked_songs']) if 'tracked_songs' in p else 0
plays_summary['sub_plays_%s'%i] += tracked_songs
plays_summary['sub_skips_%s'%i] += int(p['started_songs']) - tracked_songs
plays_summary['sub_ratio_%s'%i] += plays_summary['sub_skips_%i'%i] / float(plays_summary['sub_plays_%i'%i] + plays_summary['sub_skips_%i'%i])
plays_summary['sub_players_%s'%i] += 1.0
else:
for p in u['plays']:
i = int(float(p['day']))
tracked_songs = int(p['tracked_songs']) if 'tracked_songs' in p else 0
plays_summary['free_plays_%s'%i] += tracked_songs
plays_summary['free_skips_%s'%i] += int(p['started_songs']) - tracked_songs
plays_summary['free_ratio_%s'%i] += plays_summary['free_skips_%i'%i] / float(plays_summary['free_plays_%i'%i] + plays_summary['free_skips_%i'%i])
plays_summary['free_players_%s'%i] += 1.0
for day in range(0,10):
print "DAY %i"%day
v = plays_summary['free_skips_%i'%day] / float(plays_summary['free_plays_%i'%day] + plays_summary['free_skips_%i'%day])
print " free: %.03f ratio\t%.03f avg ratio"%(v,plays_summary['free_ratio_%s'%day]/plays_summary['free_players_%s'%day])
v = plays_summary['sub_skips_%i'%day] / float(plays_summary['sub_plays_%i'%day] + plays_summary['sub_skips_%i'%day])
print " sub: %.03f\t%.03f"%(v,plays_summary['sub_ratio_%s'%day]/plays_summary['sub_players_%s'%day])
filename = 'play_summary.csv'
file = open(filename, 'w')
cut_off_date = datetime.datetime(2013,10,8)
for key in users.keys():
u = users[key]
date_joined = '%s %s'%(u['details'][1], u['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
if (date_joined - cut_off_date).days < 0:
sub = len(users[key]['sub']) == 1
if 'plays' in u:
for p in u['plays']:
day = int(float(p['day']))
if day < 3:
tracked_songs = int(p['tracked_songs']) if 'tracked_songs' in p else 0
file.write('%s,%s,%s,%i,%s,%s,%i\n'%(key,sub,date_joined.date(),day,p['device'],p['started_songs'],tracked_songs))
else:
file.write('%s,%s,%s,%i,%s,%s,%i\n'%(key,sub,date_joined.date(),0,-1,0,0))
formal_users = {}
for key in users.keys():
u = users[key]
date_joined = '%s %s'%(u['details'][1], u['details'][2])
date_joined = datetime.datetime.strptime(date_joined, '%Y-%m-%d %H:%M:%S')
formal_users[key] = {
'id': key,
'join_date': u['details'][1],
'join_time': u['details'][2],
'sub_date': u['sub'][0][3] if len(u['sub']) == 1 else -99,
'sub_time': u['sub'][0][4] if len(u['sub']) == 1 else -99,
'follower_day_0': 0,
'follower_1_to_sub': 0,
'follower_post_sub': 0,
'followee_pre_sub': 0,
'followee_post_sub': 0,
'shared_pre_sub': 0,
'shared_post_sub': 0,
'shared_to_pre_sub': 0,
'shared_to_post_sub': 0,
'playlists_pre_sub': [],
'playlists_post_sub': [],
'gender': u['details'][5].strip(),
'loc_0': u['loc'][0],
'loc_1': u['loc'][1],
'age': u['age'],
'days_to_sub': u['days_to_sub'] if 'days_to_sub' in u else -6,
'days_active': u['misc']['days_active'],
'plays': [],
'email_domain': u['misc']['email_domain'],
'device_count': u['misc']['device_count'],
'reviews': u['misc']['reviews']
}
if 'playlists' in u:
for p in u['playlists']:
if p['day_made'] > formal_users[key]['days_to_sub'] and formal_users[key]['days_to_sub'] >= 0:
formal_users[key]['playlists_post_sub'].append(p['size'])
else:
formal_users[key]['playlists_pre_sub'].append(p['size'])
if 'is_follower' in u:
for f in u['is_follower']:
days_to_follow = (f['date'] - date_joined).days
if days_to_follow == 0:
formal_users[key]['follower_day_0'] += 1
elif days_to_follow > formal_users[key]['days_to_sub'] and formal_users[key]['days_to_sub'] >= 0:
formal_users[key]['followee_post_sub'] += 1
else:
formal_users[key]['followee_pre_sub'] += 1
if 'is_followee' in u:
for f in u['is_followee']:
days_to_follow = (f['date'] - date_joined).days
if days_to_follow > formal_users[key]['days_to_sub'] and formal_users[key]['days_to_sub'] >= 0:
formal_users[key]['followee_pre_sub'] += 1
else:
formal_users[key]['followee_post_sub'] += 1
if 'shared' in u:
for s in u['shared']:
days_to_share = (s.date_shared - date_joined).days
if days_to_share > formal_users[key]['days_to_sub'] and formal_users[key]['days_to_sub'] >= 0:
formal_users[key]['shared_post_sub'] += 1
else:
formal_users[key]['shared_pre_sub'] += 1
if 'shared_to' in u:
for s in u['shared_to']:
days_to_share = (s.date_shared - date_joined).days
if days_to_share > formal_users[key]['days_to_sub'] and formal_users[key]['days_to_sub'] >= 0:
formal_users[key]['shared_to_post_sub'] += 1
else:
formal_users[key]['shared_to_pre_sub'] += 1
if 'plays' in u:
if (date_joined - cut_off_date).days < 0:
sub = len(users[key]['sub']) == 1
for p in u['plays']:
day = int(float(p['day']))
if day < 3:
tracked_songs = int(p['tracked_songs']) if 'tracked_songs' in p else 0
formal_users[key]['plays'].append((day,int(p['device']),int(p['started_songs']),tracked_songs))
mylist = [u for u in u['plays'] if isinstance(u,dict)]
filename = 'formal_user_data_3.csv'
file = open(filename, 'w')
json.dump(formal_users, file)
reduced_formal_users = {}
keys = formal_users.keys()[:20]
for key in keys:
reduced_formal_users[key] = formal_users[key]
filename = 'formal_user_data_short_test.csv'
file = open(filename, 'w')
file.write ("var edges = ");
json.dump(reduced_formal_users, file)
file.close()
for key in users.keys():
u = users[key]
if 'playlists' in u:
print 6/0
import datetime
from rdio.auth.models import UserFollowers
from rdio.analytics.models import ShareActions
from django.conf import settings
from rdio.library.models.playlist import Playlist
import json
for day in range(1,11) :
print "Starting day %i"%day
user_ids = []
user_details = []
query = User.objects.filter(date_joined__gte=datetime.datetime(2013,10,day),date_joined__lte=datetime.datetime(2013,10,day+1))
print ' .. chunking user info'
for user_chunk in chunk_query(query, 1000):
print ' .... chunk'
for u in user_chunk:
if u.country_code == 'US':
user_ids.append(u.id)
user_details.append((u.id, u.date_joined, u.birthday, u.location, u.gender))
subs = SubscriptionHistory.objects.filter(
event__in=(SubscriptionHistoryEvent.UserSubscribe, SubscriptionHistoryEvent.UserUnsubscribe, SubscriptionHistoryEvent.UserCancelUnsubscribe),
user_id__in=user_ids).values_list('user_id', 'old_subscription_type', 'new_subscription_type','date_changed')
print " .. chewing on subscription data"
subs = list(subs)
print " .. writing to files"
filename = 'subscribe_list_%i.csv'%day
f = open(filename, 'w')
for z in subs:
f.write('%i,%i,%i,%s,%s\n'%(z[0],z[1],z[2],z[3].date(),z[3].time()))