def cursorToArray(cursor, decrypted = False, probe = '', is_researcher=False, map_to_users=False): array = [] for row in cursor: if 'timestamp' in row: row['timestamp'] = int(time.mktime(row['timestamp'].timetuple())) if 'timestamp_added' in row: row['timestamp_added'] = int(time.mktime(row['timestamp_added'].timetuple())) array.append(row) if 'ExperienceSamplingProbe' in probe: for doc in array: doc['answer'] = json.loads(base64.b64decode(doc['answer'])) if 'EpidemicProbe' in probe: for doc in array: doc['data'] = json.loads(base64.b64decode(doc['data'])) if 'BluetoothProbe' not in probe: return array if decrypted: anonymizer = Anonymizer() return anonymizer.deanonymizeDocument(array, probe) if is_researcher and map_to_users: deviceInventory = device_inventory.DeviceInventory() for doc in array: try: user_temp = deviceInventory.mapBtToUser(doc['bt_mac'], doc['timestamp'], use_mac_if_empty=False) if user_temp is not None: doc['scanned_user'] = user_temp else: doc['scanned_user'] = '' except KeyError: doc['scanned_user'] = '' return array
def cursorToArray(cursor, decrypted = False, probe = '', is_researcher=False, map_to_users=False): array = [doc for doc in cursor] if 'BluetoothProbe' in probe: if decrypted: anonymizer = Anonymizer() return anonymizer.deanonymizeDocument(array, probe) elif is_researcher and map_to_users: deviceInventory = device_inventory.DeviceInventory() for doc in array: for device in doc['data']['DEVICES']: try: user_temp = deviceInventory.mapBtToUser(device['android_bluetooth_device_extra_DEVICE']['mAddress'], doc['data']['TIMESTAMP'], use_mac_if_empty=False) if user_temp is not None: device['user'] = user_temp else: device['user'] = '' except KeyError: device['user'] = '' return array else: return array else: return array
def calculate_epi_summary():
this_path = os.path.split(os.path.realpath(__file__))[0] + '/'
db = db_wrapper.DatabaseHelper()
waves = read_waves()
an = anonymizer.Anonymizer()
inventory = device_inventory.DeviceInventory()
localtz = pytz.timezone('Europe/Copenhagen')
f = open(this_path+'epi_summary.json', 'w')
print waves
states = defaultdict(lambda: defaultdict(set))
for wave in sorted(waves):
print wave
last_id = 0
next_wave_begins = waves[wave]['next_wave_begins']
while True:
cur = db.retrieve(params={'limit':100000, 'sortby':'timestamp', 'order':1, 'after': last_id, 'start_date': wave, 'end_date': next_wave_begins}, collection='edu_mit_media_funf_probe_builtin_EpidemicProbe', roles='')
if cur.rowcount == 0: break
last_id += 1
for row in cur:
try:
if row['timestamp_added'] < row['timestamp'] : continue
user = row['user']
data = row['data']
data = json.loads(base64.b64decode(data))
#if not data['wave_no'] == int(waves[wave]['wave_no']): continue
timestamp = data['TIMESTAMP']
date = localtz.localize(datetime.datetime.fromtimestamp(timestamp)).date()
state = data['self_state']
#try: side_effects_lost_points = int(data['side_effects_lost_points'])
#except: continue
#print user, timestamp, state, side_effects_lost_points, date
#if state == 'V' and side_effects_lost_points > 0: state = 'VS'
states[user][date].add(state)
except: continue
user_day_states = defaultdict(lambda: defaultdict(str)) #final state for user in given day
all_users_in_day = defaultdict(int)
all_days = set()
all_users = set()
for user in states:
all_users.add(user)
for date in states[user]:
if 'I' in states[user][date]: user_day_state = 'I'
elif 'E' in states[user][date]: user_day_state = 'I'
elif 'VS' in states[user][date]: user_day_state = 'VS'
elif 'V' in states[user][date]: user_day_state = 'V'
elif 'S' in states[user][date]: user_day_state = 'S'
else: continue
user_day_states[user][date] = user_day_state
all_users_in_day[date] += 1
global_stats = defaultdict(lambda: defaultdict(float)) #for given day, how many users with given final state
for user in user_day_states:
for date in user_day_states[user]:
all_days.add(date)
global_stats[date][user_day_states[user][date]] += 1
for date in global_stats:
for status in global_stats[date]:
global_stats[date][status] /= float(all_users_in_day[date])
print date, status, global_stats[date][status]
interactions = defaultdict(set)
for wave in sorted(waves):
last_id = 0
next_wave_begins = waves[wave]['next_wave_begins']
while True:
cur = db.retrieve(params={'limit':100000, 'sortby':'timestamp', 'order':1, 'after': last_id, 'start_date': wave, 'end_date': next_wave_begins}, collection='edu_mit_media_funf_probe_builtin_BluetoothProbe', roles='')
if cur.rowcount == 0: break
last_id += 1
for row in cur:
if row['timestamp_added'] < row['timestamp'] : continue
user = row['user']
timestamp = row['timestamp']
bt_mac = row['bt_mac']
if bt_mac == '-1': continue
userB = inventory.mapBtToUser(bt_mac, long(timestamp.strftime('%s')))
if userB == bt_mac: continue
interactions[user].add(timestamp.strftime('%s')+'_'+userB)
for user in all_users:
infected_interactions = defaultdict(set)
vaccinated_interactions = defaultdict(set)
vaccinated_side_interactions = defaultdict(set)
susceptible_interactions = defaultdict(set)
for interaction in interactions[user]:
t = long(interaction.split('_')[0])/int(300)
tt = long(interaction.split('_')[0])
date = localtz.localize(datetime.datetime.fromtimestamp(tt)).date()
userB = interaction.split('_')[1]
all_days.add(date)
if user_day_states[userB][date] == 'I': infected_interactions[date].add(userB)
if user_day_states[userB][date] == 'VS': vaccinated_side_interactions[date].add(userB)
if user_day_states[userB][date] == 'V': vaccinated_interactions[date].add(userB)
if user_day_states[userB][date] == 'S': susceptible_interactions[date].add(userB)
user_dict = {}
user_dict['user'] = user
user_dict['values'] = {}
for day in sorted(all_days):
if day < (datetime.datetime.now().date() + datetime.timedelta(days=-4)): continue
all_l = float(len(infected_interactions[day]) + len(vaccinated_interactions[day]) + len(vaccinated_side_interactions[day]) + len(susceptible_interactions[day]))
user_dict['values'][str(day)] = {}
if all_l == 0:
user_dict['values'][str(day)]['infected_interactions'] = 0
user_dict['values'][str(day)]['vaccinated_interactions'] = 0
user_dict['values'][str(day)]['vaccinated_side_interactions'] = 0
user_dict['values'][str(day)]['susceptible_interactions'] = 0
else:
user_dict['values'][str(day)]['infected_interactions'] = len(infected_interactions[day])/all_l
user_dict['values'][str(day)]['vaccinated_interactions'] = len(vaccinated_interactions[day])/all_l
user_dict['values'][str(day)]['vaccinated_side_interactions'] = len(vaccinated_side_interactions[day])/all_l
user_dict['values'][str(day)]['susceptible_interactions'] = len(susceptible_interactions[day])/all_l
user_dict['values'][str(day)]['infected_all'] = global_stats[day]['I']
user_dict['values'][str(day)]['vaccinated_all'] = global_stats[day]['V']
user_dict['values'][str(day)]['vaccinated_side_all'] = global_stats[day]['VS']
user_dict['values'][str(day)]['susceptible_all'] = global_stats[day]['S']
user_dict['values'][str(day)]['day'] = str(day)
f.write(json.dumps(user_dict) + '\n')
f.close()
def calculate_epi_summary_OLD():
this_path = os.path.split(os.path.realpath(__file__))[0] + '/'
db = db_wrapper.DatabaseHelper()
waves = read_waves()
an = anonymizer.Anonymizer()
inventory = device_inventory.DeviceInventory()
localtz = pytz.timezone('Europe/Copenhagen')
f = open(this_path+'epi_summary.json', 'w')
for wave in sorted(waves):
last_id = 0
next_wave_begins = waves[wave]['next_wave_begins']
interactions = defaultdict(set)
while True:
cur = db.retrieve(params={'limit':100000, 'sortby':'timestamp', 'order':1, 'after': last_id, 'start_date': wave, 'end_date': next_wave_begins}, collection='edu_mit_media_funf_probe_builtin_BluetoothProbe', roles='')
if cur.rowcount == 0: break
last_id += 1
for row in cur:
if row['timestamp_added'] < row['timestamp'] : continue
user = row['user']
timestamp = row['timestamp']
bt_mac = row['bt_mac']
bt_name = row['name']
if bt_mac == '-1': continue
if bt_name: bt_name = an.deanonymizeValue('bluetooth_name', bt_name)
else: bt_name = ''
userB = inventory.mapBtToUser(bt_mac, long(timestamp.strftime('%s')))
if userB == bt_mac: continue
interactions[user].add(timestamp.strftime('%s')+'_'+userB+'_'+bt_name.decode('utf-8'))
for user in interactions:
save_string = ''
infected_interactions = defaultdict(set)
vaccinated_interactions = defaultdict(set)
susceptible_interactions = defaultdict(set)
all_days = set()
for interaction in interactions[user]:
t = long(interaction.split('_')[0])/int(300)
tt = long(interaction.split('_')[0])
date = localtz.localize(datetime.datetime.fromtimestamp(tt)).date()
userB = interaction.split('_')[1]
name = interaction.split('_')[2]
all_days.add(date)
if name.endswith(waves[wave]['infected_tag'].decode('utf-8')): infected_interactions[date].add(str(t) + userB)
elif name.endswith(waves[wave]['vaccinated_tag']): vaccinated_interactions[date].add(str(t) + userB)
else: susceptible_interactions[date].add(str(t) + userB)
user_dict = {}
user_dict['user'] = user
user_dict['values'] = {}
for day in sorted(all_days):
user_dict['values'][str(day)] = {}
user_dict['values'][str(day)]['wave_no'] = int(waves[wave]['wave_no'])
user_dict['values'][str(day)]['infected_interactions'] = len(infected_interactions[day])
user_dict['values'][str(day)]['vaccinated_interactions'] = len(vaccinated_interactions[day])
user_dict['values'][str(day)]['susceptible_interactions'] = len(susceptible_interactions[day])
f.write(json.dumps(user_dict) + '\n')
f.close()
def analyse_epidemic():
db = db_wrapper.DatabaseHelper()
#cur = db.retrieve(params={'limit':1000, 'sortby':'timestamp', 'order':-1, 'where': {'user':[USER]}}, collection='edu_mit_media_funf_probe_builtin_EpidemicProbe', roles='')
cur = db.retrieve(
params={
'limit': 10000000,
'sortby': 'timestamp',
'order': -1,
'start_date': 1407794400,
'end_date': 1407880800
},
collection='edu_mit_media_funf_probe_builtin_EpidemicProbe',
roles='')
lt = 0
inventory = device_inventory.DeviceInventory()
states = defaultdict(set)
for row in cur:
data = json.loads(base64.b64decode(row['data']))
user = row['user']
timestamp = row['timestamp']
timestamp_2 = data['TIMESTAMP']
# if not (timestamp_2 >= 1407708000 and timestamp_2 <= 1407794400): continue
# print user, timestamp, timestamp_2, lt - timestamp_2, data['self_state'], data['infected_tag'], data['to_recover_time'],
# try: print data['self_state'], timestamp
# except: print ''
# except: pass
states[user].add(data['self_state'])
lt = timestamp_2
#if timestamp < datetime.datetime(2014, 07, 29): continue
final_states = defaultdict(str)
for user in states:
if 'I' in states[user]: final_states[user] = 'I'
elif 'E' in states[user]: final_states[user] = 'I'
elif 'V' in states[user]: final_states[user] = 'V'
elif 'S' in states[user]: final_states[user] = 'S'
global_states = defaultdict(int)
for user in final_states:
global_states[final_states[user]] += 1
print global_states
cur = db.retrieve(
params={
'limit': 10000000,
'sortby': 'timestamp',
'order': -1,
'start_date': 1407794400,
'end_date': 1407880800,
'where': {
'user': [USER]
}
},
collection='edu_mit_media_funf_probe_builtin_BluetoothProbe',
roles='')
interactions = set()
for row in cur:
if row['timestamp_added'] < row['timestamp']: continue
user = row['user']
timestamp = row['timestamp']
bt_mac = row['bt_mac']
if bt_mac == '-1': continue
userB = inventory.mapBtToUser(bt_mac, long(timestamp.strftime('%s')))
if userB == bt_mac: continue
interactions.add(userB)
for user in interactions:
print user, final_states[user]