def findHotSpotsTask():
profiles = Profile.objects.all()
hot_spots_frequencies = {}
for profile in profiles:
try:
optin_object = Optin.objects.get(datastore_owner = profile, app_id = "Living Lab", lab_id = "MIT-FIT")
except Optin.DoesNotExist:
optin_object = None
if optin_object:
if optin_object.data_aggregation == 0:
continue
internal_data_store = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
hot_spots_of_user = hotSpotsComputation(internal_data_store)
for hot_spot_of_user in hot_spots_of_user:
if hot_spot_of_user in hot_spots_frequencies:
hot_spots_frequencies[hot_spot_of_user] = hot_spots_frequencies[hot_spot_of_user] + 1
else:
hot_spots_frequencies[hot_spot_of_user] = 1
hot_spots_frequencies_list = []
for hot_spot in hot_spots_frequencies:
hot_spot_frequency = { "lat": hot_spot[0], "lng": hot_spot[1], "frequency": hot_spots_frequencies[hot_spot]}
hot_spots_frequencies_list.append(hot_spot_frequency)
for profile in profiles:
internal_data_store = getInternalDataStore(profile, "Living Lab", "Frequent Locations", "")
internal_data_store.saveAnswer("hotspots", hot_spots_frequencies_list)
def findActiveLocationsTask():
profiles = Profile.objects.all()
location_frequencies = {}
for profile in profiles:
try:
optin_object = Optin.objects.get(datastore_owner = profile, app_id = "Living Lab", lab_id = "MIT-FIT")
except Optin.DoesNotExist:
optin_object = None
if optin_object:
if optin_object.data_aggregation == 0:
continue
#print profile.id
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
values = activeLocationsComputation(internalDataStore)
for value in values:
location_value = tuple((round(value[0],4), round(value[1],4)))
if location_value in location_frequencies:
location_frequencies[location_value] = location_frequencies[location_value] + 1
else:
location_frequencies[location_value] = 1
location_frequencies_list = []
for key in location_frequencies:
location_value = { "lat": key[0], "lng": key[1], "count": location_frequencies[key]}
location_frequencies_list.append(location_value)
for profile in profiles:
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
internalDataStore.saveAnswer("activeLocations", location_frequencies_list)
def leaderboardComputationTask():
profiles = Profile.objects.all()
# profiles = []
# profiles.append(Profile.objects.get(uuid="341cc5cd-0f42-45f1-9f66-273ac3ed8b2e"))
unsorted_dict = {}
for profile in profiles:
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
values = aggregateLeaderboardComputation(internalDataStore, "activityStats", leaderboardComputation, False)
unsorted_dict[profile.uuid] = LeaderboardRanking({ "average_activity_rate": values[0]["average_activity_rate"], "max_high_activity_rate": values[0]["max_high_activity_rate"], "min_low_activity_rate": values[0]["min_low_activity_rate"]})
#sorted_dict = sorted(unsorted_dict.values(), key=attrgetter('average_activity_rate'))
sorted_dict = sorted(unsorted_dict, key = lambda uuid: unsorted_dict[uuid].average_activity_rate, reverse=False)
average_activity_rates_list = []
for uuid in sorted_dict:
average_activity_rates_list.append(unsorted_dict[uuid].get_average_activity_rate())
for uuid in sorted_dict:
profile = Profile.objects.get(uuid=uuid)
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
percentileValue = calculatePercentile(average_activity_rates_list, unsorted_dict[uuid].get_average_activity_rate())
user_activity_list = []
user_activity_dict = { "average_activity_rate": unsorted_dict[uuid].get_average_activity_rate(), "max_high_activity_rate": unsorted_dict[uuid].get_max_high_activity_rate(), "min_low_activity_rate": unsorted_dict[uuid].get_min_low_activity_rate(), "rank": {"own": len(sorted_dict) - sorted_dict.index(uuid), "total": len(sorted_dict), "percentile": percentileValue} }
user_activity_list.append(user_activity_dict)
internalDataStore.saveAnswer("activityStats", user_activity_list)
def findActiveTimesTask():
profiles = Profile.objects.all()
time_averages = [0] * 24
num_users = 0
for profile in profiles:
num_users += 1
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
user_time_averages = activeTimesComputation(internalDataStore)
for i in range(len(time_averages)):
time_averages[i] += user_time_averages[i]
#print num_users
#print time_averages
for i in range(len(time_averages)):
time_averages[i] = time_averages[i] // num_users
for profile in profiles:
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
internalDataStore.saveAnswer("activeTimes", time_averages)
def recentSocialHealthScores2():
profiles = Profile.objects.all()
startTime = getStartTime(6, True)
currentTime = time.time()
timeRanges = [(start, start + 3600*4) for start in range(int(startTime), int(currentTime), 3600*4)]
sums = {"activity": 0, "social": 0, "focus": 0}
activeUsers = []
data = {}
for profile in profiles:
token = getToken(profile, "app-uuid")
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
activityLevels = aggregateForUser(internalDataStore, "RecentActivityByHour", timeRanges, activityForTimeRange, False)
if len(activityLevels) > 0:
socialLevels = aggregateForUser(internalDataStore, "RecentSocialByHour", timeRanges, socialForTimeRange, True)
focusLevels = aggregateForUser(internalDataStore, "RecentFocusByHour", timeRanges, focusForTimeRange, True)
activityScore = computeActivityScore(activityLevels)
socialScore = computeSocialScore(socialLevels)
focusScore = computeFocusScore(focusLevels)
sums["activity"] += activityScore
sums["social"] += socialScore
sums["focus"] += focusScore
activeUsers.append(profile)
data[profile.uuid] = {}
data[profile.uuid]["user"] = {
"activity": activityScore,
"social": socialScore,
"focus": focusScore
}
numUsers = len(activeUsers)
if numUsers > 0:
averages = { k: sums[k] / numUsers for k in sums }
variances = { k: [(data[p.uuid]["user"][k] - averages[k])**2 for p in activeUsers] for k in averages }
stdDevs = { k: math.sqrt(sum(variances[k]) / len(variances[k])) for k in variances }
for profile in activeUsers:
token = getToken(profile, "app-uuid")
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
data[profile.uuid]["averageLow"] = { k: max(0, averages[k] - stdDevs[k]) for k in stdDevs }
data[profile.uuid]["averageHigh"] = { k: min(averages[k] + stdDevs[k], 10) for k in stdDevs }
internalDataStore.saveAnswer("socialhealth", data[profile.uuid])
return data
def recentSocialHealthScores2():
profiles = Profile.objects.all()
startTime = getStartTime(6, True)
currentTime = time.time()
timeRanges = [(start, start + 3600*4) for start in range(int(startTime), int(currentTime), 3600*4)]
sums = {"activity": 0, "social": 0, "focus": 0}
activeUsers = []
data = {}
for profile in profiles:
token = getToken(profile, "app-uuid")
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
activityLevels = aggregateForUser(internalDataStore, "RecentActivityByHour", timeRanges, activityForTimeRange, False)
if len(activityLevels) > 0:
socialLevels = aggregateForUser(internalDataStore, "RecentSocialByHour", timeRanges, socialForTimeRange, True)
focusLevels = aggregateForUser(internalDataStore, "RecentFocusByHour", timeRanges, focusForTimeRange, True)
activityScore = computeActivityScore(activityLevels)
socialScore = computeSocialScore(socialLevels)
focusScore = computeFocusScore(focusLevels)
sums["activity"] += activityScore
sums["social"] += socialScore
sums["focus"] += focusScore
activeUsers.append(profile)
data[profile.uuid] = {}
data[profile.uuid]["user"] = {
"activity": activityScore,
"social": socialScore,
"focus": focusScore
}
numUsers = len(activeUsers)
if numUsers > 0:
averages = { k: sums[k] / numUsers for k in sums }
variances = { k: [(data[p.uuid]["user"][k] - averages[k])**2 for p in activeUsers] for k in averages }
stdDevs = { k: math.sqrt(sum(variances[k]) / len(variances[k])) for k in variances }
for profile in activeUsers:
token = getToken(profile, "app-uuid")
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
data[profile.uuid]["averageLow"] = { k: max(0, averages[k] - stdDevs[k]) for k in stdDevs }
data[profile.uuid]["averageHigh"] = { k: min(averages[k] + stdDevs[k], 10) for k in stdDevs }
internalDataStore.saveAnswer("socialhealth", data[profile.uuid])
return data
def findRecentPlaces():
currentTime = time.time()
today = date.fromtimestamp(currentTime)
startTime = time.mktime((today - timedelta(days=14)).timetuple())
# Note: we're not taking the full 9-5 sampling. Clustering is expensive, so anything we can leave out helps...
# Combined with the fact that "lunch" time might not be indicative of work locations, this might be more accurate anyway
nineToFives = [(nine, nine + 3600*8) for nine in range(int(startTime + 3600*9), int(currentTime), 3600*24)]
#nineToFives.extend([(two, two + 3600*2) for two in range(int(startTime + 3600*14), int(currentTime), 3600*24)])
midnightToSixes = [(midnight, midnight + 3600*6) for midnight in range(int(startTime), int(currentTime), 3600* 24)]
data = {}
profiles = Profile.objects.all()
for profile in profiles:
ids = getInternalDataStore(profile, "Living Lab", "My Places", "")
#ids.saveAnswer("RecentPlaces", [])
work = findTopBoundingRegion(ids, nineToFives)
home = findTopBoundingRegion(ids, midnightToSixes)
data[profile.uuid] = []
if work is not None:
data[profile.uuid].append({ "key": "work", "bounds": work})
if home is not None:
data[profile.uuid].append({ "key": "home", "bounds": home})
ids.saveAnswer("RecentPlaces", data[profile.uuid])
#print "Finding work locations..."
#data = findRecentPlaceBounds("work", nineToFives)
#print "Finding home locations..."
#data = findRecentPlaceBounds("home", midnightToSixes)
print "... done with RecentPlaces"
return data
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token': request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(
uuid=datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab",
"Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(
convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data = {}
pds_data['time'] = json_insert.get('timestamp')
pds_data['value'] = json_insert
pds_data['key'] = name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert) + '\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s" % e
result = {'success': False, 'error_message': e.message}
finally:
response_dict = {"status": "success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def data(request):
print "\n\n DATA REACHED \n\n"
'''parse json data and upload them to your PDS'''
if request.method == 'GET':
return HttpResponseBadRequest("GET requests disallowed", content_type = 'text/json')
# get necessary info from outside request
device_id = request.POST['device_id']
# flie_hash = request.POST['file_hash']
token = request.POST['bearer_token']
data = json.loads(request.POST['data'])
result = {}
# use token to authorize writing to the PDS
authorization = PDSAuthorization("ios_write", audit_enabled=True)
if (not authorization.is_authorized(request)):
print "authorization not received"
return HttpResponse("Unauthorized", status=401)
# Get the relevant internal data store
datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = device_id)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token)
# write to the datastore
result = extractAndInsertData(data, internalDataStore, token)
# let the client know what happened
print result
if result.get('status'):
return HttpResponse(json.dumps(result), content_type='application/json')
else:
return HttpResponseBadRequest(json.dumps(result), \
content_type='application/json')
def get_internal_datastore(self, request):
if request and "datastore_owner__uuid" in request.GET and "bearer_token" in request.GET:
profile, created = Profile.objects.get_or_create(
uuid=request.GET["datastore_owner__uuid"])
token = request.GET["bearer_token"]
return getInternalDataStore(profile, "", "", token)
return None
def recentProbeDataScores():
profiles = Profile.objects.all()
for profile in profiles:
startTime = socialhealth_tasks.getStartTime(6, True)
currentTime = time.time()
timeRanges = [
(start, start + 3600)
for start in range(int(startTime), int(currentTime), 3600)
]
probeAnswerKeys = {
'recentActivityProbeByHour': 'ActivityProbe',
'recentSmsProbeByHour': 'SmsProbe',
'recentCallLogProbeByHour': 'CallLogProbe',
'recentBluetoothProbeByHour': 'BluetoothProbe',
'recentWifiProbeByHour': 'WifiProbe',
'recentSimpleLocationProbeByHour': 'LocationProbe',
'recentRunningApplicationsProbeByHour': 'RunningApplicationsProbe',
'recentHardwareInfoProbeByHour': 'HardwareInfoProbe',
'recentAppUsageProbeByHour': 'AppUsageProbe'
}
internalDataStore = getInternalDataStore(profile, "Living Lab",
"MIT-FIT", "")
#for testing, currently use the following user
#if profile.uuid == "341cc5cd-0f42-45f1-9f66-273ac3ed8b2e":
for probeAnswerKey, probe in probeAnswerKeys.iteritems():
probeLevels = aggregateForUser(probe, internalDataStore,
probeAnswerKey, timeRanges,
probeForTimeRange, False)
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token':request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s"%e
result = {'success':False, 'error_message':e.message}
finally:
response_dict = {"status":"success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def recentSocialHealthScores():
profiles = Profile.objects.all()
data = {}
activityScores = recentActivityScore()
socialScores = recentSocialScore()
focusScores = recentFocusScore()
scoresList = [activityScores.values(), socialScores.values(), focusScores.values()]
print scoresList
# scoresList = [[d for d in scoreList if d > 0.0] for scoreList in scoresList]
averages = [sum(scores) / len(scores) if len(scores) > 0 else 0 for scores in scoresList]
variances = [map(lambda x: (x - averages[i]) * (x - averages[i]), scoresList[i]) for i in range(len(scoresList))]
stdDevs = [math.sqrt(sum(variances[i]) / len(scoresList[i])) for i in range(len(scoresList))]
activityStdDev = stdDevs[0]
socialStdDev = stdDevs[1]
focusStdDev = stdDevs[2]
print "Averages (activity, social, focus):"
print averages
print "Standard Deviations (activity, social, focus):"
print stdDevs
for profile in [p for p in profiles if p.uuid in activityScores.keys()]:
print "storing %s" % profile.uuid
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", "")
data[profile.uuid] = []
#pdb.set_trace()
#data[profile.uuid].append({ "key": "activity", "layer": "User", "value": activityScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "social", "layer": "User", "value": socialScores.get(profile.uuid, 0) })
#data[profile.uuid].append({ "key": "focus", "layer": "User", "value": focusScores.get(profile.uuid, 0) })
#data[profile.uuid].append({ "key": "activity", "layer": "averageLow", "value": max(0, averages[0] - stdDevs[0])})
data[profile.uuid].append({ "key": "social", "layer": "averageLow", "value": max(0, averages[1] - stdDevs[1]) })
#data[profile.uuid].append({ "key": "focus", "layer": "averageLow", "value": max(0, averages[2] - stdDevs[2]) })
#data[profile.uuid].append({ "key": "activity", "layer": "averageHigh", "value": min(averages[0] + stdDevs[0], 10) })
data[profile.uuid].append({ "key": "social", "layer": "averageHigh", "value": min(averages[1] + stdDevs[1], 10) })
#data[profile.uuid].append({ "key": "focus", "layer": "averageHigh", "value": min(averages[2] + stdDevs[2], 10) })
data[profile.uuid].append({ "key": "regularity", "layer": "User", "value": focusScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "regularity", "layer": "averageLow", "value": max(0, averages[2] - stdDevs[2]) })
data[profile.uuid].append({ "key": "regularity", "layer": "averageHigh", "value": min(averages[2] + stdDevs[2], 10) })
data[profile.uuid].append({ "key": "physical activity", "layer": "User", "value": activityScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "physical activity", "layer": "averageLow", "value": max(0, averages[0] - stdDevs[0])})
data[profile.uuid].append({ "key": "physical activity", "layer": "averageHigh", "value": min(averages[0] + stdDevs[0], 10) })
internalDataStore.saveAnswer("socialhealth", data[profile.uuid])
# After we're done, re-compute the time graph data to include zeros for blanks
# not ideal to compute this twice, but it gets the job done
recentActivityLevels(True)
# Purposely excluding social and focus scores - blanks are includede in their calculations as blank could imply actual zeroes, rather than missing data
#recentSocialLevels(True)
#recentFocusLevels(True)
return data
def recentSocialHealthScores():
profiles = Profile.objects.all()
data = {}
activityScores = recentActivityScore()
socialScores = recentSocialScore()
focusScores = recentFocusScore()
scoresList = [activityScores.values(), socialScores.values(), focusScores.values()]
print scoresList
# scoresList = [[d for d in scoreList if d > 0.0] for scoreList in scoresList]
averages = [sum(scores) / len(scores) if len(scores) > 0 else 0 for scores in scoresList]
variances = [map(lambda x: (x - averages[i]) * (x - averages[i]), scoresList[i]) for i in range(len(scoresList))]
stdDevs = [math.sqrt(sum(variances[i]) / len(scoresList[i])) if len(scoresList[i]) > 0 else 0 for i in range(len(scoresList))]
activityStdDev = stdDevs[0]
socialStdDev = stdDevs[1]
focusStdDev = stdDevs[2]
print "Averages (activity, social, focus):"
print averages
print "Standard Deviations (activity, social, focus):"
print stdDevs
for profile in [p for p in profiles if p.uuid in activityScores.keys()]:
print "storing %s" % profile.uuid
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", "")
data[profile.uuid] = []
#pdb.set_trace()
#data[profile.uuid].append({ "key": "activity", "layer": "User", "value": activityScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "social", "layer": "User", "value": socialScores.get(profile.uuid, 0) })
#data[profile.uuid].append({ "key": "focus", "layer": "User", "value": focusScores.get(profile.uuid, 0) })
#data[profile.uuid].append({ "key": "activity", "layer": "averageLow", "value": max(0, averages[0] - stdDevs[0])})
data[profile.uuid].append({ "key": "social", "layer": "averageLow", "value": max(0, averages[1] - stdDevs[1]) })
#data[profile.uuid].append({ "key": "focus", "layer": "averageLow", "value": max(0, averages[2] - stdDevs[2]) })
#data[profile.uuid].append({ "key": "activity", "layer": "averageHigh", "value": min(averages[0] + stdDevs[0], 10) })
data[profile.uuid].append({ "key": "social", "layer": "averageHigh", "value": min(averages[1] + stdDevs[1], 10) })
#data[profile.uuid].append({ "key": "focus", "layer": "averageHigh", "value": min(averages[2] + stdDevs[2], 10) })
data[profile.uuid].append({ "key": "regularity", "layer": "User", "value": focusScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "regularity", "layer": "averageLow", "value": max(0, averages[2] - stdDevs[2]) })
data[profile.uuid].append({ "key": "regularity", "layer": "averageHigh", "value": min(averages[2] + stdDevs[2], 10) })
data[profile.uuid].append({ "key": "physical activity", "layer": "User", "value": activityScores.get(profile.uuid, 0) })
data[profile.uuid].append({ "key": "physical activity", "layer": "averageLow", "value": max(0, averages[0] - stdDevs[0])})
data[profile.uuid].append({ "key": "physical activity", "layer": "averageHigh", "value": min(averages[0] + stdDevs[0], 10) })
internalDataStore.saveAnswer("socialhealth", data[profile.uuid])
# After we're done, re-compute the time graph data to include zeros for blanks
# not ideal to compute this twice, but it gets the job done
recentActivityLevels(True)
# Purposely excluding social and focus scores - blanks are includede in their calculations as blank could imply actual zeroes, rather than missing data
#recentSocialLevels(True)
#recentFocusLevels(True)
return data
def findHotSpotsTask():
profiles = Profile.objects.all()
hot_spots_frequencies = {}
for profile in profiles:
try:
optin_object = Optin.objects.get(datastore_owner=profile,
app_id="Living Lab",
lab_id="MIT-FIT")
except Optin.DoesNotExist:
optin_object = None
if optin_object:
if optin_object.data_aggregation == 0:
continue
internal_data_store = getInternalDataStore(profile, "Living Lab",
"MIT-FIT", "")
hot_spots_of_user = hotSpotsComputation(internal_data_store)
for hot_spot_of_user in hot_spots_of_user:
if hot_spot_of_user in hot_spots_frequencies:
hot_spots_frequencies[
hot_spot_of_user] = hot_spots_frequencies[
hot_spot_of_user] + 1
else:
hot_spots_frequencies[hot_spot_of_user] = 1
hot_spots_frequencies_list = []
for hot_spot in hot_spots_frequencies:
hot_spot_frequency = {
"lat": hot_spot[0],
"lng": hot_spot[1],
"frequency": hot_spots_frequencies[hot_spot]
}
hot_spots_frequencies_list.append(hot_spot_frequency)
for profile in profiles:
internal_data_store = getInternalDataStore(profile, "Living Lab",
"Frequent Locations", "")
internal_data_store.saveAnswer("hotspots", hot_spots_frequencies_list)
def testGetData():
profiles = Profile.objects.all()
#print profiles[17].uuid
# token = socialhealth_tasks.getToken(profiles[17], "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profiles[17], "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
probes = ["LocationProbe", "ActivityProbe", "SmsProbe", "CallLogProbe", "BluetoothProbe", "WifiProbe", "ScreenProbe"]
startTime = 1403136000
endTime = 1403222400
internalDataStore.getData(probes[1], startTime, endTime)
def recentProbeCounts():
profiles = Profile.objects.all()
startTime = getStartTime(1, False)
for profile in profiles:
ids = getInternalDataStore(profile, "", "Living Lab", "")
probes = ["ActivityProbe", "SimpleLocationProbe", "CallLogProbe", "SmsProbe", "WifiProbe", "BluetoothProbe"]
answer = {}
for probe in probes:
data = ids.getData(probe, startTime, None)
answer[probe] = data.count()
ids.saveAnswer("RecentProbeCounts", answer)
def data(request):
print "\n\n DATA REACHED \n\n"
'''parse json data and upload them to your PDS'''
# does not allow get method when uploading json data. currently not parsing jsons correctly
if request.method == 'GET':
return HttpResponseBadRequest("GET requests disallowed",
content_type='text/json')
# get necessary info from outside request
device_id = request.POST['device_id']
# flie_hash = request.POST['file_hash']
# temp token to insert data
# 'bearer_token' = 3f4851fd8a
token = request.POST['bearer_token']
data = json.loads(request.POST['data'])
result = {}
funfresult = {}
# use token to authorize writing to the PDS
authorization = PDSAuthorization("ios_write", audit_enabled=True)
if (not authorization.is_authorized(request)):
print "authorization not received"
return HttpResponse("Unauthorized", status=401)
# Get the relevant internal data store
datastore_owner, ds_owner_created = Profile.objects.get_or_create(
uuid=device_id)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab",
"Social Health Tracker", token)
# write to the datastore
result = extractAndInsertData(data, internalDataStore, token)
# write to funf data
funfresult = insertfunf(data, internalDataStore, token)
blankdata = getfunfdata.insertblankdata()
try:
internalDataStore.saveData(blankdata, 'funf')
blankdata = {'status': 'ok', 'entries_inserted': 1}
except Exception as e:
print "\n\nException from os_connector on pds: %s\n" % e
blankdata = {'success': False, 'error_message': e.message}
# let the client know what happened
print result
if result.get('status'):
return HttpResponse(json.dumps(result),
content_type='application/json')
else:
return HttpResponseBadRequest(json.dumps(result), \
content_type='application/json')
def populateEventsForUsers():
profiles = Profile.objects.all()
for profile in profiles:
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
events = []
random_numbers = random.sample(range(25), 7)
for random_number in random_numbers:
events.append(data[random_number])
#print events
internalDataStore.saveAnswer("mitfitEventRegistrations", events)
def aggregateForAllUsers(answerKey, timeRanges, aggregator, serviceId, includeBlanks = False, mean = None, dev = None):
profiles = Profile.objects.all()
aggregates = {}
for profile in profiles:
# NOTE: need a means of getting at a token for authorizing this task to run. For now, we're not checking anyway, so it's blank
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", "")
# if mean is None or dev is None:
data = aggregateForUser(internalDataStore, answerKey, timeRanges, aggregator, includeBlanks)
# else:
# data = aggregateForUser(profile, answerKey, timeRanges, aggregator, includeBlanks, mean.get(profile.uuid), dev.get(profile.uuid))
if data is not None and len(data) > 0:
aggregates[profile.uuid] = data
return aggregates
def aggregateForAllUsers(answerKey, timeRanges, aggregator, serviceId, includeBlanks = False, mean = None, dev = None):
profiles = Profile.objects.all()
aggregates = {}
for profile in profiles:
# NOTE: need a means of getting at a token for authorizing this task to run. For now, we're not checking anyway, so it's blank
internalDataStore = getInternalDataStore(profile, "Living Lab", "Social Health Tracker", "")
# if mean is None or dev is None:
data = aggregateForUser(internalDataStore, answerKey, timeRanges, aggregator, includeBlanks)
# else:
# data = aggregateForUser(profile, answerKey, timeRanges, aggregator, includeBlanks, mean.get(profile.uuid), dev.get(profile.uuid))
if data is not None and len(data) > 0:
aggregates[profile.uuid] = data
return aggregates
def recentProbeCounts():
profiles = Profile.objects.all()
startTime = getStartTime(1, False)
for profile in profiles:
ids = getInternalDataStore(profile, "", "Living Lab", "")
probes = [
"ActivityProbe", "SimpleLocationProbe", "CallLogProbe", "SmsProbe",
"WifiProbe", "BluetoothProbe"
]
answer = {}
for probe in probes:
data = ids.getData(probe, startTime, None)
answer[probe] = data.count()
ids.saveAnswer("RecentProbeCounts", answer)
def data(request):
print "\n\n DATA REACHED \n\n"
'''parse json data and upload them to your PDS'''
# does not allow get method when uploading json data. currently not parsing jsons correctly
if request.method == 'GET':
return HttpResponseBadRequest("GET requests disallowed", content_type = 'text/json')
# get necessary info from outside request
device_id = request.POST['device_id']
# flie_hash = request.POST['file_hash']
# temp token to insert data
# 'bearer_token' = 3f4851fd8a
token = request.POST['bearer_token']
data = json.loads(request.POST['data'])
result = {}
funfresult = {}
# use token to authorize writing to the PDS
authorization = PDSAuthorization("ios_write", audit_enabled=True)
if (not authorization.is_authorized(request)):
print "authorization not received"
return HttpResponse("Unauthorized", status=401)
# Get the relevant internal data store
datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = device_id)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token)
# write to the datastore
result = extractAndInsertData(data, internalDataStore, token)
# write to funf data
funfresult = insertfunf(data, internalDataStore, token)
blankdata = getfunfdata.insertblankdata()
try:
internalDataStore.saveData(blankdata, 'funf')
blankdata = {'status':'ok', 'entries_inserted':1}
except Exception as e:
print "\n\nException from os_connector on pds: %s\n" %e
blankdata = {'success':False, 'error_message':e.message}
# let the client know what happened
print result
if result.get('status'):
return HttpResponse(json.dumps(result), content_type='application/json')
else:
return HttpResponseBadRequest(json.dumps(result), \
content_type='application/json')
def recommendEvents():
profiles = Profile.objects.all()
eventRegistrations = {}
userRegistrations = {}
for profile in profiles:
#print profile.uuid
# token = socialhealth_tasks.getToken(profile, "app-uuid")
# internalDataStore = socialhealth_tasks.getInternalDataStore(profile, "Living Lab", "Social Health Tracker", token)
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
eventRegistrations, userEventRegistrations = eventRecommendationComputation(internalDataStore, eventRegistrations, profile.uuid)
userRegistrations[profile.uuid] = userEventRegistrations
#print eventRegistrations
eventSet = set()
jaccardCoefficientDict = {}
for event1 in eventRegistrations.keys():
for event2 in eventRegistrations.keys():
if event1 != event2:
usersEvent1 = eventRegistrations[event1]
usersEvent2 = eventRegistrations[event2]
intersectUsers = list(set(usersEvent1) & set(usersEvent2))
unionUsers = list(set(usersEvent1) | set(usersEvent2))
jaccardCoefficientKey = (event1, event2)
eventSet.add(event1)
eventSet.add(event2)
if len(unionUsers) > 0:
jaccardCoefficientDict[jaccardCoefficientKey] = len(intersectUsers)/len(unionUsers)
else:
jaccardCoefficientDict[jaccardCoefficientKey] = 0
#print jaccardCoefficientDict
for profile in profiles:
#print profile.uuid
recommendedEvents = {}
for userRegisteredEvent in userRegistrations[profile.uuid]:
for event in eventSet:
if userRegisteredEvent != event:
if event in recommendedEvents:
if jaccardCoefficientDict[(userRegisteredEvent, event)] > recommendedEvents[event]:
recommendedEvents[event] = jaccardCoefficientDict[(userRegisteredEvent, event)]
else:
recommendedEvents[event] = jaccardCoefficientDict[(userRegisteredEvent, event)]
#print recommendedEvents
sortedRecommendedEvents = sorted(recommendedEvents.items(), key = lambda recommendedEvent: recommendedEvent[1], reverse=True)
def recentProbeDataScores():
profiles = Profile.objects.all()
for profile in profiles:
startTime = socialhealth_tasks.getStartTime(6, True)
currentTime = time.time()
timeRanges = [(start, start + 3600) for start in range(int(startTime), int(currentTime), 3600)]
probeAnswerKeys = {'recentActivityProbeByHour': 'ActivityProbe', 'recentSmsProbeByHour': 'SmsProbe', 'recentCallLogProbeByHour': 'CallLogProbe',
'recentBluetoothProbeByHour': 'BluetoothProbe', 'recentWifiProbeByHour': 'WifiProbe', 'recentSimpleLocationProbeByHour': 'LocationProbe',
'recentRunningApplicationsProbeByHour': 'RunningApplicationsProbe', 'recentHardwareInfoProbeByHour': 'HardwareInfoProbe',
'recentAppUsageProbeByHour': 'AppUsageProbe'}
internalDataStore = getInternalDataStore(profile, "Living Lab", "MIT-FIT", "")
#for testing, currently use the following user
#if profile.uuid == "341cc5cd-0f42-45f1-9f66-273ac3ed8b2e":
for probeAnswerKey, probe in probeAnswerKeys.iteritems():
probeLevels = aggregateForUser(probe, internalDataStore, probeAnswerKey, timeRanges, probeForTimeRange, False)
def findRecentPlaces():
currentTime = time.time()
today = date.fromtimestamp(currentTime)
startTime = time.mktime((today - timedelta(days=14)).timetuple())
# Note: we're not taking the full 9-5 sampling. Clustering is expensive, so anything we can leave out helps...
# Combined with the fact that "lunch" time might not be indicative of work locations, this might be more accurate anyway
nineToFives = [
(nine, nine + 3600 * 8)
for nine in range(int(startTime + 3600 * 9), int(currentTime), 3600 *
24)
]
#nineToFives.extend([(two, two + 3600*2) for two in range(int(startTime + 3600*14), int(currentTime), 3600*24)])
midnightToSixes = [
(midnight, midnight + 3600 * 6)
for midnight in range(int(startTime), int(currentTime), 3600 * 24)
]
data = {}
profiles = Profile.objects.all()
for profile in profiles:
ids = getInternalDataStore(profile, "Living Lab", "My Places", "")
#ids.saveAnswer("RecentPlaces", [])
work = findTopBoundingRegion(ids, nineToFives)
home = findTopBoundingRegion(ids, midnightToSixes)
data[profile.uuid] = []
if work is not None:
data[profile.uuid].append({"key": "work", "bounds": work})
if home is not None:
data[profile.uuid].append({"key": "home", "bounds": home})
ids.saveAnswer("RecentPlaces", data[profile.uuid])
#print "Finding work locations..."
#data = findRecentPlaceBounds("work", nineToFives)
#print "Finding home locations..."
#data = findRecentPlaceBounds("home", midnightToSixes)
print "... done with RecentPlaces"
return data
def computeAllFingerprints():
for profile in Profile.objects.all():
print profile.uuid
ids = getInternalDataStore(profile, "Living Lab", "Auth", "")
computeFingerprint(ids)
print "... done computing wifi auth fingerprints"
def findRecentPlaceBounds(recentPlaceKey, timeRanges, numPlaces=1, answerKey="RecentPlaces"):
profiles = Profile.objects.all()
data = {}
for profile in profiles:
# TODO: figure out how to get at a token here...
internalDataStore = getInternalDataStore(profile, "Living Lab", "My Places", "")
#dbName = profile.getDBName()
#collection = connection[dbName]["funf"]
locations = []
# An explanation for why we're doing things the way we are below
# (there are a few obvious strategies for finding places in location data):
# 1) Naive approach - take all location samples in all time ranges, find clusters within them,
# take the one with the most points in it.
# 2) Faster, but more complicated - do 1) for each time range individually to get candidate regions.
# Loop over candidate regions, collapsing and "voting" for those that overlap. Take the one with the most votes.
# Notes: This is essentially 2-levels of clustering with the simplification that overlapping regions would
# have been clustered together anyway (ie; bounding boxes should be similar, but not the same, as strategy 1)
# Pros: Faster - each clustering is limited to 100 entries. In practice, this is more than enough.
# If this poses an issue, time ranges can be chosen more carefully (more / shorter time ranges)
# Cons: Bounding boxes aren't the same as 1). In particular, two candidate boxes may not overlap, but should
# have been clustered together anyway.
# 3) Binning pre-process - Same as 1), but perform a binning pre-process on the location data, collapsing multiple
# samples into single entries, with associaated weights.
# Notes: This is essentially a lower-resolution version of strategy 1. Bounding boxes should be lower-resolution
# versions of those from strategy 1.
# Pros: Bounding boxes should be the same as #1. Takes into account all entries when clustering.
# Cons: Less fine-grained control over the number of entries per cluster than #2. In particular, for sparse
# location data, this may not reduce the number of entries we must cluster.
# The following is an implementation of method #2:
potentialRegions = []
#pdb.set_trace()
for timeRange in timeRanges:
# NOTE: is a limit on the number of entries still necessary, if we're choosing the timeRanges carefully?
values = [entry["value"] for entry in internalDataStore.getData("LocationProbe", timeRange[0], timeRange[1]) or []]
# Use all locations except the most gratuitously inaccurate ones
values = [value for value in values if float(value["maccuracy"]) < 100]
clusters = clusterFunfLocations(values, 100)
if (len(clusters) > 0):
#clusters.sort(key = lambda cluster: -len(cluster))
#topClusters = clusters[:min(len(clusters), numPlaces)]
clusterLocations = max(clusters, key= lambda cluster: len(cluster))
if isinstance(clusterLocations, list):
lats = [loc[0] for loc in clusterLocations]
longs = [loc[1] for loc in clusterLocations]
if min(lats) != max(lats) and min(longs) != max(longs):
#Only add regions that aren't degenerate (single points)
potentialRegions.append([min(lats), min(longs), max(lats), max(longs)])
if len(potentialRegions) > 0:
overlaps = [{ "region": r1, "overlapList": [r2 for r2 in potentialRegions if r2 is not r1 and boundsOverlap(r1, r2)]} for r1 in potentialRegions]
reduced = [{ "region": reduce(lambda r1, r2: mergeBoxes(r1,r2), r["overlapList"], r["region"]), "votes": len(r["overlapList"])} for r in overlaps]
reduced.sort(key = lambda r: -r["votes"])
final = []
for r in reduced:
if not listContainsOverlap([f["region"] for f in final], r["region"]):
final.append(r)
mostOverlap = final[:min(len(final), numPlaces)]
mostVoted = [r["region"] for r in mostOverlap]
if numPlaces == 1:
mostVoted = mostVoted[0]
answer = internalDataStore.getAnswerList(answerKey)
answer = answer[0]["value"] if answer.count() > 0 else []
data[profile.uuid] = [datum for datum in answer if datum["key"] != recentPlaceKey]
data[profile.uuid].append({ "key": recentPlaceKey, "bounds": mostVoted})
answer = data[profile.uuid]
internalDataStore.saveAnswer(answerKey, answer)
return data
def recentGfsaScores():
for profile in Profile.objects.all():
ids = getInternalDataStore(profile, "Living Lab", "gfsa", "")
recentGfsaScore(ids)
def get_internal_datastore(self, request):
if request and "datastore_owner__uuid" in request.GET and "bearer_token" in request.GET:
profile, created = Profile.objects.get_or_create(uuid = request.GET["datastore_owner__uuid"])
token = request.GET["bearer_token"]
return getInternalDataStore(profile, "", "", token)
return None
def findRecentPlaceBounds(recentPlaceKey,
timeRanges,
numPlaces=1,
answerKey="RecentPlaces"):
profiles = Profile.objects.all()
data = {}
for profile in profiles:
# TODO: figure out how to get at a token here...
internalDataStore = getInternalDataStore(profile, "Living Lab",
"My Places", "")
#dbName = profile.getDBName()
#collection = connection[dbName]["funf"]
locations = []
# An explanation for why we're doing things the way we are below
# (there are a few obvious strategies for finding places in location data):
# 1) Naive approach - take all location samples in all time ranges, find clusters within them,
# take the one with the most points in it.
# 2) Faster, but more complicated - do 1) for each time range individually to get candidate regions.
# Loop over candidate regions, collapsing and "voting" for those that overlap. Take the one with the most votes.
# Notes: This is essentially 2-levels of clustering with the simplification that overlapping regions would
# have been clustered together anyway (ie; bounding boxes should be similar, but not the same, as strategy 1)
# Pros: Faster - each clustering is limited to 100 entries. In practice, this is more than enough.
# If this poses an issue, time ranges can be chosen more carefully (more / shorter time ranges)
# Cons: Bounding boxes aren't the same as 1). In particular, two candidate boxes may not overlap, but should
# have been clustered together anyway.
# 3) Binning pre-process - Same as 1), but perform a binning pre-process on the location data, collapsing multiple
# samples into single entries, with associaated weights.
# Notes: This is essentially a lower-resolution version of strategy 1. Bounding boxes should be lower-resolution
# versions of those from strategy 1.
# Pros: Bounding boxes should be the same as #1. Takes into account all entries when clustering.
# Cons: Less fine-grained control over the number of entries per cluster than #2. In particular, for sparse
# location data, this may not reduce the number of entries we must cluster.
# The following is an implementation of method #2:
potentialRegions = []
#pdb.set_trace()
for timeRange in timeRanges:
# NOTE: is a limit on the number of entries still necessary, if we're choosing the timeRanges carefully?
values = [
entry["value"] for entry in internalDataStore.getData(
"LocationProbe", timeRange[0], timeRange[1]) or []
]
# Use all locations except the most gratuitously inaccurate ones
values = [
value for value in values if float(value["maccuracy"]) < 100
]
clusters = clusterFunfLocations(values, 100)
if (len(clusters) > 0):
#clusters.sort(key = lambda cluster: -len(cluster))
#topClusters = clusters[:min(len(clusters), numPlaces)]
clusterLocations = max(clusters,
key=lambda cluster: len(cluster))
if isinstance(clusterLocations, list):
lats = [loc[0] for loc in clusterLocations]
longs = [loc[1] for loc in clusterLocations]
if min(lats) != max(lats) and min(longs) != max(longs):
#Only add regions that aren't degenerate (single points)
potentialRegions.append(
[min(lats),
min(longs),
max(lats),
max(longs)])
if len(potentialRegions) > 0:
overlaps = [{
"region":
r1,
"overlapList": [
r2 for r2 in potentialRegions
if r2 is not r1 and boundsOverlap(r1, r2)
]
} for r1 in potentialRegions]
reduced = [{
"region":
reduce(lambda r1, r2: mergeBoxes(r1, r2), r["overlapList"],
r["region"]),
"votes":
len(r["overlapList"])
} for r in overlaps]
reduced.sort(key=lambda r: -r["votes"])
final = []
for r in reduced:
if not listContainsOverlap([f["region"]
for f in final], r["region"]):
final.append(r)
mostOverlap = final[:min(len(final), numPlaces)]
mostVoted = [r["region"] for r in mostOverlap]
if numPlaces == 1:
mostVoted = mostVoted[0]
answer = internalDataStore.getAnswerList(answerKey)
answer = answer[0]["value"] if answer.count() > 0 else []
data[profile.uuid] = [
datum for datum in answer if datum["key"] != recentPlaceKey
]
data[profile.uuid].append({
"key": recentPlaceKey,
"bounds": mostVoted
})
answer = data[profile.uuid]
internalDataStore.saveAnswer(answerKey, answer)
return data
def computeAllFingerprints():
for profile in Profile.objects.all():
print profile.uuid
ids = getInternalDataStore(profile, "Living Lab", "Auth", "")
computeFingerprint(ids)
print "... done computing wifi auth fingerprints"
