def mobile_app_availability_marker(all_streams, stream_name: str, owner_id,
CC: CerebralCortex, config: dict):
"""
This algorithm uses phone battery percentages to decide whether mobile app was available or unavailable.
Theoretically, phone battery data shall be collected 24/7.
:param raw_stream_id:
:param CC:
:param config:
"""
marker_version = "0.0.1"
if config["stream_names"]["phone_battery"] in all_streams:
raw_stream_ids = all_streams[config["stream_names"]
["phone_battery"]]["stream_ids"]
dd_stream_name = config["stream_names"]["app_availability_marker"]
# using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker
app_availability_marker_stream_id = generate_dd_stream_uuid(
dd_stream_name, marker_version, owner_id,
"MOBILE APP AVAILABILITY MARKER")
input_streams = [{
"owner_id": owner_id,
"id": raw_stream_ids,
"name": stream_name
}]
output_stream = {
"id": app_availability_marker_stream_id,
"name": dd_stream_name,
"algo_type": config["algo_type"]["app_availability_marker"]
}
metadata = get_metadata(dd_stream_name, input_streams, config)
if isinstance(raw_stream_ids, list):
for raw_stream_id in raw_stream_ids:
stream_days = CC.get_stream_days(
raw_stream_id, app_availability_marker_stream_id, CC)
for day in stream_days:
try:
stream = CC.get_stream(raw_stream_id,
day=day,
data_type=DataSet.COMPLETE)
if len(stream.data) > 0:
windowed_data = window(
stream.data, config['general']['window_size'],
True)
results = process_windows(windowed_data, config)
merged_windows = merge_consective_windows(results)
if len(merged_windows) > 0:
store(merged_windows, input_streams,
output_stream, metadata, CC, config)
except Exception as e:
CC.logging.log(
"Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: "
% (str(owner_id), str(raw_stream_id),
str(stream_name), str(day), str(e)))
def sensor_availability(all_streams, wrist: str, owner_id: uuid, CC: CerebralCortex, config: dict):
"""
Mark missing data as wireless disconnection if a participate walks away from phone or sensor
:param raw_stream_id:
:param stream_name:
:param owner_id:
:param dd_stream_name:
:param phone_physical_activity:
:param CC:
:param config:
"""
marker_version = "0.0.1"
if config["stream_names"]["phone_physical_activity"] in all_streams:
phone_physical_activity = all_streams[config["stream_names"]["phone_physical_activity"]]["stream_ids"]
else:
phone_physical_activity = None
key0 = "motionsense_hrv_accel_"+wrist
key1 = "motionsense_hrv_"+wrist+"_wireless_marker"
raw_stream_ids = all_streams[config["stream_names"][key0]]["stream_ids"],
stream_name = all_streams[config["stream_names"][key0]]["name"]
dd_stream_name = config["stream_names"][key1]
if config["stream_names"]["phone_physical_activity"] in all_streams:
# using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker
wireless_marker_stream_id = generate_dd_stream_uuid(dd_stream_name, marker_version, owner_id, "WIRELESS DISCONNECTION MARKER")
input_streams = [{"owner_id": owner_id, "id": raw_stream_ids, "name": stream_name}]
output_stream = {"id": wireless_marker_stream_id, "name": dd_stream_name,
"algo_type": config["algo_type"]["sensor_unavailable_marker"]}
metadata = get_metadata(dd_stream_name, input_streams, config)
if isinstance(raw_stream_ids, list):
for raw_stream_id in raw_stream_ids:
stream_days = CC.get_stream_days(raw_stream_id, wireless_marker_stream_id, CC)
for day in stream_days:
try:
# load stream data to be diagnosed
raw_stream = CC.get_stream(raw_stream_id, day=day, data_type=DataSet.COMPLETE)
if len(raw_stream.data) > 0:
windowed_data = window(raw_stream.data, config['general']['window_size'], True)
results = process_windows(windowed_data, day, CC, phone_physical_activity, config)
merged_windows = merge_consective_windows(results)
if len(merged_windows) > 0:
store(merged_windows, input_streams, output_stream, metadata, CC, config)
except Exception as e:
CC.logging.log("Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: "
%(str(owner_id), str(raw_stream_id), str(stream_name), str(day), str(e)))
def attachment_marker(all_streams, wrist, owner_id: uuid, CC: CerebralCortex,
config: dict):
"""
Label sensor data as sensor-on-body, sensor-off-body, or improper-attachment.
All the labeled data (st, et, label) with its metadata are then stored in a datastore
"""
marker_version = "0.0.1"
# TODO: quality streams could be multiple so find the one computed with CC
# using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker
key0 = "motionsense_hrv_led_quality_" + wrist
key1 = "motionsense_hrv_" + wrist + "_attachment_marker"
raw_stream_ids = all_streams[config["stream_names"][key0]]["stream_ids"]
stream_name = all_streams[config["stream_names"][key0]]["name"]
dd_stream_name = config["stream_names"][key1]
if config["stream_names"][key0] in all_streams:
attachment_marker_stream_id = generate_dd_stream_uuid(
dd_stream_name, marker_version, owner_id, "ATTACHMENT MARKER")
input_streams = [{
"owner_id": owner_id,
"id": raw_stream_ids,
"name": stream_name
}]
output_stream = {
"id": attachment_marker_stream_id,
"name": dd_stream_name,
"algo_type": config["algo_type"]["attachment_marker"]
}
metadata = get_metadata(dd_stream_name, input_streams, config)
if isinstance(raw_stream_ids, list):
for raw_stream_id in raw_stream_ids:
stream_days = CC.get_stream_days(raw_stream_id,
attachment_marker_stream_id,
CC)
for day in stream_days:
try:
# load stream data to be diagnosed
raw_stream = CC.get_stream(raw_stream_id,
day=day,
data_type=DataSet.COMPLETE)
if len(raw_stream.data) > 0:
windowed_data = window(
raw_stream.data,
config['general']['window_size'], True)
results = process_windows(windowed_data, config)
merged_windows = merge_consective_windows(results)
store(merged_windows, input_streams, output_stream,
metadata, CC, config)
except Exception as e:
CC.logging.log(
"Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: "
% (str(owner_id), str(raw_stream_id),
str(stream_name), str(day), str(e)))
def battery_marker(all_streams, owner_id, stream_name, CC: CerebralCortex,
config: dict):
"""
This algorithm uses battery percentages to decide whether device was powered-off or battery was low.
All the labeled data (st, et, label) with its metadata are then stored in a datastore.
:param raw_stream_id:
:param CC:
:param config:
"""
marker_version = "0.0.1"
if stream_name in all_streams:
raw_stream_ids = all_streams[config["stream_names"]
["phone_battery"]]["stream_ids"]
dd_stream_name = config["stream_names"]["phone_battery_marker"]
# using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker
battery_marker_stream_id = generate_dd_stream_uuid(
dd_stream_name, marker_version, owner_id, "BATTERY MARKER")
input_streams = [{
"owner_id": owner_id,
"id": raw_stream_ids,
"name": stream_name
}]
output_stream = {
"id": battery_marker_stream_id,
"name": dd_stream_name,
"algo_type": config["algo_type"]["battery_marker"]
}
metadata = get_metadata(dd_stream_name, input_streams, config)
if isinstance(raw_stream_ids, list):
for raw_stream_id in raw_stream_ids:
stream_days = CC.get_stream_days(raw_stream_id,
battery_marker_stream_id, CC)
for day in stream_days:
try:
stream = CC.get_stream(raw_stream_id,
day=day,
data_type=DataSet.COMPLETE)
if len(stream.data) > 0:
windowed_data = window(
stream.data, config['general']['window_size'],
True)
results = process_windows(windowed_data,
stream_name, config)
merged_windows = merge_consective_windows(results)
if len(merged_windows) > 0:
labelled_windows = mark_windows(
battery_marker_stream_id, merged_windows,
CC, config)
store(labelled_windows, input_streams,
output_stream, metadata, CC, config)
except Exception as e:
CC.logging.log(
"Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: "
% (str(owner_id), str(raw_stream_id),
str(stream_name), str(day), str(e)))
def sensor_failure_marker(all_streams, wrist: str, owner_id: uuid,
CC: CerebralCortex, config: dict):
"""
Label a window as packet-loss if received packets are less than the expected packets.
All the labeled data (st, et, label) with its metadata are then stored in a datastore.
:param stream_id:
:param CC_obj:
:param config:
"""
marker_version = "0.0.1"
key0 = "motionsense_hrv_" + wrist + "_attachment_marker"
key1 = "motionsense_hrv_" + wrist + "_attachment_marker"
key2 = "motionsense_hrv_accel_" + wrist
key3 = "motionsense_hrv_gyro_" + wrist
key4 = "motionsense_hrv_" + wrist + "_sensor_failure_marker"
stream_name = all_streams[config["stream_names"][key0]]["name"]
raw_stream_ids = all_streams[config["stream_names"][key1]]["stream_ids"]
mshrv_accel_id = all_streams[config["stream_names"][key2]]["stream_ids"]
mshrv_gyro_id = all_streams[config["stream_names"][key3]]["stream_ids"]
dd_stream_name = config["stream_names"][key4]
if config["stream_names"][key2] in all_streams:
# using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker
sensor_failure_stream_id = generate_dd_stream_uuid(
dd_stream_name, marker_version, owner_id, "SENSOR FAILURE MARKER")
input_streams = [{
"owner_id": owner_id,
"id": raw_stream_ids,
"name": stream_name
}]
output_stream = {
"id": sensor_failure_stream_id,
"name": dd_stream_name,
"algo_type": config["algo_type"]["sensor_failure"]
}
metadata = get_metadata(dd_stream_name, input_streams, config)
if isinstance(raw_stream_ids, list):
for raw_stream_id in raw_stream_ids:
stream_days = CC.get_stream_days(raw_stream_id,
sensor_failure_stream_id, CC)
for day in stream_days:
try:
# load stream data to be diagnosed
attachment_marker_stream = CC.get_stream(
raw_stream_id, day, data_type=DataSet.COMPLETE)
results = OrderedDict()
if attachment_marker_stream.data:
for marker_window in attachment_marker_stream.data:
if "MOTIONSENSE-ON-BODY" in marker_window.sample:
mshrv_accel_stream = CC.get_stream(
mshrv_accel_id,
day,
start_time=marker_window.start_time,
end_time=marker_window.end_time,
data_type=DataSet.ONLY_DATA)
mshrv_gyro_stream = CC.get_stream(
mshrv_gyro_id,
day,
start_time=marker_window.start_time,
end_time=marker_window.end_time,
data_type=DataSet.ONLY_DATA)
results_accel = process_windows(
mshrv_accel_stream, config)
results_gyro = process_windows(
mshrv_gyro_stream, config)
key = marker_window.start_time, marker_window.end_time
# if sensor failure period is more than 12 hours then mark it as a sensor failure
if results_accel > 0 and results_gyro < 1:
sample = "MOTIONSENE-HRV-" + str(
wrist) + "ACCELEROMETER-FAILURE"
results[key].append(
DataPoint(marker_window.start_time,
marker_window.end_time,
sample))
elif results_accel < 1 and results_gyro > 0:
sample = "MOTIONSENE-HRV-" + str(
wrist) + "GYRO-FAILURE"
results[key].append(
DataPoint(marker_window.start_time,
marker_window.end_time,
sample))
if len(results) > 0:
merged_windows = merge_consective_windows(
results)
store(merged_windows, input_streams,
output_stream, metadata, CC, config)
except Exception as e:
CC.logging.log(
"Error processing: owner-id: %s, stream-id: %s, Algo-name: %s, day: %s. Error: "
% (str(owner_id), str(raw_stream_id),
"sensor_failure_marker", str(day), str(e)))
class SqlToCCStream():
def __init__(self, config):
self.CC = CerebralCortex(config)
self.config = self.CC.config
self.sqlData = SqlData(self.config,
dbName="environmental_data_collection")
self.process()
def process(self):
user_ids = self.filter_user_ids()
# get all locations lats/longs
all_locations = self.sqlData.get_latitude_llongitude()
with open("weather_data.json", "r") as wd:
metadata = wd.read()
metadata = json.loads(metadata)
input_stream_name = 'LOCATION--org.md2k.phonesensor--PHONE'
for uid in user_ids:
stream_ids = self.CC.get_stream_id(uid, input_stream_name)
# START TEST CODE
# location_id = self.get_location_id((37.439168,-122.086283), all_locations)
# day = datetime.strptime("20171221", "%Y%m%d").strftime("%Y-%m-%d")
# weather_data = self.sqlData.get_weather_data_by_city_id(location_id, day)
# dps = []
#
# for wd in weather_data:
# dp_sample = []
# wd["temperature"] = json.loads(wd["temperature"])
# wd["wind"] = json.loads(wd["wind"])
#
# dp_sample["sunrise"] = wd["sunrise"]
# dp_sample["sunset"] = wd["sunset"]
# dp_sample["wind_deg"] = wd.get("wind").get("deg","")
# dp_sample["wind_speed"] = wd.get("wind").get("speed","")
# dp_sample["current_temp"] = wd["temperature"]["temp"]
# dp_sample["max_temp"] = wd["temperature"]["temp_max"]
# dp_sample["min_temp"] = wd["temperature"]["temp_min"]
# dp_sample["humidity"] = int(wd["humidity"])
# dp_sample["clouds"] = int(wd["clouds"])
# dp_sample["other"] = wd["other"]
# dp_sample = [wd["sunrise"],wd["sunset"],wd.get("wind").get("deg",""),wd.get("wind").get("speed",""),wd["temperature"]["temp"],wd["temperature"]["temp_max"],wd["temperature"]["temp_min"],int(wd["humidity"]),int(wd["clouds"]),wd["other"]]
# dps.append(DataPoint(wd["start_time"], None, None, dp_sample))
# END TEST CODE
if len(stream_ids) > 0:
print("Processing:", uid)
for sid in stream_ids:
sid = sid["identifier"]
days = self.CC.get_stream_days(sid)
for day in days:
print("User ID, Stream ID, Day", uid, sid, day)
output_stream_id = ""
# get gps data from stream-name 'LOCATION--org.md2k.phonesensor--PHONE'
location_stream = self.CC.get_stream(stream_id=sid,
day=day)
if len(location_stream.data) > 0:
# compute median on lat. and long. vals
user_loc = self.compute_lat_long_median(
location_stream.data)
if user_loc != (0, 0):
offset = location_stream.data[0].offset
# get weather data for match lat/long values
location_id = self.get_location_id(
user_loc, all_locations)
if location_id is not None:
formated_day = datetime.strptime(
day, "%Y%m%d").strftime("%Y-%m-%d")
weather_data = self.sqlData.get_weather_data_by_city_id(
location_id, formated_day)
# convert data into datastream
execution_context = metadata[
"execution_context"]
input_streams_metadata = [{
"id":
sid,
"name":
input_stream_name
}]
metadata["execution_context"]["processing_module"]["input_streams"] \
= input_streams_metadata
dps = []
for wd in weather_data:
dp_sample = []
wd["temperature"] = json.loads(
wd["temperature"])
wd["wind"] = json.loads(wd["wind"])
day_light_duration = (
(wd["sunset"] -
wd["sunrise"]).seconds
) / 3600 # difference in hours
dp_sample = [
wd["sunrise"], wd["sunset"],
day_light_duration,
wd.get("wind", float('nan')).get(
"deg", float('nan')),
wd.get("wind", float('nan')).get(
"speed", float('nan')),
wd["temperature"]["temp"],
wd["temperature"]["temp_max"],
wd["temperature"]["temp_min"],
int(wd["humidity"]),
int(wd["clouds"]), wd["other"]
]
dps.append(
DataPoint(wd["start_time"], None,
offset, dp_sample))
if len(dps) > 0:
# generate UUID for stream
output_stream_id = str(
metadata["data_descriptor"]) + str(
execution_context) + str(
metadata["annotations"])
output_stream_id += "weather-data-stream"
output_stream_id += "weather-data-stream"
output_stream_id += str(uid)
output_stream_id += str(sid)
# output_stream_id += str(day)
output_stream_id = str(
uuid.uuid3(uuid.NAMESPACE_DNS,
output_stream_id))
ds = DataStream(
identifier=output_stream_id,
owner=uid,
name=metadata["name"],
data_descriptor=metadata[
"data_descriptor"],
execution_context=execution_context,
annotations=metadata[
"annotations"],
stream_type=metadata["type"],
data=dps)
# store data stream
self.CC.save_stream(ds)
def compute_lat_long_median(self, data):
latitude = []
longitude = []
valid_data = False
for dp in data:
if isinstance(dp.sample, list) and len(dp.sample) == 6:
latitude.append(dp.sample[0])
longitude.append(dp.sample[1])
valid_data = True
if valid_data:
return statistics.median(latitude), statistics.median(longitude)
else:
return 0, 0
def get_location_id(self, user_loc, all_locations):
# find distance between user location and weather lat/long
closest = None
location_id = None
for loc in all_locations:
distance = haversine(
user_loc, (float(loc["latitude"]), float(loc["longitude"])),
miles=True)
if closest is None:
closest = distance
location_id = loc["id"]
elif distance < closest:
closest = distance
location_id = loc["id"]
if closest <= 30: #if distance is below then 30 miles then select it as weather location
return location_id
else:
return None
def filter_user_ids(self):
active_users = []
all_users = []
for uid in self.CC.get_all_users("mperf"):
all_users.append(uid["identifier"])
data_dir = self.config["data_replay"]["data_dir"]
for owner_dir in os.scandir(data_dir):
if owner_dir.name in all_users:
active_users.append(owner_dir.name)
return active_users