def mobile_app_availability_marker(all_streams, stream_name: str, owner_id, CC: CerebralCortex, config: dict): """ This algorithm uses phone battery percentages to decide whether mobile app was available or unavailable. Theoretically, phone battery data shall be collected 24/7. :param raw_stream_id: :param CC: :param config: """ marker_version = "0.0.1" if config["stream_names"]["phone_battery"] in all_streams: raw_stream_ids = all_streams[config["stream_names"] ["phone_battery"]]["stream_ids"] dd_stream_name = config["stream_names"]["app_availability_marker"] # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker app_availability_marker_stream_id = generate_dd_stream_uuid( dd_stream_name, marker_version, owner_id, "MOBILE APP AVAILABILITY MARKER") input_streams = [{ "owner_id": owner_id, "id": raw_stream_ids, "name": stream_name }] output_stream = { "id": app_availability_marker_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["app_availability_marker"] } metadata = get_metadata(dd_stream_name, input_streams, config) if isinstance(raw_stream_ids, list): for raw_stream_id in raw_stream_ids: stream_days = CC.get_stream_days( raw_stream_id, app_availability_marker_stream_id, CC) for day in stream_days: try: stream = CC.get_stream(raw_stream_id, day=day, data_type=DataSet.COMPLETE) if len(stream.data) > 0: windowed_data = window( stream.data, config['general']['window_size'], True) results = process_windows(windowed_data, config) merged_windows = merge_consective_windows(results) if len(merged_windows) > 0: store(merged_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: CC.logging.log( "Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: " % (str(owner_id), str(raw_stream_id), str(stream_name), str(day), str(e)))
def sensor_availability(all_streams, wrist: str, owner_id: uuid, CC: CerebralCortex, config: dict): """ Mark missing data as wireless disconnection if a participate walks away from phone or sensor :param raw_stream_id: :param stream_name: :param owner_id: :param dd_stream_name: :param phone_physical_activity: :param CC: :param config: """ marker_version = "0.0.1" if config["stream_names"]["phone_physical_activity"] in all_streams: phone_physical_activity = all_streams[config["stream_names"]["phone_physical_activity"]]["stream_ids"] else: phone_physical_activity = None key0 = "motionsense_hrv_accel_"+wrist key1 = "motionsense_hrv_"+wrist+"_wireless_marker" raw_stream_ids = all_streams[config["stream_names"][key0]]["stream_ids"], stream_name = all_streams[config["stream_names"][key0]]["name"] dd_stream_name = config["stream_names"][key1] if config["stream_names"]["phone_physical_activity"] in all_streams: # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker wireless_marker_stream_id = generate_dd_stream_uuid(dd_stream_name, marker_version, owner_id, "WIRELESS DISCONNECTION MARKER") input_streams = [{"owner_id": owner_id, "id": raw_stream_ids, "name": stream_name}] output_stream = {"id": wireless_marker_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["sensor_unavailable_marker"]} metadata = get_metadata(dd_stream_name, input_streams, config) if isinstance(raw_stream_ids, list): for raw_stream_id in raw_stream_ids: stream_days = CC.get_stream_days(raw_stream_id, wireless_marker_stream_id, CC) for day in stream_days: try: # load stream data to be diagnosed raw_stream = CC.get_stream(raw_stream_id, day=day, data_type=DataSet.COMPLETE) if len(raw_stream.data) > 0: windowed_data = window(raw_stream.data, config['general']['window_size'], True) results = process_windows(windowed_data, day, CC, phone_physical_activity, config) merged_windows = merge_consective_windows(results) if len(merged_windows) > 0: store(merged_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: CC.logging.log("Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: " %(str(owner_id), str(raw_stream_id), str(stream_name), str(day), str(e)))
def attachment_marker(all_streams, wrist, owner_id: uuid, CC: CerebralCortex, config: dict): """ Label sensor data as sensor-on-body, sensor-off-body, or improper-attachment. All the labeled data (st, et, label) with its metadata are then stored in a datastore """ marker_version = "0.0.1" # TODO: quality streams could be multiple so find the one computed with CC # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker key0 = "motionsense_hrv_led_quality_" + wrist key1 = "motionsense_hrv_" + wrist + "_attachment_marker" raw_stream_ids = all_streams[config["stream_names"][key0]]["stream_ids"] stream_name = all_streams[config["stream_names"][key0]]["name"] dd_stream_name = config["stream_names"][key1] if config["stream_names"][key0] in all_streams: attachment_marker_stream_id = generate_dd_stream_uuid( dd_stream_name, marker_version, owner_id, "ATTACHMENT MARKER") input_streams = [{ "owner_id": owner_id, "id": raw_stream_ids, "name": stream_name }] output_stream = { "id": attachment_marker_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["attachment_marker"] } metadata = get_metadata(dd_stream_name, input_streams, config) if isinstance(raw_stream_ids, list): for raw_stream_id in raw_stream_ids: stream_days = CC.get_stream_days(raw_stream_id, attachment_marker_stream_id, CC) for day in stream_days: try: # load stream data to be diagnosed raw_stream = CC.get_stream(raw_stream_id, day=day, data_type=DataSet.COMPLETE) if len(raw_stream.data) > 0: windowed_data = window( raw_stream.data, config['general']['window_size'], True) results = process_windows(windowed_data, config) merged_windows = merge_consective_windows(results) store(merged_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: CC.logging.log( "Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: " % (str(owner_id), str(raw_stream_id), str(stream_name), str(day), str(e)))
def battery_marker(all_streams, owner_id, stream_name, CC: CerebralCortex, config: dict): """ This algorithm uses battery percentages to decide whether device was powered-off or battery was low. All the labeled data (st, et, label) with its metadata are then stored in a datastore. :param raw_stream_id: :param CC: :param config: """ marker_version = "0.0.1" if stream_name in all_streams: raw_stream_ids = all_streams[config["stream_names"] ["phone_battery"]]["stream_ids"] dd_stream_name = config["stream_names"]["phone_battery_marker"] # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker battery_marker_stream_id = generate_dd_stream_uuid( dd_stream_name, marker_version, owner_id, "BATTERY MARKER") input_streams = [{ "owner_id": owner_id, "id": raw_stream_ids, "name": stream_name }] output_stream = { "id": battery_marker_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["battery_marker"] } metadata = get_metadata(dd_stream_name, input_streams, config) if isinstance(raw_stream_ids, list): for raw_stream_id in raw_stream_ids: stream_days = CC.get_stream_days(raw_stream_id, battery_marker_stream_id, CC) for day in stream_days: try: stream = CC.get_stream(raw_stream_id, day=day, data_type=DataSet.COMPLETE) if len(stream.data) > 0: windowed_data = window( stream.data, config['general']['window_size'], True) results = process_windows(windowed_data, stream_name, config) merged_windows = merge_consective_windows(results) if len(merged_windows) > 0: labelled_windows = mark_windows( battery_marker_stream_id, merged_windows, CC, config) store(labelled_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: CC.logging.log( "Error processing: owner-id: %s, stream-id: %s, stream-name: %s, day: %s. Error: " % (str(owner_id), str(raw_stream_id), str(stream_name), str(day), str(e)))
def sensor_failure_marker(all_streams, wrist: str, owner_id: uuid, CC: CerebralCortex, config: dict): """ Label a window as packet-loss if received packets are less than the expected packets. All the labeled data (st, et, label) with its metadata are then stored in a datastore. :param stream_id: :param CC_obj: :param config: """ marker_version = "0.0.1" key0 = "motionsense_hrv_" + wrist + "_attachment_marker" key1 = "motionsense_hrv_" + wrist + "_attachment_marker" key2 = "motionsense_hrv_accel_" + wrist key3 = "motionsense_hrv_gyro_" + wrist key4 = "motionsense_hrv_" + wrist + "_sensor_failure_marker" stream_name = all_streams[config["stream_names"][key0]]["name"] raw_stream_ids = all_streams[config["stream_names"][key1]]["stream_ids"] mshrv_accel_id = all_streams[config["stream_names"][key2]]["stream_ids"] mshrv_gyro_id = all_streams[config["stream_names"][key3]]["stream_ids"] dd_stream_name = config["stream_names"][key4] if config["stream_names"][key2] in all_streams: # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker sensor_failure_stream_id = generate_dd_stream_uuid( dd_stream_name, marker_version, owner_id, "SENSOR FAILURE MARKER") input_streams = [{ "owner_id": owner_id, "id": raw_stream_ids, "name": stream_name }] output_stream = { "id": sensor_failure_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["sensor_failure"] } metadata = get_metadata(dd_stream_name, input_streams, config) if isinstance(raw_stream_ids, list): for raw_stream_id in raw_stream_ids: stream_days = CC.get_stream_days(raw_stream_id, sensor_failure_stream_id, CC) for day in stream_days: try: # load stream data to be diagnosed attachment_marker_stream = CC.get_stream( raw_stream_id, day, data_type=DataSet.COMPLETE) results = OrderedDict() if attachment_marker_stream.data: for marker_window in attachment_marker_stream.data: if "MOTIONSENSE-ON-BODY" in marker_window.sample: mshrv_accel_stream = CC.get_stream( mshrv_accel_id, day, start_time=marker_window.start_time, end_time=marker_window.end_time, data_type=DataSet.ONLY_DATA) mshrv_gyro_stream = CC.get_stream( mshrv_gyro_id, day, start_time=marker_window.start_time, end_time=marker_window.end_time, data_type=DataSet.ONLY_DATA) results_accel = process_windows( mshrv_accel_stream, config) results_gyro = process_windows( mshrv_gyro_stream, config) key = marker_window.start_time, marker_window.end_time # if sensor failure period is more than 12 hours then mark it as a sensor failure if results_accel > 0 and results_gyro < 1: sample = "MOTIONSENE-HRV-" + str( wrist) + "ACCELEROMETER-FAILURE" results[key].append( DataPoint(marker_window.start_time, marker_window.end_time, sample)) elif results_accel < 1 and results_gyro > 0: sample = "MOTIONSENE-HRV-" + str( wrist) + "GYRO-FAILURE" results[key].append( DataPoint(marker_window.start_time, marker_window.end_time, sample)) if len(results) > 0: merged_windows = merge_consective_windows( results) store(merged_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: CC.logging.log( "Error processing: owner-id: %s, stream-id: %s, Algo-name: %s, day: %s. Error: " % (str(owner_id), str(raw_stream_id), "sensor_failure_marker", str(day), str(e)))
class SqlToCCStream(): def __init__(self, config): self.CC = CerebralCortex(config) self.config = self.CC.config self.sqlData = SqlData(self.config, dbName="environmental_data_collection") self.process() def process(self): user_ids = self.filter_user_ids() # get all locations lats/longs all_locations = self.sqlData.get_latitude_llongitude() with open("weather_data.json", "r") as wd: metadata = wd.read() metadata = json.loads(metadata) input_stream_name = 'LOCATION--org.md2k.phonesensor--PHONE' for uid in user_ids: stream_ids = self.CC.get_stream_id(uid, input_stream_name) # START TEST CODE # location_id = self.get_location_id((37.439168,-122.086283), all_locations) # day = datetime.strptime("20171221", "%Y%m%d").strftime("%Y-%m-%d") # weather_data = self.sqlData.get_weather_data_by_city_id(location_id, day) # dps = [] # # for wd in weather_data: # dp_sample = [] # wd["temperature"] = json.loads(wd["temperature"]) # wd["wind"] = json.loads(wd["wind"]) # # dp_sample["sunrise"] = wd["sunrise"] # dp_sample["sunset"] = wd["sunset"] # dp_sample["wind_deg"] = wd.get("wind").get("deg","") # dp_sample["wind_speed"] = wd.get("wind").get("speed","") # dp_sample["current_temp"] = wd["temperature"]["temp"] # dp_sample["max_temp"] = wd["temperature"]["temp_max"] # dp_sample["min_temp"] = wd["temperature"]["temp_min"] # dp_sample["humidity"] = int(wd["humidity"]) # dp_sample["clouds"] = int(wd["clouds"]) # dp_sample["other"] = wd["other"] # dp_sample = [wd["sunrise"],wd["sunset"],wd.get("wind").get("deg",""),wd.get("wind").get("speed",""),wd["temperature"]["temp"],wd["temperature"]["temp_max"],wd["temperature"]["temp_min"],int(wd["humidity"]),int(wd["clouds"]),wd["other"]] # dps.append(DataPoint(wd["start_time"], None, None, dp_sample)) # END TEST CODE if len(stream_ids) > 0: print("Processing:", uid) for sid in stream_ids: sid = sid["identifier"] days = self.CC.get_stream_days(sid) for day in days: print("User ID, Stream ID, Day", uid, sid, day) output_stream_id = "" # get gps data from stream-name 'LOCATION--org.md2k.phonesensor--PHONE' location_stream = self.CC.get_stream(stream_id=sid, day=day) if len(location_stream.data) > 0: # compute median on lat. and long. vals user_loc = self.compute_lat_long_median( location_stream.data) if user_loc != (0, 0): offset = location_stream.data[0].offset # get weather data for match lat/long values location_id = self.get_location_id( user_loc, all_locations) if location_id is not None: formated_day = datetime.strptime( day, "%Y%m%d").strftime("%Y-%m-%d") weather_data = self.sqlData.get_weather_data_by_city_id( location_id, formated_day) # convert data into datastream execution_context = metadata[ "execution_context"] input_streams_metadata = [{ "id": sid, "name": input_stream_name }] metadata["execution_context"]["processing_module"]["input_streams"] \ = input_streams_metadata dps = [] for wd in weather_data: dp_sample = [] wd["temperature"] = json.loads( wd["temperature"]) wd["wind"] = json.loads(wd["wind"]) day_light_duration = ( (wd["sunset"] - wd["sunrise"]).seconds ) / 3600 # difference in hours dp_sample = [ wd["sunrise"], wd["sunset"], day_light_duration, wd.get("wind", float('nan')).get( "deg", float('nan')), wd.get("wind", float('nan')).get( "speed", float('nan')), wd["temperature"]["temp"], wd["temperature"]["temp_max"], wd["temperature"]["temp_min"], int(wd["humidity"]), int(wd["clouds"]), wd["other"] ] dps.append( DataPoint(wd["start_time"], None, offset, dp_sample)) if len(dps) > 0: # generate UUID for stream output_stream_id = str( metadata["data_descriptor"]) + str( execution_context) + str( metadata["annotations"]) output_stream_id += "weather-data-stream" output_stream_id += "weather-data-stream" output_stream_id += str(uid) output_stream_id += str(sid) # output_stream_id += str(day) output_stream_id = str( uuid.uuid3(uuid.NAMESPACE_DNS, output_stream_id)) ds = DataStream( identifier=output_stream_id, owner=uid, name=metadata["name"], data_descriptor=metadata[ "data_descriptor"], execution_context=execution_context, annotations=metadata[ "annotations"], stream_type=metadata["type"], data=dps) # store data stream self.CC.save_stream(ds) def compute_lat_long_median(self, data): latitude = [] longitude = [] valid_data = False for dp in data: if isinstance(dp.sample, list) and len(dp.sample) == 6: latitude.append(dp.sample[0]) longitude.append(dp.sample[1]) valid_data = True if valid_data: return statistics.median(latitude), statistics.median(longitude) else: return 0, 0 def get_location_id(self, user_loc, all_locations): # find distance between user location and weather lat/long closest = None location_id = None for loc in all_locations: distance = haversine( user_loc, (float(loc["latitude"]), float(loc["longitude"])), miles=True) if closest is None: closest = distance location_id = loc["id"] elif distance < closest: closest = distance location_id = loc["id"] if closest <= 30: #if distance is below then 30 miles then select it as weather location return location_id else: return None def filter_user_ids(self): active_users = [] all_users = [] for uid in self.CC.get_all_users("mperf"): all_users.append(uid["identifier"]) data_dir = self.config["data_replay"]["data_dir"] for owner_dir in os.scandir(data_dir): if owner_dir.name in all_users: active_users.append(owner_dir.name) return active_users