def gen_battery_data(CC, study_name, user_id, stream_name, version=1, hours=1): """ Create pyspark dataframe with some sample phone battery data Returns: DataFrame: pyspark dataframe object with columns: ["timestamp", "battery_level", "version", "user"] """ column_name = [ "timestamp", "localtime", "user", "version", "level", "voltage", "temperature" ] sample_data = [] timestamp = datetime(2019, 1, 9, 11, 34, 59) sample = 100 voltage = 3700 temperature = 70 sqlContext = get_or_create_sc("sqlContext") total_data = hours * 60 * 60 for row in range(total_data, 1, -1): sample = float(sample - 0.01) timestamp = timestamp + timedelta(0, 1) localtime = timestamp - timedelta(hours=5) sample_data.append((timestamp, localtime, user_id, version, sample, voltage, temperature)) df = sqlContext.createDataFrame(sample_data, column_name) stream_metadata = Metadata() stream_metadata.set_study_name(study_name).set_name(stream_name).set_description("battery sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \ .add_dataDescriptor( DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \ .add_dataDescriptor( DataDescriptor().set_name("level").set_type("float").set_attribute("description", "current battery charge")) \ .add_dataDescriptor( DataDescriptor().set_name("voltage").set_type("float").set_attribute("description", "current battery voltage level")) \ .add_dataDescriptor( DataDescriptor().set_name("temperature").set_type("float").set_attribute("description", "current battery temperature")) \ .add_module( ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author( "Nasir Ali", "*****@*****.**")) stream_metadata.is_valid() ds = DataStream(df, stream_metadata) CC.save_stream(ds)
def gen_phone_battery_metadata(stream_name) -> Metadata: """ Create Metadata object with some sample metadata of phone battery data Returns: Metadata: metadata of phone battery stream """ stream_metadata = Metadata() stream_metadata.set_study_name("default").set_name(stream_name).set_description("mobile phone battery sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("level").set_attribute("description", "current battery charge")) \ .add_module( ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author( "Nasir Ali", "*****@*****.**")) stream_metadata.is_valid() return stream_metadata
def gen_accel_gyro_data(CC, study_name, user_id, stream_name, version=1, hours=1, frequency=32): """ Create pyspark dataframe with some sample phone battery data Returns: DataFrame: pyspark dataframe object with columns: ["timestamp", "battery_level", "version", "user"] """ column_name = ["timestamp", "localtime", "user" ,"version", "x", "y", "z"] sample_data = [] timestamp = datetime(2019, 1, 9, 11, 34, 59) sqlContext = get_or_create_sc("sqlContext") total_hours = (hours*60*60)*frequency for row in range(total_hours): x = round(random.uniform(-2,2),8) y = round(random.uniform(-2,2),8) z = round(random.uniform(-2,2),8) timestamp = timestamp + timedelta(milliseconds=1) localtime = timestamp - timedelta(hours=5) sample_data.append((timestamp, localtime, user_id, version, x, y, z)) df = sqlContext.createDataFrame(sample_data, column_name) stream_metadata = Metadata() stream_metadata.set_study_name(study_name).set_name(stream_name).set_description("wrist watch sensor sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \ .add_dataDescriptor( DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \ .add_dataDescriptor( DataDescriptor().set_name("x").set_type("float").set_attribute("description", "x-axis")) \ .add_dataDescriptor( DataDescriptor().set_name("y").set_type("float").set_attribute("description", "y-axis")) \ .add_dataDescriptor( DataDescriptor().set_name("z").set_type("float").set_attribute("description", "z-axis")) \ .add_module( ModuleMetadata().set_name("phone.sensors").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author( "Nasir Ali", "*****@*****.**")) stream_metadata.is_valid() ds = DataStream(df, stream_metadata) CC.save_stream(ds)
def gen_phone_battery_metadata() -> Metadata: """ Create Metadata object with some sample metadata of phone battery data Returns: Metadata: metadata of phone battery stream """ stream_metadata = Metadata() stream_metadata.set_study_name("default").set_description("this is a test-stream.").set_name("BATTERY--org.md2k.phonesensor--PHONE") \ .add_dataDescriptor( DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp")) \ .add_dataDescriptor( DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp")) \ .add_dataDescriptor( DataDescriptor().set_name("battery_level").set_type("float").set_attribute("description", "current battery charge")) \ .add_dataDescriptor( DataDescriptor().set_name("version").set_type("int").set_attribute("description", "stream version")) \ .add_dataDescriptor( DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \ .add_module( ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author( "test_user", "test_user@test_email.com")) stream_metadata.is_valid() return stream_metadata
def gen_location_datastream(user_id, stream_name) -> object: """ Create pyspark dataframe with some sample gps data (Memphis, TN, lat, long, alt coordinates) Args: user_id (str): id of a user stream_name (str): sample gps stream name Returns: DataStream: datastream object of gps location stream with its metadata """ column_name = [ "timestamp", "localtime", "user", "version", "latitude", "longitude", "altitude", "speed", "bearing", "accuracy" ] sample_data = [] timestamp = datetime(2019, 9, 1, 11, 34, 59) sqlContext = get_or_create_sc("sqlContext") lower_left = [35.079678, -90.074136] upper_right = [35.194771, -89.868766] alt = [i for i in range(83, 100)] for location in range(5): lat = random.uniform(lower_left[0], upper_right[0]) long = random.uniform(lower_left[1], upper_right[1]) for dp in range(150): lat_val = random.gauss(lat, 0.001) long_val = random.gauss(long, 0.001) alt_val = random.choice(alt) speed_val = round(random.uniform(0.0, 5.0), 6) bearing_val = round(random.uniform(0.0, 350), 6) accuracy_val = round(random.uniform(10.0, 30.4), 6) timestamp = timestamp + timedelta(minutes=1) localtime = timestamp + timedelta(hours=5) sample_data.append( (timestamp, localtime, user_id, 1, lat_val, long_val, alt_val, speed_val, bearing_val, accuracy_val)) df = sqlContext.createDataFrame(sample_data, column_name) stream_metadata = Metadata() stream_metadata.set_study_name("default").set_name(stream_name).set_description("GPS sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \ .add_dataDescriptor( DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \ .add_dataDescriptor( DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \ .add_dataDescriptor( DataDescriptor().set_name("latitude").set_type("float").set_attribute("description", "gps latitude")) \ .add_dataDescriptor( DataDescriptor().set_name("longitude").set_type("float").set_attribute("description", "gps longitude")) \ .add_dataDescriptor( DataDescriptor().set_name("altitude").set_type("float").set_attribute("description", "gps altitude")) \ .add_dataDescriptor( DataDescriptor().set_name("speed").set_type("float").set_attribute("description", "speed info")) \ .add_dataDescriptor( DataDescriptor().set_name("bearing").set_type("float").set_attribute("description", "bearing info")) \ .add_dataDescriptor( DataDescriptor().set_name("accuracy").set_type("float").set_attribute("description", "accuracy of gps location")) \ .add_module( ModuleMetadata().set_name("examples.util.data_helper.gen_location_data").set_attribute("attribute_key", "attribute_value").set_author( "Nasir Ali", "*****@*****.**")) stream_metadata.is_valid() ds = DataStream(data=df, metadata=stream_metadata) return ds
def gen_stress_data(stream_name, spark_df=False): data = [ [0.7, "road", "Driving", "Was Tailgated", "IN_VEHICLE"], [0.3, "work", "Job", "Bored / Not enough to do", "STILL"], [0.5, "home", "Health", "Physical inability", "STILL"], [0.6, "road", "Driving", "Saw a police car", "IN_VEHICLE"], [0.38, "work", "Job", "Technology barriers", "STILL"], [0.2, "home", "Finance", "Missed payment", "UNKNOWN"], [0.9, "work", "Finance", "Unexpected losses", "WALKING"], [0.54, "road", "Driving", "Difficulty in navigating", "IN_VEHICLE"], [0.79, "work", "Job", "Unpleasant conversation", "ON_FOOT"], [0.28, "road", "Health", "My eating habits", "IN_VEHICLE"], [ 0.47, "road", "Driving", "Indecision at a traffic intersection", "IN_VEHICLE" ], [0.67, "work", "Job", "Late arrival", "WALKING"], ] column_name = [ 'user', 'timestamp', 'localtime', 'version', 'start_time', 'end_time', 'density', 'location', 'stresser_main', 'stresser_sub', 'activity' ] sample_data = [] timestamp = datetime(2019, 1, 9, 11, 34, 59) for row in range(20, 1, -1): if row > 10: user_id = "00000000-afb8-476e-9872-6472b4e66b68" else: user_id = "b1117354-ce48-4325-b2e3-78b0cc932819" timestamp = timestamp + timedelta( hours=random.choice([1, 3, 7, 2, 4, 5])) localtime = timestamp - timedelta(hours=5) start_time = timestamp end_time = timestamp + timedelta( minutes=random.choice([12, 6, 8, 16, 29, 45, 2, 3, 8])) data_vals = random.choice(data) sample_data.append([ user_id, timestamp, localtime, 1, start_time, end_time, data_vals[0], data_vals[1], data_vals[2], data_vals[3], data_vals[4] ]) stream_metadata = Metadata() stream_metadata.set_study_name("default").set_name(stream_name).set_description("GPS sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("start_time").set_type("datetime").set_attribute("description", "start time of a stress episode.")) \ .add_dataDescriptor( DataDescriptor().set_name("end_time").set_type("datetime").set_attribute("description", "end time of a stress episode.")) \ .add_dataDescriptor( DataDescriptor().set_name("density").set_type("float").set_attribute("description", "density of stress")) \ .add_dataDescriptor( DataDescriptor().set_name("location").set_type("string").set_attribute("description", "location where stress episode was captured.")) \ .add_dataDescriptor( DataDescriptor().set_name("stresser_main").set_type("string").set_attribute("description", "stressers' main category.")) \ .add_dataDescriptor( DataDescriptor().set_name("stresser_sub").set_type("string").set_attribute("description", "stressers' sub category.")) \ .add_dataDescriptor( DataDescriptor().set_name("activity").set_type("string").set_attribute("description", "physical activity name")) \ .add_module( ModuleMetadata().set_name("examples.util.data_helper.gen_stress_data").set_attribute("attribute_key", "attribute_value").set_author( "Nasir Ali", "*****@*****.**")) stream_metadata.is_valid() if spark_df: sqlContext = get_or_create_sc("sqlContext") df = sqlContext.createDataFrame(sample_data, column_name) else: df = pd.DataFrame(sample_data, columns=column_name) ds = DataStream(df, stream_metadata) return ds
def gen_location_datastream(user_id, stream_name) -> object: """ Create pyspark dataframe with some sample gps data (Memphis, TN, lat, long, alt coordinates) Args: user_id (str): id of a user stream_name (str): sample gps stream name Returns: DataStream: datastream object of gps location stream with its metadata """ column_name = [ "timestamp", "localtime", "user", "version", "latitude", "longitude", "altitude", "speed", "bearing", "accuracy" ] sample_data = [] timestamp = datetime(2019, 1, 9, 11, 34, 59) sqlContext = get_or_create_sc("sqlContext") lat = [ 35.1247391, 35.1257391, 35.1217391, 35.1117391, 35.1317391, 35.1287391, 35.5217391 ] long = [ -89.9750021, -89.9710021, -89.9800021, -89.9670021, -89.9790021, -89.9710021, -89.8700021 ] alt = [83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0] for dp in range(500): lat_val = random.choice(lat) long_val = random.choice(long) alt_val = random.choice(alt) #ts_val = 15094)+(16272882+(dp*1000000)) speed_val = round(random.uniform(0.0, 5.0), 6) bearing_val = round(random.uniform(0.0, 350), 6) accuracy_val = round(random.uniform(10.0, 30.4), 6) #all_dps = ",".join([ts_val, lat_val, long_val, alt_val, speed_val, bearing_val, accuracy_val]) timestamp = timestamp + timedelta(minutes=1) localtime = timestamp + timedelta(hours=5) sample_data.append( (timestamp, localtime, user_id, 1, lat_val, long_val, alt_val, speed_val, bearing_val, accuracy_val)) df = sqlContext.createDataFrame(sample_data, column_name) stream_metadata = Metadata() stream_metadata.set_name(stream_name).set_version(1).set_description("GPS sample data stream.") \ .add_dataDescriptor( DataDescriptor().set_name("latitude").set_type("float").set_attribute("description", "gps latitude")) \ .add_dataDescriptor( DataDescriptor().set_name("longitude").set_type("float").set_attribute("description", "gps longitude")) \ .add_dataDescriptor( DataDescriptor().set_name("altitude").set_type("float").set_attribute("description", "gps altitude")) \ .add_dataDescriptor( DataDescriptor().set_name("speed").set_type("float").set_attribute("description", "speed info")) \ .add_dataDescriptor( DataDescriptor().set_name("bearing").set_type("float").set_attribute("description", "bearing info")) \ .add_dataDescriptor( DataDescriptor().set_name("accuracy").set_type("float").set_attribute("description", "accuracy of gps location")) \ .add_module( ModuleMetadata().set_name("examples.util.data_helper.gen_location_data").set_version("0.0.1").set_attribute("attribute_key", "attribute_value").set_author( "test_user", "test_user@test_email.com")) stream_metadata.is_valid() return DataStream(data=df, metadata=stream_metadata)