def gen_battery_data(CC, study_name, user_id, stream_name, version=1, hours=1):
"""
Create pyspark dataframe with some sample phone battery data
Returns:
DataFrame: pyspark dataframe object with columns: ["timestamp", "battery_level", "version", "user"]
"""
column_name = [
"timestamp", "localtime", "user", "version", "level", "voltage",
"temperature"
]
sample_data = []
timestamp = datetime(2019, 1, 9, 11, 34, 59)
sample = 100
voltage = 3700
temperature = 70
sqlContext = get_or_create_sc("sqlContext")
total_data = hours * 60 * 60
for row in range(total_data, 1, -1):
sample = float(sample - 0.01)
timestamp = timestamp + timedelta(0, 1)
localtime = timestamp - timedelta(hours=5)
sample_data.append((timestamp, localtime, user_id, version, sample,
voltage, temperature))
df = sqlContext.createDataFrame(sample_data, column_name)
stream_metadata = Metadata()
stream_metadata.set_study_name(study_name).set_name(stream_name).set_description("battery sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \
.add_dataDescriptor(
DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \
.add_dataDescriptor(
DataDescriptor().set_name("level").set_type("float").set_attribute("description", "current battery charge")) \
.add_dataDescriptor(
DataDescriptor().set_name("voltage").set_type("float").set_attribute("description", "current battery voltage level")) \
.add_dataDescriptor(
DataDescriptor().set_name("temperature").set_type("float").set_attribute("description", "current battery temperature")) \
.add_module(
ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author(
"Nasir Ali", "*****@*****.**"))
stream_metadata.is_valid()
ds = DataStream(df, stream_metadata)
CC.save_stream(ds)
def gen_phone_battery_metadata(stream_name) -> Metadata:
"""
Create Metadata object with some sample metadata of phone battery data
Returns:
Metadata: metadata of phone battery stream
"""
stream_metadata = Metadata()
stream_metadata.set_study_name("default").set_name(stream_name).set_description("mobile phone battery sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("level").set_attribute("description", "current battery charge")) \
.add_module(
ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author(
"Nasir Ali", "*****@*****.**"))
stream_metadata.is_valid()
return stream_metadata
def gen_accel_gyro_data(CC, study_name, user_id, stream_name, version=1, hours=1, frequency=32):
"""
Create pyspark dataframe with some sample phone battery data
Returns:
DataFrame: pyspark dataframe object with columns: ["timestamp", "battery_level", "version", "user"]
"""
column_name = ["timestamp", "localtime", "user" ,"version", "x", "y", "z"]
sample_data = []
timestamp = datetime(2019, 1, 9, 11, 34, 59)
sqlContext = get_or_create_sc("sqlContext")
total_hours = (hours*60*60)*frequency
for row in range(total_hours):
x = round(random.uniform(-2,2),8)
y = round(random.uniform(-2,2),8)
z = round(random.uniform(-2,2),8)
timestamp = timestamp + timedelta(milliseconds=1)
localtime = timestamp - timedelta(hours=5)
sample_data.append((timestamp, localtime, user_id, version, x, y, z))
df = sqlContext.createDataFrame(sample_data, column_name)
stream_metadata = Metadata()
stream_metadata.set_study_name(study_name).set_name(stream_name).set_description("wrist watch sensor sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \
.add_dataDescriptor(
DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \
.add_dataDescriptor(
DataDescriptor().set_name("x").set_type("float").set_attribute("description", "x-axis")) \
.add_dataDescriptor(
DataDescriptor().set_name("y").set_type("float").set_attribute("description", "y-axis")) \
.add_dataDescriptor(
DataDescriptor().set_name("z").set_type("float").set_attribute("description", "z-axis")) \
.add_module(
ModuleMetadata().set_name("phone.sensors").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author(
"Nasir Ali", "*****@*****.**"))
stream_metadata.is_valid()
ds = DataStream(df, stream_metadata)
CC.save_stream(ds)
def gen_phone_battery_metadata() -> Metadata:
"""
Create Metadata object with some sample metadata of phone battery data
Returns:
Metadata: metadata of phone battery stream
"""
stream_metadata = Metadata()
stream_metadata.set_study_name("default").set_description("this is a test-stream.").set_name("BATTERY--org.md2k.phonesensor--PHONE") \
.add_dataDescriptor(
DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp")) \
.add_dataDescriptor(
DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp")) \
.add_dataDescriptor(
DataDescriptor().set_name("battery_level").set_type("float").set_attribute("description", "current battery charge")) \
.add_dataDescriptor(
DataDescriptor().set_name("version").set_type("int").set_attribute("description", "stream version")) \
.add_dataDescriptor(
DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \
.add_module(
ModuleMetadata().set_name("battery").set_version("1.2.4").set_attribute("attribute_key", "attribute_value").set_author(
"test_user", "test_user@test_email.com"))
stream_metadata.is_valid()
return stream_metadata
def gen_location_datastream(user_id, stream_name) -> object:
"""
Create pyspark dataframe with some sample gps data (Memphis, TN, lat, long, alt coordinates)
Args:
user_id (str): id of a user
stream_name (str): sample gps stream name
Returns:
DataStream: datastream object of gps location stream with its metadata
"""
column_name = [
"timestamp", "localtime", "user", "version", "latitude", "longitude",
"altitude", "speed", "bearing", "accuracy"
]
sample_data = []
timestamp = datetime(2019, 9, 1, 11, 34, 59)
sqlContext = get_or_create_sc("sqlContext")
lower_left = [35.079678, -90.074136]
upper_right = [35.194771, -89.868766]
alt = [i for i in range(83, 100)]
for location in range(5):
lat = random.uniform(lower_left[0], upper_right[0])
long = random.uniform(lower_left[1], upper_right[1])
for dp in range(150):
lat_val = random.gauss(lat, 0.001)
long_val = random.gauss(long, 0.001)
alt_val = random.choice(alt)
speed_val = round(random.uniform(0.0, 5.0), 6)
bearing_val = round(random.uniform(0.0, 350), 6)
accuracy_val = round(random.uniform(10.0, 30.4), 6)
timestamp = timestamp + timedelta(minutes=1)
localtime = timestamp + timedelta(hours=5)
sample_data.append(
(timestamp, localtime, user_id, 1, lat_val, long_val, alt_val,
speed_val, bearing_val, accuracy_val))
df = sqlContext.createDataFrame(sample_data, column_name)
stream_metadata = Metadata()
stream_metadata.set_study_name("default").set_name(stream_name).set_description("GPS sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("timestamp").set_type("datetime").set_attribute("description", "UTC timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("localtime").set_type("datetime").set_attribute("description", "local timestamp of data point collection.")) \
.add_dataDescriptor(
DataDescriptor().set_name("user").set_type("string").set_attribute("description", "user id")) \
.add_dataDescriptor(
DataDescriptor().set_name("version").set_type("int").set_attribute("description", "version of the data")) \
.add_dataDescriptor(
DataDescriptor().set_name("latitude").set_type("float").set_attribute("description", "gps latitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("longitude").set_type("float").set_attribute("description", "gps longitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("altitude").set_type("float").set_attribute("description", "gps altitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("speed").set_type("float").set_attribute("description", "speed info")) \
.add_dataDescriptor(
DataDescriptor().set_name("bearing").set_type("float").set_attribute("description", "bearing info")) \
.add_dataDescriptor(
DataDescriptor().set_name("accuracy").set_type("float").set_attribute("description", "accuracy of gps location")) \
.add_module(
ModuleMetadata().set_name("examples.util.data_helper.gen_location_data").set_attribute("attribute_key", "attribute_value").set_author(
"Nasir Ali", "*****@*****.**"))
stream_metadata.is_valid()
ds = DataStream(data=df, metadata=stream_metadata)
return ds
def gen_stress_data(stream_name, spark_df=False):
data = [
[0.7, "road", "Driving", "Was Tailgated", "IN_VEHICLE"],
[0.3, "work", "Job", "Bored / Not enough to do", "STILL"],
[0.5, "home", "Health", "Physical inability", "STILL"],
[0.6, "road", "Driving", "Saw a police car", "IN_VEHICLE"],
[0.38, "work", "Job", "Technology barriers", "STILL"],
[0.2, "home", "Finance", "Missed payment", "UNKNOWN"],
[0.9, "work", "Finance", "Unexpected losses", "WALKING"],
[0.54, "road", "Driving", "Difficulty in navigating", "IN_VEHICLE"],
[0.79, "work", "Job", "Unpleasant conversation", "ON_FOOT"],
[0.28, "road", "Health", "My eating habits", "IN_VEHICLE"],
[
0.47, "road", "Driving", "Indecision at a traffic intersection",
"IN_VEHICLE"
],
[0.67, "work", "Job", "Late arrival", "WALKING"],
]
column_name = [
'user', 'timestamp', 'localtime', 'version', 'start_time', 'end_time',
'density', 'location', 'stresser_main', 'stresser_sub', 'activity'
]
sample_data = []
timestamp = datetime(2019, 1, 9, 11, 34, 59)
for row in range(20, 1, -1):
if row > 10:
user_id = "00000000-afb8-476e-9872-6472b4e66b68"
else:
user_id = "b1117354-ce48-4325-b2e3-78b0cc932819"
timestamp = timestamp + timedelta(
hours=random.choice([1, 3, 7, 2, 4, 5]))
localtime = timestamp - timedelta(hours=5)
start_time = timestamp
end_time = timestamp + timedelta(
minutes=random.choice([12, 6, 8, 16, 29, 45, 2, 3, 8]))
data_vals = random.choice(data)
sample_data.append([
user_id, timestamp, localtime, 1, start_time, end_time,
data_vals[0], data_vals[1], data_vals[2], data_vals[3],
data_vals[4]
])
stream_metadata = Metadata()
stream_metadata.set_study_name("default").set_name(stream_name).set_description("GPS sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("start_time").set_type("datetime").set_attribute("description", "start time of a stress episode.")) \
.add_dataDescriptor(
DataDescriptor().set_name("end_time").set_type("datetime").set_attribute("description", "end time of a stress episode.")) \
.add_dataDescriptor(
DataDescriptor().set_name("density").set_type("float").set_attribute("description", "density of stress")) \
.add_dataDescriptor(
DataDescriptor().set_name("location").set_type("string").set_attribute("description", "location where stress episode was captured.")) \
.add_dataDescriptor(
DataDescriptor().set_name("stresser_main").set_type("string").set_attribute("description", "stressers' main category.")) \
.add_dataDescriptor(
DataDescriptor().set_name("stresser_sub").set_type("string").set_attribute("description", "stressers' sub category.")) \
.add_dataDescriptor(
DataDescriptor().set_name("activity").set_type("string").set_attribute("description", "physical activity name")) \
.add_module(
ModuleMetadata().set_name("examples.util.data_helper.gen_stress_data").set_attribute("attribute_key", "attribute_value").set_author(
"Nasir Ali", "*****@*****.**"))
stream_metadata.is_valid()
if spark_df:
sqlContext = get_or_create_sc("sqlContext")
df = sqlContext.createDataFrame(sample_data, column_name)
else:
df = pd.DataFrame(sample_data, columns=column_name)
ds = DataStream(df, stream_metadata)
return ds
def gen_location_datastream(user_id, stream_name) -> object:
"""
Create pyspark dataframe with some sample gps data (Memphis, TN, lat, long, alt coordinates)
Args:
user_id (str): id of a user
stream_name (str): sample gps stream name
Returns:
DataStream: datastream object of gps location stream with its metadata
"""
column_name = [
"timestamp", "localtime", "user", "version", "latitude", "longitude",
"altitude", "speed", "bearing", "accuracy"
]
sample_data = []
timestamp = datetime(2019, 1, 9, 11, 34, 59)
sqlContext = get_or_create_sc("sqlContext")
lat = [
35.1247391, 35.1257391, 35.1217391, 35.1117391, 35.1317391, 35.1287391,
35.5217391
]
long = [
-89.9750021, -89.9710021, -89.9800021, -89.9670021, -89.9790021,
-89.9710021, -89.8700021
]
alt = [83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0]
for dp in range(500):
lat_val = random.choice(lat)
long_val = random.choice(long)
alt_val = random.choice(alt)
#ts_val = 15094)+(16272882+(dp*1000000))
speed_val = round(random.uniform(0.0, 5.0), 6)
bearing_val = round(random.uniform(0.0, 350), 6)
accuracy_val = round(random.uniform(10.0, 30.4), 6)
#all_dps = ",".join([ts_val, lat_val, long_val, alt_val, speed_val, bearing_val, accuracy_val])
timestamp = timestamp + timedelta(minutes=1)
localtime = timestamp + timedelta(hours=5)
sample_data.append(
(timestamp, localtime, user_id, 1, lat_val, long_val, alt_val,
speed_val, bearing_val, accuracy_val))
df = sqlContext.createDataFrame(sample_data, column_name)
stream_metadata = Metadata()
stream_metadata.set_name(stream_name).set_version(1).set_description("GPS sample data stream.") \
.add_dataDescriptor(
DataDescriptor().set_name("latitude").set_type("float").set_attribute("description", "gps latitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("longitude").set_type("float").set_attribute("description", "gps longitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("altitude").set_type("float").set_attribute("description", "gps altitude")) \
.add_dataDescriptor(
DataDescriptor().set_name("speed").set_type("float").set_attribute("description", "speed info")) \
.add_dataDescriptor(
DataDescriptor().set_name("bearing").set_type("float").set_attribute("description", "bearing info")) \
.add_dataDescriptor(
DataDescriptor().set_name("accuracy").set_type("float").set_attribute("description", "accuracy of gps location")) \
.add_module(
ModuleMetadata().set_name("examples.util.data_helper.gen_location_data").set_version("0.0.1").set_attribute("attribute_key", "attribute_value").set_author(
"test_user", "test_user@test_email.com"))
stream_metadata.is_valid()
return DataStream(data=df, metadata=stream_metadata)
