def test_add_sensor_data(self):
manager = DBManager(filename='openimu.db', overwrite=True)
# Create sensor in DB
group = manager.update_group(
Group(name='Group Name', description='Group Description'))
participant = manager.update_participant(
Participant(name='Participant Name',
description='Participant Description',
group=group))
sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name',
'Hardware Name', 'Wrist', 30.0, 1)
channel = manager.add_channel(sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X')
timestamps = SensorTimestamps()
timestamps.timestamps = np.zeros(40, dtype=np.float64)
# will set start and end
timestamps.update_timestamps()
recordset = manager.add_recordset(participant, 'My Record',
timestamps.start_timestamp,
timestamps.end_timestamp)
data = np.zeros(40, dtype=np.float32)
sensordata = manager.add_sensor_data(recordset, sensor, channel,
timestamps, data)
manager.commit()
sensordata2 = manager.get_sensor_data(sensordata.id_sensor_data)
self.assertEqual(sensordata, sensordata2)
manager.close()
def import_power_to_database(self, timestamp, sensors, channels, recordset,
data: dict):
print('import_imu_to_database')
values = np.array(data['values'], dtype=np.float32)
if len(values) == 0:
return False
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = data['times']
sensor_timestamps.update_timestamps()
self.add_sensor_data_to_db(recordset, sensors['battery'],
channels['battery'], sensor_timestamps,
values[:, 0])
self.add_sensor_data_to_db(recordset, sensors['current'],
channels['current'], sensor_timestamps,
values[:, 1])
self.db.commit()
return True
def import_gps_to_database(self, timestamp, sensors, channels, recordset,
data: dict):
print('import_imu_to_database')
values = np.array(data['values'], dtype=np.float32)
if len(values) == 0:
return False
# Regenerate GPS data to be stored in the DB as SIRF data
# TODO Better GPS solution?
# We have one sample per second of GPS data
for i, val in enumerate(values):
geo = GPSGeodetic()
# Discard invalid data
if math.isnan(val[1]) or math.isnan(val[2]):
continue
geo.latitude = val[1] * 1e7
geo.longitude = val[2] * 1e7
# altitude = val[3] * 1e7
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = data['times'][i:i + 1]
sensor_timestamps.update_timestamps()
self.add_sensor_data_to_db(recordset, sensors['gps'],
channels['gps'], sensor_timestamps, geo)
# Commit to file
self.db.commit()
return True
def import_coordinates_to_database(self, sample_rate, coordinates: dict):
# DL Oct. 17 2018, New import to database
coordinates_sensor = self.add_sensor_to_db(SensorType.GPS,
'Coordinates', 'AppleWatch',
'Wrist', sample_rate, 1)
coordinates_channel = self.add_channel_to_db(coordinates_sensor,
Units.NONE,
DataFormat.UINT8,
'Coordinates')
for timestamp in coordinates:
# print('coordinates', timestamp, len(coordinates[timestamp]['times']),
# len(coordinates[timestamp]['values']))
# Create time array as float64
timesarray = np.asarray(coordinates[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
self.last_error = "Aucune données temporelles."
return
# Other values are float32
valuesarray = np.asarray(coordinates[timestamp]['values'],
dtype=np.float32)
# Create one entry per timestamp ?
# Could we store a vector instead ?
for i, value in enumerate(valuesarray):
# Build gps data
geo = GPSGeodetic()
geo.latitude = value[0] * 1e7
geo.longitude = value[1] * 1e7
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray[i:i + 1]
sensor_timestamps.update_timestamps()
# Calculate recordset
recordset = self.get_recordset(
sensor_timestamps.start_timestamp.timestamp(),
session_name=self.session_name)
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Store
self.add_sensor_data_to_db(recordset, coordinates_sensor,
coordinates_channel,
sensor_timestamps, geo)
self.update_progress.emit(50 + np.floor(i / len(valuesarray) /
2 * 100))
def create_sensor_timestamps(times, recordset):
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = times
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
return sensor_timestamps
def import_battery_to_database(self, sampling_rate, battery: dict):
# DL Oct. 16 2018, New import to database
battery_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery',
'AppleWatch', 'Wrist',
sampling_rate, 1)
battery_channel = self.add_channel_to_db(battery_sensor, Units.VOLTS,
DataFormat.FLOAT32,
'Battery Percentage')
# Data is already hour-aligned iterate through hours
for timestamp in battery:
# print('battery', timestamp, len(battery[timestamp]['times']),
# len(battery[timestamp]['values']))
# Calculate recordset
recordset = self.get_recordset(timestamp.timestamp())
# Import to database
# Create time array as float64
timesarray = np.asarray(battery[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are float32
valuesarray = np.asarray(battery[timestamp]['values'],
dtype=np.float32)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
self.add_sensor_data_to_db(recordset, battery_sensor,
battery_channel, sensor_timestamps,
valuesarray[:, 0])
def import_heartrate_to_database(self, sample_rate, heartrate: dict):
# DL Oct. 17 2018, New import to database
heartrate_sensor = self.add_sensor_to_db(SensorType.HEARTRATE,
'Heartrate', 'AppleWatch',
'Wrist', sample_rate, 1)
heartrate_channel = self.add_channel_to_db(heartrate_sensor, Units.BPM,
DataFormat.UINT8,
'Heartrate')
for timestamp in heartrate:
# print('heartrate', timestamp, len(heartrate[timestamp]['times']),
# len(heartrate[timestamp]['values']))
# Calculate recordset
recordset = self.get_recordset(timestamp.timestamp())
# Create time array as float64
timesarray = np.asarray(heartrate[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are float32
valuesarray = np.asarray(heartrate[timestamp]['values'],
dtype=np.uint8)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Store data
self.add_sensor_data_to_db(recordset, heartrate_sensor,
heartrate_channel, sensor_timestamps,
valuesarray[:, 0])
def import_baro_to_database(self, timestamp, sensors, channels, recordset, data: list):
# print('import_imu_to_database')
values = np.array(data['values'], dtype=np.float32)
if len(values) == 0:
return False
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = data['times']
sensor_timestamps.update_timestamps()
self.add_sensor_data_to_db(recordset, sensors['baro'], channels['baro'],
sensor_timestamps, values[:, 1])
# Commit to file
self.db.commit()
return True
def import_imu_to_database(self, timestamp, sample_rate, sensors, channels,
recordset, data: dict):
print('import_imu_to_database')
values = np.array(data['values'], dtype=np.float32)
if len(values) == 0:
return False
# print("Values shape: ", values.shape)
end_timestamp = np.floor(data['end_time'])
# Update end_timestamp if required
if end_timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = datetime.datetime.fromtimestamp(
end_timestamp)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = data['times']
sensor_timestamps.update_timestamps()
# Acc
for i in range(len(channels['acc'])):
self.add_sensor_data_to_db(recordset, sensors['acc'],
channels['acc'][i], sensor_timestamps,
values[:, i])
# Gyro
for i in range(len(channels['gyro'])):
self.add_sensor_data_to_db(recordset, sensors['gyro'],
channels['gyro'][i], sensor_timestamps,
values[:, i + 3])
# Magnetometer
for i in range(len(channels['mag'])):
self.add_sensor_data_to_db(recordset, sensors['mag'],
channels['mag'][i], sensor_timestamps,
values[:, i + 6])
self.db.commit()
return True
def import_battery_to_database(self, info, battery: dict):
# Create sensor
volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery',
info['Device Type'], 'General', 10,
1)
# Create channel
volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS,
DataFormat.FLOAT32, 'Battery')
for epoch in battery:
timestamp = datetime.datetime.fromtimestamp(epoch[0])
session_name = str(timestamp) + '_' + info['Device Type'] + '_' + info['Subject Name'] \
+ '_SN:' + info['Serial Number']
value = np.float32(epoch[1])
recordset = self.get_recordset(epoch[0], session_name)
timevect = np.linspace(epoch[0],
epoch[0] + 1,
num=1,
endpoint=False,
dtype=np.float64)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if epoch[0] > recordset.end_timestamp.timestamp():
recordset.end_timestamp = timestamp
self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel,
sensor_timestamps, value)
# Flush to DB (ram)
self.db.flush()
def import_lux_to_database(self, info, lux: dict):
# Create sensor
lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux',
info['Device Type'], 'Unknown', 0,
1)
# Create channel
lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX,
DataFormat.FLOAT32, 'Lux')
for epoch in lux:
timestamp = datetime.datetime.fromtimestamp(epoch[0])
session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
value = np.float32(epoch[1])
recordset = self.get_recordset(epoch[0], session_name)
timevect = np.linspace(epoch[0],
epoch[0] + 1,
num=1,
endpoint=False,
dtype=np.float64)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if epoch[0] > recordset.end_timestamp.timestamp():
recordset.end_timestamp = timestamp
self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel,
sensor_timestamps, value)
# Flush to DB (ram)
self.db.flush()
def import_motion_to_database(self, sampling_rate, motion: dict):
# DL Oct. 16 2018, New import to database
# Create channels and sensors
accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
'Accelerometer',
'AppleWatch', 'Wrist',
sampling_rate, 1)
accelerometer_channels = list()
# Create channels
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Y'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Z'))
# Create sensor
gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER, 'Gyro',
'AppleWatch', 'Wrist',
sampling_rate, 1)
gyro_channels = list()
# Create channels
gyro_channels.append(
self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_X'))
gyro_channels.append(
self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Y'))
gyro_channels.append(
self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Z'))
# Data is already hour-aligned iterate through hours
for timestamp in motion:
# print('motion', timestamp, len(motion[timestamp]['times']),
# len(motion[timestamp]['values']))
# Calculate recordset
recordset = self.get_recordset(timestamp.timestamp())
# Add motion data to database
# Create time array as float64
timesarray = np.asarray(motion[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are float32
valuesarray = np.asarray(motion[timestamp]['values'],
dtype=np.float32)
# Motion contains in this order
# acceleration(x, y, z)
# gravity vector(x, y, z)
# gyroscope(x, y, z)
# attitude quaternion(w, x, y, z)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Acc
for i in range(len(accelerometer_channels)):
self.add_sensor_data_to_db(recordset, accelerometer_sensor,
accelerometer_channels[i],
sensor_timestamps, valuesarray[:,
i])
# Gyro
for i in range(len(gyro_channels)):
self.add_sensor_data_to_db(recordset, gyro_sensor,
gyro_channels[i], sensor_timestamps,
valuesarray[:, i + 6])
def import_beacons_to_database(self, sample_rate, beacons: dict):
# DL Oct. 17 2018, New import to database
beacons_sensor = self.add_sensor_to_db(SensorType.BEACON, 'Beacons',
'Kontact', 'Environment',
sample_rate, 1)
channel_values = dict()
# Data is already hour-aligned iterate through hours
for timestamp in beacons:
# print('beacons', timestamp, len(beacons[timestamp]['times']),
# len(beacons[timestamp]['values']))
# Create time array as float64
timesarray = np.asarray(beacons[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are int8
valuesarray = np.asarray(beacons[timestamp]['values'],
dtype=np.int8)
# Iterate through each entry to generate data for each beacon_id
for i in range(0, len(timesarray)):
name = [
str(format(x, 'x')).rjust(2, '0')
for x in beacons[timestamp]['values'][i][0:16]
]
beacon_id = ''.join(name[0:10]) + '_' + ''.join(name[10:])
# Create channel if it does not exist
if not channel_values.__contains__(beacon_id):
channel_values[beacon_id] = []
channel_values[beacon_id].append(
(timesarray[i], valuesarray[i][14], valuesarray[i][15]))
# Store each beacon_id in separate channels
for key in channel_values:
timevect = np.asarray([x[0] for x in channel_values[key]],
dtype=np.float64)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
# Calculate recordset
recordset = self.get_recordset(
sensor_timestamps.start_timestamp.timestamp())
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Create channel
channel_txPower = self.add_channel_to_db(
beacons_sensor, Units.NONE, DataFormat.SINT8,
key + '_TxPower')
channel_RSSI = self.add_channel_to_db(beacons_sensor,
Units.NONE,
DataFormat.SINT8,
key + '_RSSI')
tx_power_vect = np.asarray([x[1] for x in channel_values[key]],
dtype=np.int8)
rssi_vect = np.asarray([x[2] for x in channel_values[key]],
dtype=np.int8)
# Add data
self.add_sensor_data_to_db(recordset, beacons_sensor,
channel_txPower, sensor_timestamps,
tx_power_vect)
self.add_sensor_data_to_db(recordset, beacons_sensor,
channel_RSSI, sensor_timestamps,
rssi_vect)
def import_sensoria_to_database(self, sample_rate, sensoria: dict):
# DL Oct. 17 2018, New import to database
sensoria_acc_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
'Accelerometer',
'Sensoria', 'Foot',
sample_rate, 1)
sensoria_acc_channels = list()
sensoria_acc_channels.append(
self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X'))
sensoria_acc_channels.append(
self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Y'))
sensoria_acc_channels.append(
self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Z'))
sensoria_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER,
'Gyrometer', 'Sensoria',
'Foot', sample_rate, 1)
sensoria_gyro_channels = list()
sensoria_gyro_channels.append(
self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_X'))
sensoria_gyro_channels.append(
self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Y'))
sensoria_gyro_channels.append(
self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Z'))
sensoria_mag_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER,
'Magnetometer', 'Sensoria',
'Foot', sample_rate, 1)
sensoria_mag_channels = list()
sensoria_mag_channels.append(
self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
DataFormat.FLOAT32, 'Mag_X'))
sensoria_mag_channels.append(
self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
DataFormat.FLOAT32, 'Mag_Y'))
sensoria_mag_channels.append(
self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
DataFormat.FLOAT32, 'Mag_Z'))
sensoria_fsr_sensor = self.add_sensor_to_db(SensorType.FSR, 'FSR',
'Sensoria', 'Foot',
sample_rate, 1)
sensoria_fsr_channels = list()
sensoria_fsr_channels.append(
self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
DataFormat.FLOAT32, 'META-1'))
sensoria_fsr_channels.append(
self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
DataFormat.FLOAT32, 'META-5'))
sensoria_fsr_channels.append(
self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
DataFormat.FLOAT32, 'HEEL'))
for timestamp in sensoria:
# print('sensoria', timestamp, len(sensoria[timestamp]['times']),
# len(sensoria[timestamp]['values']))
# Calculate recordset
recordset = self.get_recordset(timestamp.timestamp())
# Create time array as float64
timesarray = np.asarray(sensoria[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are float32
valuesarray = np.asarray(sensoria[timestamp]['values'],
dtype=np.float32)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Store FSR
for i in range(len(sensoria_fsr_channels)):
self.add_sensor_data_to_db(recordset, sensoria_fsr_sensor,
sensoria_fsr_channels[i],
sensor_timestamps,
valuesarray[:, i + 1])
# Store Acc
for i in range(len(sensoria_acc_channels)):
self.add_sensor_data_to_db(recordset, sensoria_acc_sensor,
sensoria_acc_channels[i],
sensor_timestamps,
valuesarray[:, i + 4])
# Store Gyro
for i in range(len(sensoria_gyro_channels)):
self.add_sensor_data_to_db(recordset, sensoria_gyro_sensor,
sensoria_gyro_channels[i],
sensor_timestamps,
valuesarray[:, i + 7])
# Magneto
for i in range(len(sensoria_mag_channels)):
self.add_sensor_data_to_db(recordset, sensoria_mag_sensor,
sensoria_mag_channels[i],
sensor_timestamps,
valuesarray[:, i + 10])
def import_raw_gyro_to_database(self, sample_rate, raw_gyro: dict):
# DL Oct. 17 2018, New import to database
# Create sensor
raw_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER,
'Raw Gyro', 'AppleWatch',
'Wrist', sample_rate, 1)
raw_gyro_channels = list()
# Create channels
raw_gyro_channels.append(
self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_X'))
raw_gyro_channels.append(
self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Y'))
raw_gyro_channels.append(
self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyro_Z'))
# Data is already hour-aligned iterate through hours
for timestamp in raw_gyro:
# print('raw_gyro', timestamp, len(raw_gyro[timestamp]['times']),
# len(raw_gyro[timestamp]['values']))
# Calculate recordset
recordset = self.get_recordset(timestamp.timestamp())
# Create time array as float64
timesarray = np.asarray(raw_gyro[timestamp]['times'],
dtype=np.float64)
if len(timesarray) is 0:
print('Empty data, returning')
return
# Other values are float32
valuesarray = np.asarray(raw_gyro[timestamp]['values'],
dtype=np.float32)
# raw_accelero contains in this order
# gyro(x, y, z)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timesarray
sensor_timestamps.update_timestamps()
# Update timestamps in recordset
# This should not happen, recordset is initialized at the beginning of the hour
if sensor_timestamps.start_timestamp < recordset.start_timestamp:
recordset.start_timestamp = sensor_timestamps.start_timestamp
# This can occur though
if sensor_timestamps.end_timestamp > recordset.end_timestamp:
recordset.end_timestamp = sensor_timestamps.end_timestamp
# Gyro
for i in range(len(raw_gyro_channels)):
self.add_sensor_data_to_db(recordset, raw_gyro_sensor,
raw_gyro_channels[i],
sensor_timestamps, valuesarray[:,
i])
def test_get_all_sensor_data_with_args(self):
manager = DBManager(filename='openimu.db', overwrite=True, echo=False)
# Create sensor in DB
group = manager.update_group(
Group(name='Group Name', description='Group Description'))
participant = manager.update_participant(
Participant(name='Participant Name',
description='Participant Description',
group=group))
sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name',
'Hardware Name', 'Wrist', 30.0, 1)
sensor2 = manager.add_sensor(SensorType.GYROMETER, 'Sensor Name',
'Hardware Name', 'Wrist', 30.0, 1)
channel1 = manager.add_channel(sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X')
channel2 = manager.add_channel(sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Y')
timestamps = SensorTimestamps()
timestamps.timestamps = np.zeros(40, dtype=np.float64)
# will set start and end
timestamps.update_timestamps()
recordset = manager.add_recordset(participant, 'My Record',
timestamps.start_timestamp,
timestamps.end_timestamp)
data = np.zeros(40, dtype=np.float32)
sensordata = manager.add_sensor_data(recordset, sensor, channel1,
timestamps, data)
sensordata = manager.add_sensor_data(recordset, sensor, channel2,
timestamps, data)
manager.commit()
# Test with no args, return everything in the recordset
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True)
self.assertEqual(len(sensordata_res), 2)
for sensor_data in sensordata_res:
self.assertEqual(len(sensor_data.data), len(data))
# Test with a valid sensor arg
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
sensor=sensor)
self.assertEqual(len(sensordata_res), 2)
for sensor_data in sensordata_res:
self.assertEqual(len(sensor_data.data), len(data))
# Test with not the right sensor arg
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
sensor=sensor2)
self.assertEqual(len(sensordata_res), 0)
# Testing with invalid sensor arg
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
sensor=Sensor())
self.assertEqual(len(sensordata_res), 0)
# Testing with channel1
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
channel=channel1)
self.assertEqual(len(sensordata_res), 1)
for sensor_data in sensordata_res:
self.assertEqual(len(sensor_data.data), len(data))
# Testing with channel2
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
channel=channel2)
self.assertEqual(len(sensordata_res), 1)
for sensor_data in sensordata_res:
self.assertEqual(len(sensor_data.data), len(data))
# Testing with invalid channel
sensordata_res = manager.get_all_sensor_data(recordset=recordset,
convert=True,
channel=Channel())
self.assertEqual(len(sensordata_res), 0)
manager.close()
def import_sensor_data_to_database(self, info, sensor_data: dict):
if len(sensor_data) == 0:
return
accelerometer_sensor = None
gyroscope_sensor = None
magneto_sensor = None
temp_sensor = None
accelerometer_channels = list()
gyroscope_channels = list()
magneto_channels = list()
temp_channel = None
# Create channels for sensors
sensor_names = sensor_data[0][1].keys()
for sensor in sensor_names:
if sensor == 'Accelerometer':
# Create sensor
accelerometer_sensor = self.add_sensor_to_db(
SensorType.ACCELEROMETER, 'Accelerometer',
info['Device Type'], 'Other', info['Sample Rate'], 1)
# Create channels
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor,
Units.GRAVITY_G, DataFormat.FLOAT32,
'Accelerometer_X'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor,
Units.GRAVITY_G, DataFormat.FLOAT32,
'Accelerometer_Y'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor,
Units.GRAVITY_G, DataFormat.FLOAT32,
'Accelerometer_Z'))
if sensor == 'Gyroscope':
# Create sensor
gyroscope_sensor = self.add_sensor_to_db(
SensorType.GYROMETER, 'Gyroscope', info['Device Type'],
'Other', info['Sample Rate'], 1)
# Create channels
gyroscope_channels.append(
self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyroscope_X'))
gyroscope_channels.append(
self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyroscope_Y'))
gyroscope_channels.append(
self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
DataFormat.FLOAT32, 'Gyroscope_Z'))
if sensor == 'Magnetometer':
# Create sensor
magneto_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER,
'Magnetometer',
info['Device Type'],
'Other',
info['Sample Rate'], 1)
# Create channels
magneto_channels.append(
self.add_channel_to_db(magneto_sensor, Units.UTESLA,
DataFormat.FLOAT32,
'Magnetometer_X'))
magneto_channels.append(
self.add_channel_to_db(magneto_sensor, Units.UTESLA,
DataFormat.FLOAT32,
'Magnetometer_Y'))
magneto_channels.append(
self.add_channel_to_db(magneto_sensor, Units.UTESLA,
DataFormat.FLOAT32,
'Magnetometer_Z'))
if sensor == 'Temperature':
# Create sensor
temp_sensor = self.add_sensor_to_db(SensorType.TEMPERATURE,
'Temperature',
info['Device Type'],
'Other',
info['Sample Rate'], 1)
temp_channel = self.add_channel_to_db(temp_sensor,
Units.CELCIUS,
DataFormat.FLOAT32,
'Temperature')
# Reshape data vectors
data_times = [data[0] for data in sensor_data]
acc_data = []
gyro_data = []
mag_data = []
temp_data = []
if 'Accelerometer' in sensor_names:
values = [data[1]['Accelerometer'] for data in sensor_data]
for index, value in enumerate(values):
timevect = np.linspace(data_times[index],
data_times[index] + 1,
num=len(value),
endpoint=False,
dtype=np.float64)
shaped_values = np.reshape(value, [-1, 3])
acc_data.extend(list(zip(timevect, shaped_values)))
if 'Gyroscope' in sensor_names:
values = [data[1]['Gyroscope'] for data in sensor_data]
for index, value in enumerate(values):
timevect = np.linspace(data_times[index],
data_times[index] + 1,
num=len(value),
endpoint=False,
dtype=np.float64)
shaped_values = np.reshape(value, [-1, 3])
gyro_data.extend(list(zip(timevect, shaped_values)))
if 'Magnetometer' in sensor_names:
values = [data[1]['Magnetometer'] for data in sensor_data]
for index, value in enumerate(values):
timevect = np.linspace(data_times[index],
data_times[index] + 1,
num=len(value),
endpoint=False,
dtype=np.float64)
shaped_values = np.reshape(value, [-1, 3])
mag_data.extend(list(zip(timevect, shaped_values)))
if 'Temperature' in sensor_names:
values = [data[1]['Temperature'] for data in sensor_data]
for index, value in enumerate(values):
timevect = np.linspace(data_times[index],
data_times[index] + 1,
num=len(value),
endpoint=False,
dtype=np.float64)
temp_data.extend(list(zip(timevect, value)))
# Find holes in the recording
split_indexes = [0]
split_indexes.extend([
index + 1 for index, value in enumerate(np.diff(data_times))
if value != 1
])
split_indexes.append(len(data_times) - 1)
# Find chunks of data longer than 1h
indexes_to_add = []
for index in range(1, len(split_indexes)):
if (split_indexes[index] - split_indexes[index - 1]) > 3600:
current_index = split_indexes[index - 1]
while current_index < split_indexes[index]:
current_index += 3600
if current_index < split_indexes[-1]:
indexes_to_add.append(current_index)
split_indexes.extend(indexes_to_add)
split_indexes.sort()
if 'Accelerometer' in sensor_names:
# Detect hours transition to prevent recordset from being longer than 1h
for record_index in range(1, len(split_indexes)):
if record_index == len(split_indexes):
next_cut_timestamp = acc_data[-1][0] + 1
else:
next_cut_timestamp = data_times[
split_indexes[record_index]]
current_timestamp = data_times[split_indexes[record_index - 1]]
data_recordset = [
data for data in acc_data
if current_timestamp <= data[0] < next_cut_timestamp
]
base_timestamp = data_recordset[0][0]
session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
recordset = self.get_recordset(base_timestamp, session_name)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = np.asarray(
[row[0] for row in data_recordset], dtype=np.float64)
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if base_timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = base_timestamp
for index in range(0, len(accelerometer_channels)):
values = np.asarray(
[row[1][index] for row in data_recordset],
dtype=np.float32)
self.add_sensor_data_to_db(recordset, accelerometer_sensor,
accelerometer_channels[index],
sensor_timestamps, values)
if 'Gyroscope' in sensor_names:
# Detect hours transition to prevent recordset from being longer than 1h
for record_index in range(1, len(split_indexes)):
if record_index == len(split_indexes):
next_cut_timestamp = gyro_data[-1][0] + 1
else:
next_cut_timestamp = data_times[
split_indexes[record_index]]
current_timestamp = data_times[split_indexes[record_index - 1]]
data_recordset = [
data for data in gyro_data
if current_timestamp <= data[0] < next_cut_timestamp
]
base_timestamp = data_recordset[0][0]
session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
recordset = self.get_recordset(base_timestamp, session_name)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = np.asarray(
[row[0] for row in data_recordset], dtype=np.float64)
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if base_timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = base_timestamp
for index in range(0, len(gyroscope_channels)):
values = np.asarray(
[row[1][index] for row in data_recordset],
dtype=np.float32)
self.add_sensor_data_to_db(recordset, gyroscope_sensor,
gyroscope_channels[index],
sensor_timestamps, values)
next_cut_timestamp += 3600
if 'Magnetometer' in sensor_names:
# Detect hours transition to prevent recordset from being longer than 1h
for record_index in range(1, len(split_indexes)):
if record_index == len(split_indexes):
next_cut_timestamp = mag_data[-1][0] + 1
else:
next_cut_timestamp = data_times[
split_indexes[record_index]]
current_timestamp = data_times[split_indexes[record_index - 1]]
data_recordset = [
data for data in mag_data
if current_timestamp <= data[0] < next_cut_timestamp
]
base_timestamp = data_recordset[0][0]
session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
recordset = self.get_recordset(base_timestamp, session_name)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = np.asarray(
[row[0] for row in data_recordset], dtype=np.float64)
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if base_timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = base_timestamp
for index in range(0, len(magneto_channels)):
values = np.asarray(
[row[1][index] for row in data_recordset],
dtype=np.float32)
self.add_sensor_data_to_db(recordset, magneto_sensor,
magneto_channels[index],
sensor_timestamps, values)
next_cut_timestamp += 3600
if 'Temperature' in sensor_names:
# Detect hours transition to prevent recordset from being longer than 1h
for record_index in range(1, len(split_indexes)):
if record_index == len(split_indexes):
next_cut_timestamp = temp_data[-1][0] + 1
else:
next_cut_timestamp = data_times[
split_indexes[record_index]]
current_timestamp = data_times[split_indexes[record_index - 1]]
data_recordset = [
data for data in temp_data
if current_timestamp <= data[0] < next_cut_timestamp
]
base_timestamp = data_recordset[0][0]
session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
recordset = self.get_recordset(base_timestamp, session_name)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = np.asarray(
[row[0] for row in data_recordset], dtype=np.float64)
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if base_timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = base_timestamp
values = np.asarray([row[1] for row in data_recordset],
dtype=np.float32)
self.add_sensor_data_to_db(recordset, temp_sensor,
temp_channel, sensor_timestamps,
values)
next_cut_timestamp += 3600
# Flush DB
self.db.flush()
def import_to_database(self, results):
[info, data] = results
# print(info)
# Creating recordset
# print(info['Start Date'], info['Last Sample Time'])
# start = int(info['Start Date'])
# stop = int(info['Last Sample Time'])
# start_timestamp = ticksconverter(start)
# end_timestamp = ticksconverter(stop)
# all_counts = [0, 0, 0]
if data.__contains__('activity'):
# print('activity found')
# Create sensor
accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', info['Device Type'],
'Unknown', info['Sample Rate'], 1)
accelerometer_channels = list()
# Create channels
accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Y'))
accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X'))
accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Z'))
# Should be 1970, epoch
last_timestamp = 0
all_timestamps = []
value_dict = {}
# Import data
for epoch in data['activity']:
# An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z
current_timestamp = epoch[0]
# print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1))
# Check for consecutive timestamps
create_array = current_timestamp != (last_timestamp + 1)
# Do not allow more than one hour of consecutive data
if create_array is not True:
if current_timestamp - all_timestamps[-1] >= 3600:
create_array = True
# Consecutive timestamps?
if create_array is True:
all_timestamps.append(current_timestamp)
# Create list for all values for this timestamp
value_dict[current_timestamp] = [list(), list(), list()]
# Get data
samples = epoch[1]
# Separate write for each channel
for index in range(0, len(accelerometer_channels)):
# Using last timestamp to append data
value_dict[all_timestamps[-1]][index].append(samples[:, index])
# print("samples shape", samples.shape, samples[:, index].shape)
# Update timestamp
last_timestamp = current_timestamp
# Insert into DB as chunks of data
# print('should insert records count: ', len(all_timestamps))
# print('should insert data count:', len(value_dict))
counters = [0, 0, 0]
for timestamp in all_timestamps:
session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
for index in range(0, len(value_dict[timestamp])):
# print(index, timestamp, len(value_dict[timestamp][index]))
vector = np.concatenate(value_dict[timestamp][index])
# print('inserting values :', len(value_dict[timestamp][index]))
# print('vector: ', len(vector), vector.shape, vector.dtype)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
# Create time vector
# TODO Share time vector for all accelerometers?
timevect = np.linspace(timestamp, timestamp + len(value_dict[timestamp][index]),
num=len(vector), endpoint=False, dtype=np.float64)
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
recordset = self.get_recordset(timestamp, session_name)
# Update end_timestamp if required
if timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = datetime.datetime.fromtimestamp(timestamp)
counters[index] += len(vector)
if len(vector) > 0:
self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[index],
sensor_timestamps, vector)
# print('total samples inserted:', counters)
# print('total timestamps processed:', len(all_timestamps))
# Flush DB
self.db.flush()
if data.__contains__('battery'):
# print('battery found')
# Create sensor
volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery', info['Device Type'], 'Unknown',
0, 1)
# Create channel
volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS, DataFormat.FLOAT32, 'Battery')
for epoch in data['battery']:
timestamp = datetime.datetime.fromtimestamp(epoch[0])
session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
value = np.float32(epoch[1])
recordset = self.get_recordset(epoch[0], session_name)
timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if epoch[0] > recordset.end_timestamp.timestamp():
recordset.end_timestamp = timestamp
self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel, sensor_timestamps, value)
# Flush to DB (ram)
self.db.flush()
if data.__contains__('lux'):
# print('lux found')
# Create sensor
lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux', info['Device Type'], 'Unknown', 0, 1)
# Create channel
lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX, DataFormat.FLOAT32, 'Lux')
for epoch in data['lux']:
timestamp = datetime.datetime.fromtimestamp(epoch[0])
session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
value = np.float32(epoch[1])
recordset = self.get_recordset(epoch[0], session_name)
timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
# Update end_timestamp if required
if epoch[0] > recordset.end_timestamp.timestamp():
recordset.end_timestamp = timestamp
self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel, sensor_timestamps, value)
# Flush to DB (ram)
self.db.flush()
# Write data to file
self.db.commit()
self.update_progress.emit(100)
def import_activity_to_database(self, info, activity: dict):
# print('activity found')
# Create sensor
accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
'Accelerometer',
info['Device Type'],
'Activity',
info['Sample Rate'], 1)
accelerometer_channels = list()
# Create channels
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Y'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_X'))
accelerometer_channels.append(
self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
DataFormat.FLOAT32, 'Accelerometer_Z'))
# Should be 1970, epoch
last_timestamp = 0
all_timestamps = []
value_dict = {}
# Import data
for epoch in activity:
# An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z
current_timestamp = epoch[0]
# print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1))
# Check for consecutive timestamps
create_array = current_timestamp != (last_timestamp + 1)
# Do not allow more than one hour of consecutive data
if create_array is not True:
if current_timestamp - all_timestamps[-1] >= 3600:
create_array = True
# Consecutive timestamps?
if create_array is True:
all_timestamps.append(current_timestamp)
# Create list for all values for this timestamp
value_dict[current_timestamp] = [list(), list(), list()]
# Get data
samples = epoch[1]
# Separate write for each channel
for index in range(0, len(accelerometer_channels)):
# Using last timestamp to append data
value_dict[all_timestamps[-1]][index].append(samples[:, index])
# print("samples shape", samples.shape, samples[:, index].shape)
# Update timestamp
last_timestamp = current_timestamp
# Insert into DB as chunks of data
# print('should insert records count: ', len(all_timestamps))
# print('should insert data count:', len(value_dict))
counters = [0, 0, 0]
for timestamp in all_timestamps:
session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
+ info['Subject Name'] + '_SN:' + info['Serial Number']
for index in range(0, len(value_dict[timestamp])):
# print(index, timestamp, len(value_dict[timestamp][index]))
vector = np.concatenate(value_dict[timestamp][index])
# print('inserting values :', len(value_dict[timestamp][index]))
# print('vector: ', len(vector), vector.shape, vector.dtype)
# Create sensor timestamps first
sensor_timestamps = SensorTimestamps()
# Create time vector
# TODO Share time vector for all accelerometers?
timevect = np.linspace(timestamp,
timestamp +
len(value_dict[timestamp][index]),
num=len(vector),
endpoint=False,
dtype=np.float64)
sensor_timestamps.timestamps = timevect
sensor_timestamps.update_timestamps()
recordset = self.get_recordset(timestamp, session_name)
# Update end_timestamp if required
if timestamp > recordset.end_timestamp.timestamp():
recordset.end_timestamp = datetime.datetime.fromtimestamp(
timestamp)
counters[index] += len(vector)
if len(vector) > 0:
self.add_sensor_data_to_db(recordset, accelerometer_sensor,
accelerometer_channels[index],
sensor_timestamps, vector)
# print('total samples inserted:', counters)
# print('total timestamps processed:', len(all_timestamps))
# Flush DB
self.db.flush()