def test_add_sensor_data(self): manager = DBManager(filename='openimu.db', overwrite=True) # Create sensor in DB group = manager.update_group( Group(name='Group Name', description='Group Description')) participant = manager.update_participant( Participant(name='Participant Name', description='Participant Description', group=group)) sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name', 'Hardware Name', 'Wrist', 30.0, 1) channel = manager.add_channel(sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X') timestamps = SensorTimestamps() timestamps.timestamps = np.zeros(40, dtype=np.float64) # will set start and end timestamps.update_timestamps() recordset = manager.add_recordset(participant, 'My Record', timestamps.start_timestamp, timestamps.end_timestamp) data = np.zeros(40, dtype=np.float32) sensordata = manager.add_sensor_data(recordset, sensor, channel, timestamps, data) manager.commit() sensordata2 = manager.get_sensor_data(sensordata.id_sensor_data) self.assertEqual(sensordata, sensordata2) manager.close()
def import_power_to_database(self, timestamp, sensors, channels, recordset, data: dict): print('import_imu_to_database') values = np.array(data['values'], dtype=np.float32) if len(values) == 0: return False # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = data['times'] sensor_timestamps.update_timestamps() self.add_sensor_data_to_db(recordset, sensors['battery'], channels['battery'], sensor_timestamps, values[:, 0]) self.add_sensor_data_to_db(recordset, sensors['current'], channels['current'], sensor_timestamps, values[:, 1]) self.db.commit() return True
def import_gps_to_database(self, timestamp, sensors, channels, recordset, data: dict): print('import_imu_to_database') values = np.array(data['values'], dtype=np.float32) if len(values) == 0: return False # Regenerate GPS data to be stored in the DB as SIRF data # TODO Better GPS solution? # We have one sample per second of GPS data for i, val in enumerate(values): geo = GPSGeodetic() # Discard invalid data if math.isnan(val[1]) or math.isnan(val[2]): continue geo.latitude = val[1] * 1e7 geo.longitude = val[2] * 1e7 # altitude = val[3] * 1e7 # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = data['times'][i:i + 1] sensor_timestamps.update_timestamps() self.add_sensor_data_to_db(recordset, sensors['gps'], channels['gps'], sensor_timestamps, geo) # Commit to file self.db.commit() return True
def import_coordinates_to_database(self, sample_rate, coordinates: dict): # DL Oct. 17 2018, New import to database coordinates_sensor = self.add_sensor_to_db(SensorType.GPS, 'Coordinates', 'AppleWatch', 'Wrist', sample_rate, 1) coordinates_channel = self.add_channel_to_db(coordinates_sensor, Units.NONE, DataFormat.UINT8, 'Coordinates') for timestamp in coordinates: # print('coordinates', timestamp, len(coordinates[timestamp]['times']), # len(coordinates[timestamp]['values'])) # Create time array as float64 timesarray = np.asarray(coordinates[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: self.last_error = "Aucune données temporelles." return # Other values are float32 valuesarray = np.asarray(coordinates[timestamp]['values'], dtype=np.float32) # Create one entry per timestamp ? # Could we store a vector instead ? for i, value in enumerate(valuesarray): # Build gps data geo = GPSGeodetic() geo.latitude = value[0] * 1e7 geo.longitude = value[1] * 1e7 # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray[i:i + 1] sensor_timestamps.update_timestamps() # Calculate recordset recordset = self.get_recordset( sensor_timestamps.start_timestamp.timestamp(), session_name=self.session_name) # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Store self.add_sensor_data_to_db(recordset, coordinates_sensor, coordinates_channel, sensor_timestamps, geo) self.update_progress.emit(50 + np.floor(i / len(valuesarray) / 2 * 100))
def create_sensor_timestamps(times, recordset): # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = times sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp return sensor_timestamps
def import_battery_to_database(self, sampling_rate, battery: dict): # DL Oct. 16 2018, New import to database battery_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery', 'AppleWatch', 'Wrist', sampling_rate, 1) battery_channel = self.add_channel_to_db(battery_sensor, Units.VOLTS, DataFormat.FLOAT32, 'Battery Percentage') # Data is already hour-aligned iterate through hours for timestamp in battery: # print('battery', timestamp, len(battery[timestamp]['times']), # len(battery[timestamp]['values'])) # Calculate recordset recordset = self.get_recordset(timestamp.timestamp()) # Import to database # Create time array as float64 timesarray = np.asarray(battery[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are float32 valuesarray = np.asarray(battery[timestamp]['values'], dtype=np.float32) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp self.add_sensor_data_to_db(recordset, battery_sensor, battery_channel, sensor_timestamps, valuesarray[:, 0])
def import_heartrate_to_database(self, sample_rate, heartrate: dict): # DL Oct. 17 2018, New import to database heartrate_sensor = self.add_sensor_to_db(SensorType.HEARTRATE, 'Heartrate', 'AppleWatch', 'Wrist', sample_rate, 1) heartrate_channel = self.add_channel_to_db(heartrate_sensor, Units.BPM, DataFormat.UINT8, 'Heartrate') for timestamp in heartrate: # print('heartrate', timestamp, len(heartrate[timestamp]['times']), # len(heartrate[timestamp]['values'])) # Calculate recordset recordset = self.get_recordset(timestamp.timestamp()) # Create time array as float64 timesarray = np.asarray(heartrate[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are float32 valuesarray = np.asarray(heartrate[timestamp]['values'], dtype=np.uint8) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Store data self.add_sensor_data_to_db(recordset, heartrate_sensor, heartrate_channel, sensor_timestamps, valuesarray[:, 0])
def import_baro_to_database(self, timestamp, sensors, channels, recordset, data: list): # print('import_imu_to_database') values = np.array(data['values'], dtype=np.float32) if len(values) == 0: return False # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = data['times'] sensor_timestamps.update_timestamps() self.add_sensor_data_to_db(recordset, sensors['baro'], channels['baro'], sensor_timestamps, values[:, 1]) # Commit to file self.db.commit() return True
def import_imu_to_database(self, timestamp, sample_rate, sensors, channels, recordset, data: dict): print('import_imu_to_database') values = np.array(data['values'], dtype=np.float32) if len(values) == 0: return False # print("Values shape: ", values.shape) end_timestamp = np.floor(data['end_time']) # Update end_timestamp if required if end_timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = datetime.datetime.fromtimestamp( end_timestamp) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = data['times'] sensor_timestamps.update_timestamps() # Acc for i in range(len(channels['acc'])): self.add_sensor_data_to_db(recordset, sensors['acc'], channels['acc'][i], sensor_timestamps, values[:, i]) # Gyro for i in range(len(channels['gyro'])): self.add_sensor_data_to_db(recordset, sensors['gyro'], channels['gyro'][i], sensor_timestamps, values[:, i + 3]) # Magnetometer for i in range(len(channels['mag'])): self.add_sensor_data_to_db(recordset, sensors['mag'], channels['mag'][i], sensor_timestamps, values[:, i + 6]) self.db.commit() return True
def import_battery_to_database(self, info, battery: dict): # Create sensor volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery', info['Device Type'], 'General', 10, 1) # Create channel volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS, DataFormat.FLOAT32, 'Battery') for epoch in battery: timestamp = datetime.datetime.fromtimestamp(epoch[0]) session_name = str(timestamp) + '_' + info['Device Type'] + '_' + info['Subject Name'] \ + '_SN:' + info['Serial Number'] value = np.float32(epoch[1]) recordset = self.get_recordset(epoch[0], session_name) timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() # Update end_timestamp if required if epoch[0] > recordset.end_timestamp.timestamp(): recordset.end_timestamp = timestamp self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel, sensor_timestamps, value) # Flush to DB (ram) self.db.flush()
def import_lux_to_database(self, info, lux: dict): # Create sensor lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux', info['Device Type'], 'Unknown', 0, 1) # Create channel lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX, DataFormat.FLOAT32, 'Lux') for epoch in lux: timestamp = datetime.datetime.fromtimestamp(epoch[0]) session_name = str(timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] value = np.float32(epoch[1]) recordset = self.get_recordset(epoch[0], session_name) timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() # Update end_timestamp if required if epoch[0] > recordset.end_timestamp.timestamp(): recordset.end_timestamp = timestamp self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel, sensor_timestamps, value) # Flush to DB (ram) self.db.flush()
def test_to_time_series(self): sensordata = SensorData() sensordata.id_sensor_data = 1 recordset = Recordset() sensordata.recordset = recordset sensor = Sensor() sensor.sampling_rate = 5 sensor.data_rate = 1 sensordata.sensor = sensor channel = Channel() sensordata.channel = channel sensordata.data = np.array([1, 2, 3, 4, 5], dtype=float) sensordata.timestamps = SensorTimestamps() sensordata.timestamps.timestamps = np.array([0, 1, 2, 3, 4], dtype=np.float64) sensordata.timestamps.update_timestamps() print(sensordata.to_time_series()) print(sensordata)
def import_motion_to_database(self, sampling_rate, motion: dict): # DL Oct. 16 2018, New import to database # Create channels and sensors accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', 'AppleWatch', 'Wrist', sampling_rate, 1) accelerometer_channels = list() # Create channels accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Z')) # Create sensor gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER, 'Gyro', 'AppleWatch', 'Wrist', sampling_rate, 1) gyro_channels = list() # Create channels gyro_channels.append( self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_X')) gyro_channels.append( self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Y')) gyro_channels.append( self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Z')) # Data is already hour-aligned iterate through hours for timestamp in motion: # print('motion', timestamp, len(motion[timestamp]['times']), # len(motion[timestamp]['values'])) # Calculate recordset recordset = self.get_recordset(timestamp.timestamp()) # Add motion data to database # Create time array as float64 timesarray = np.asarray(motion[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are float32 valuesarray = np.asarray(motion[timestamp]['values'], dtype=np.float32) # Motion contains in this order # acceleration(x, y, z) # gravity vector(x, y, z) # gyroscope(x, y, z) # attitude quaternion(w, x, y, z) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Acc for i in range(len(accelerometer_channels)): self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[i], sensor_timestamps, valuesarray[:, i]) # Gyro for i in range(len(gyro_channels)): self.add_sensor_data_to_db(recordset, gyro_sensor, gyro_channels[i], sensor_timestamps, valuesarray[:, i + 6])
def import_beacons_to_database(self, sample_rate, beacons: dict): # DL Oct. 17 2018, New import to database beacons_sensor = self.add_sensor_to_db(SensorType.BEACON, 'Beacons', 'Kontact', 'Environment', sample_rate, 1) channel_values = dict() # Data is already hour-aligned iterate through hours for timestamp in beacons: # print('beacons', timestamp, len(beacons[timestamp]['times']), # len(beacons[timestamp]['values'])) # Create time array as float64 timesarray = np.asarray(beacons[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are int8 valuesarray = np.asarray(beacons[timestamp]['values'], dtype=np.int8) # Iterate through each entry to generate data for each beacon_id for i in range(0, len(timesarray)): name = [ str(format(x, 'x')).rjust(2, '0') for x in beacons[timestamp]['values'][i][0:16] ] beacon_id = ''.join(name[0:10]) + '_' + ''.join(name[10:]) # Create channel if it does not exist if not channel_values.__contains__(beacon_id): channel_values[beacon_id] = [] channel_values[beacon_id].append( (timesarray[i], valuesarray[i][14], valuesarray[i][15])) # Store each beacon_id in separate channels for key in channel_values: timevect = np.asarray([x[0] for x in channel_values[key]], dtype=np.float64) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() # Calculate recordset recordset = self.get_recordset( sensor_timestamps.start_timestamp.timestamp()) # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Create channel channel_txPower = self.add_channel_to_db( beacons_sensor, Units.NONE, DataFormat.SINT8, key + '_TxPower') channel_RSSI = self.add_channel_to_db(beacons_sensor, Units.NONE, DataFormat.SINT8, key + '_RSSI') tx_power_vect = np.asarray([x[1] for x in channel_values[key]], dtype=np.int8) rssi_vect = np.asarray([x[2] for x in channel_values[key]], dtype=np.int8) # Add data self.add_sensor_data_to_db(recordset, beacons_sensor, channel_txPower, sensor_timestamps, tx_power_vect) self.add_sensor_data_to_db(recordset, beacons_sensor, channel_RSSI, sensor_timestamps, rssi_vect)
def import_sensoria_to_database(self, sample_rate, sensoria: dict): # DL Oct. 17 2018, New import to database sensoria_acc_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', 'Sensoria', 'Foot', sample_rate, 1) sensoria_acc_channels = list() sensoria_acc_channels.append( self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X')) sensoria_acc_channels.append( self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y')) sensoria_acc_channels.append( self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Z')) sensoria_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER, 'Gyrometer', 'Sensoria', 'Foot', sample_rate, 1) sensoria_gyro_channels = list() sensoria_gyro_channels.append( self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_X')) sensoria_gyro_channels.append( self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Y')) sensoria_gyro_channels.append( self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Z')) sensoria_mag_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER, 'Magnetometer', 'Sensoria', 'Foot', sample_rate, 1) sensoria_mag_channels = list() sensoria_mag_channels.append( self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS, DataFormat.FLOAT32, 'Mag_X')) sensoria_mag_channels.append( self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS, DataFormat.FLOAT32, 'Mag_Y')) sensoria_mag_channels.append( self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS, DataFormat.FLOAT32, 'Mag_Z')) sensoria_fsr_sensor = self.add_sensor_to_db(SensorType.FSR, 'FSR', 'Sensoria', 'Foot', sample_rate, 1) sensoria_fsr_channels = list() sensoria_fsr_channels.append( self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE, DataFormat.FLOAT32, 'META-1')) sensoria_fsr_channels.append( self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE, DataFormat.FLOAT32, 'META-5')) sensoria_fsr_channels.append( self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE, DataFormat.FLOAT32, 'HEEL')) for timestamp in sensoria: # print('sensoria', timestamp, len(sensoria[timestamp]['times']), # len(sensoria[timestamp]['values'])) # Calculate recordset recordset = self.get_recordset(timestamp.timestamp()) # Create time array as float64 timesarray = np.asarray(sensoria[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are float32 valuesarray = np.asarray(sensoria[timestamp]['values'], dtype=np.float32) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Store FSR for i in range(len(sensoria_fsr_channels)): self.add_sensor_data_to_db(recordset, sensoria_fsr_sensor, sensoria_fsr_channels[i], sensor_timestamps, valuesarray[:, i + 1]) # Store Acc for i in range(len(sensoria_acc_channels)): self.add_sensor_data_to_db(recordset, sensoria_acc_sensor, sensoria_acc_channels[i], sensor_timestamps, valuesarray[:, i + 4]) # Store Gyro for i in range(len(sensoria_gyro_channels)): self.add_sensor_data_to_db(recordset, sensoria_gyro_sensor, sensoria_gyro_channels[i], sensor_timestamps, valuesarray[:, i + 7]) # Magneto for i in range(len(sensoria_mag_channels)): self.add_sensor_data_to_db(recordset, sensoria_mag_sensor, sensoria_mag_channels[i], sensor_timestamps, valuesarray[:, i + 10])
def import_raw_gyro_to_database(self, sample_rate, raw_gyro: dict): # DL Oct. 17 2018, New import to database # Create sensor raw_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER, 'Raw Gyro', 'AppleWatch', 'Wrist', sample_rate, 1) raw_gyro_channels = list() # Create channels raw_gyro_channels.append( self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_X')) raw_gyro_channels.append( self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Y')) raw_gyro_channels.append( self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyro_Z')) # Data is already hour-aligned iterate through hours for timestamp in raw_gyro: # print('raw_gyro', timestamp, len(raw_gyro[timestamp]['times']), # len(raw_gyro[timestamp]['values'])) # Calculate recordset recordset = self.get_recordset(timestamp.timestamp()) # Create time array as float64 timesarray = np.asarray(raw_gyro[timestamp]['times'], dtype=np.float64) if len(timesarray) is 0: print('Empty data, returning') return # Other values are float32 valuesarray = np.asarray(raw_gyro[timestamp]['values'], dtype=np.float32) # raw_accelero contains in this order # gyro(x, y, z) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timesarray sensor_timestamps.update_timestamps() # Update timestamps in recordset # This should not happen, recordset is initialized at the beginning of the hour if sensor_timestamps.start_timestamp < recordset.start_timestamp: recordset.start_timestamp = sensor_timestamps.start_timestamp # This can occur though if sensor_timestamps.end_timestamp > recordset.end_timestamp: recordset.end_timestamp = sensor_timestamps.end_timestamp # Gyro for i in range(len(raw_gyro_channels)): self.add_sensor_data_to_db(recordset, raw_gyro_sensor, raw_gyro_channels[i], sensor_timestamps, valuesarray[:, i])
def test_get_all_sensor_data_with_args(self): manager = DBManager(filename='openimu.db', overwrite=True, echo=False) # Create sensor in DB group = manager.update_group( Group(name='Group Name', description='Group Description')) participant = manager.update_participant( Participant(name='Participant Name', description='Participant Description', group=group)) sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name', 'Hardware Name', 'Wrist', 30.0, 1) sensor2 = manager.add_sensor(SensorType.GYROMETER, 'Sensor Name', 'Hardware Name', 'Wrist', 30.0, 1) channel1 = manager.add_channel(sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X') channel2 = manager.add_channel(sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y') timestamps = SensorTimestamps() timestamps.timestamps = np.zeros(40, dtype=np.float64) # will set start and end timestamps.update_timestamps() recordset = manager.add_recordset(participant, 'My Record', timestamps.start_timestamp, timestamps.end_timestamp) data = np.zeros(40, dtype=np.float32) sensordata = manager.add_sensor_data(recordset, sensor, channel1, timestamps, data) sensordata = manager.add_sensor_data(recordset, sensor, channel2, timestamps, data) manager.commit() # Test with no args, return everything in the recordset sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True) self.assertEqual(len(sensordata_res), 2) for sensor_data in sensordata_res: self.assertEqual(len(sensor_data.data), len(data)) # Test with a valid sensor arg sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, sensor=sensor) self.assertEqual(len(sensordata_res), 2) for sensor_data in sensordata_res: self.assertEqual(len(sensor_data.data), len(data)) # Test with not the right sensor arg sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, sensor=sensor2) self.assertEqual(len(sensordata_res), 0) # Testing with invalid sensor arg sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, sensor=Sensor()) self.assertEqual(len(sensordata_res), 0) # Testing with channel1 sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, channel=channel1) self.assertEqual(len(sensordata_res), 1) for sensor_data in sensordata_res: self.assertEqual(len(sensor_data.data), len(data)) # Testing with channel2 sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, channel=channel2) self.assertEqual(len(sensordata_res), 1) for sensor_data in sensordata_res: self.assertEqual(len(sensor_data.data), len(data)) # Testing with invalid channel sensordata_res = manager.get_all_sensor_data(recordset=recordset, convert=True, channel=Channel()) self.assertEqual(len(sensordata_res), 0) manager.close()
def import_sensor_data_to_database(self, info, sensor_data: dict): if len(sensor_data) == 0: return accelerometer_sensor = None gyroscope_sensor = None magneto_sensor = None temp_sensor = None accelerometer_channels = list() gyroscope_channels = list() magneto_channels = list() temp_channel = None # Create channels for sensors sensor_names = sensor_data[0][1].keys() for sensor in sensor_names: if sensor == 'Accelerometer': # Create sensor accelerometer_sensor = self.add_sensor_to_db( SensorType.ACCELEROMETER, 'Accelerometer', info['Device Type'], 'Other', info['Sample Rate'], 1) # Create channels accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Z')) if sensor == 'Gyroscope': # Create sensor gyroscope_sensor = self.add_sensor_to_db( SensorType.GYROMETER, 'Gyroscope', info['Device Type'], 'Other', info['Sample Rate'], 1) # Create channels gyroscope_channels.append( self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyroscope_X')) gyroscope_channels.append( self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyroscope_Y')) gyroscope_channels.append( self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC, DataFormat.FLOAT32, 'Gyroscope_Z')) if sensor == 'Magnetometer': # Create sensor magneto_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER, 'Magnetometer', info['Device Type'], 'Other', info['Sample Rate'], 1) # Create channels magneto_channels.append( self.add_channel_to_db(magneto_sensor, Units.UTESLA, DataFormat.FLOAT32, 'Magnetometer_X')) magneto_channels.append( self.add_channel_to_db(magneto_sensor, Units.UTESLA, DataFormat.FLOAT32, 'Magnetometer_Y')) magneto_channels.append( self.add_channel_to_db(magneto_sensor, Units.UTESLA, DataFormat.FLOAT32, 'Magnetometer_Z')) if sensor == 'Temperature': # Create sensor temp_sensor = self.add_sensor_to_db(SensorType.TEMPERATURE, 'Temperature', info['Device Type'], 'Other', info['Sample Rate'], 1) temp_channel = self.add_channel_to_db(temp_sensor, Units.CELCIUS, DataFormat.FLOAT32, 'Temperature') # Reshape data vectors data_times = [data[0] for data in sensor_data] acc_data = [] gyro_data = [] mag_data = [] temp_data = [] if 'Accelerometer' in sensor_names: values = [data[1]['Accelerometer'] for data in sensor_data] for index, value in enumerate(values): timevect = np.linspace(data_times[index], data_times[index] + 1, num=len(value), endpoint=False, dtype=np.float64) shaped_values = np.reshape(value, [-1, 3]) acc_data.extend(list(zip(timevect, shaped_values))) if 'Gyroscope' in sensor_names: values = [data[1]['Gyroscope'] for data in sensor_data] for index, value in enumerate(values): timevect = np.linspace(data_times[index], data_times[index] + 1, num=len(value), endpoint=False, dtype=np.float64) shaped_values = np.reshape(value, [-1, 3]) gyro_data.extend(list(zip(timevect, shaped_values))) if 'Magnetometer' in sensor_names: values = [data[1]['Magnetometer'] for data in sensor_data] for index, value in enumerate(values): timevect = np.linspace(data_times[index], data_times[index] + 1, num=len(value), endpoint=False, dtype=np.float64) shaped_values = np.reshape(value, [-1, 3]) mag_data.extend(list(zip(timevect, shaped_values))) if 'Temperature' in sensor_names: values = [data[1]['Temperature'] for data in sensor_data] for index, value in enumerate(values): timevect = np.linspace(data_times[index], data_times[index] + 1, num=len(value), endpoint=False, dtype=np.float64) temp_data.extend(list(zip(timevect, value))) # Find holes in the recording split_indexes = [0] split_indexes.extend([ index + 1 for index, value in enumerate(np.diff(data_times)) if value != 1 ]) split_indexes.append(len(data_times) - 1) # Find chunks of data longer than 1h indexes_to_add = [] for index in range(1, len(split_indexes)): if (split_indexes[index] - split_indexes[index - 1]) > 3600: current_index = split_indexes[index - 1] while current_index < split_indexes[index]: current_index += 3600 if current_index < split_indexes[-1]: indexes_to_add.append(current_index) split_indexes.extend(indexes_to_add) split_indexes.sort() if 'Accelerometer' in sensor_names: # Detect hours transition to prevent recordset from being longer than 1h for record_index in range(1, len(split_indexes)): if record_index == len(split_indexes): next_cut_timestamp = acc_data[-1][0] + 1 else: next_cut_timestamp = data_times[ split_indexes[record_index]] current_timestamp = data_times[split_indexes[record_index - 1]] data_recordset = [ data for data in acc_data if current_timestamp <= data[0] < next_cut_timestamp ] base_timestamp = data_recordset[0][0] session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] recordset = self.get_recordset(base_timestamp, session_name) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = np.asarray( [row[0] for row in data_recordset], dtype=np.float64) sensor_timestamps.update_timestamps() # Update end_timestamp if required if base_timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = base_timestamp for index in range(0, len(accelerometer_channels)): values = np.asarray( [row[1][index] for row in data_recordset], dtype=np.float32) self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[index], sensor_timestamps, values) if 'Gyroscope' in sensor_names: # Detect hours transition to prevent recordset from being longer than 1h for record_index in range(1, len(split_indexes)): if record_index == len(split_indexes): next_cut_timestamp = gyro_data[-1][0] + 1 else: next_cut_timestamp = data_times[ split_indexes[record_index]] current_timestamp = data_times[split_indexes[record_index - 1]] data_recordset = [ data for data in gyro_data if current_timestamp <= data[0] < next_cut_timestamp ] base_timestamp = data_recordset[0][0] session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] recordset = self.get_recordset(base_timestamp, session_name) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = np.asarray( [row[0] for row in data_recordset], dtype=np.float64) sensor_timestamps.update_timestamps() # Update end_timestamp if required if base_timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = base_timestamp for index in range(0, len(gyroscope_channels)): values = np.asarray( [row[1][index] for row in data_recordset], dtype=np.float32) self.add_sensor_data_to_db(recordset, gyroscope_sensor, gyroscope_channels[index], sensor_timestamps, values) next_cut_timestamp += 3600 if 'Magnetometer' in sensor_names: # Detect hours transition to prevent recordset from being longer than 1h for record_index in range(1, len(split_indexes)): if record_index == len(split_indexes): next_cut_timestamp = mag_data[-1][0] + 1 else: next_cut_timestamp = data_times[ split_indexes[record_index]] current_timestamp = data_times[split_indexes[record_index - 1]] data_recordset = [ data for data in mag_data if current_timestamp <= data[0] < next_cut_timestamp ] base_timestamp = data_recordset[0][0] session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] recordset = self.get_recordset(base_timestamp, session_name) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = np.asarray( [row[0] for row in data_recordset], dtype=np.float64) sensor_timestamps.update_timestamps() # Update end_timestamp if required if base_timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = base_timestamp for index in range(0, len(magneto_channels)): values = np.asarray( [row[1][index] for row in data_recordset], dtype=np.float32) self.add_sensor_data_to_db(recordset, magneto_sensor, magneto_channels[index], sensor_timestamps, values) next_cut_timestamp += 3600 if 'Temperature' in sensor_names: # Detect hours transition to prevent recordset from being longer than 1h for record_index in range(1, len(split_indexes)): if record_index == len(split_indexes): next_cut_timestamp = temp_data[-1][0] + 1 else: next_cut_timestamp = data_times[ split_indexes[record_index]] current_timestamp = data_times[split_indexes[record_index - 1]] data_recordset = [ data for data in temp_data if current_timestamp <= data[0] < next_cut_timestamp ] base_timestamp = data_recordset[0][0] session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] recordset = self.get_recordset(base_timestamp, session_name) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = np.asarray( [row[0] for row in data_recordset], dtype=np.float64) sensor_timestamps.update_timestamps() # Update end_timestamp if required if base_timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = base_timestamp values = np.asarray([row[1] for row in data_recordset], dtype=np.float32) self.add_sensor_data_to_db(recordset, temp_sensor, temp_channel, sensor_timestamps, values) next_cut_timestamp += 3600 # Flush DB self.db.flush()
def import_to_database(self, results): [info, data] = results # print(info) # Creating recordset # print(info['Start Date'], info['Last Sample Time']) # start = int(info['Start Date']) # stop = int(info['Last Sample Time']) # start_timestamp = ticksconverter(start) # end_timestamp = ticksconverter(stop) # all_counts = [0, 0, 0] if data.__contains__('activity'): # print('activity found') # Create sensor accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', info['Device Type'], 'Unknown', info['Sample Rate'], 1) accelerometer_channels = list() # Create channels accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y')) accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X')) accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Z')) # Should be 1970, epoch last_timestamp = 0 all_timestamps = [] value_dict = {} # Import data for epoch in data['activity']: # An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z current_timestamp = epoch[0] # print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1)) # Check for consecutive timestamps create_array = current_timestamp != (last_timestamp + 1) # Do not allow more than one hour of consecutive data if create_array is not True: if current_timestamp - all_timestamps[-1] >= 3600: create_array = True # Consecutive timestamps? if create_array is True: all_timestamps.append(current_timestamp) # Create list for all values for this timestamp value_dict[current_timestamp] = [list(), list(), list()] # Get data samples = epoch[1] # Separate write for each channel for index in range(0, len(accelerometer_channels)): # Using last timestamp to append data value_dict[all_timestamps[-1]][index].append(samples[:, index]) # print("samples shape", samples.shape, samples[:, index].shape) # Update timestamp last_timestamp = current_timestamp # Insert into DB as chunks of data # print('should insert records count: ', len(all_timestamps)) # print('should insert data count:', len(value_dict)) counters = [0, 0, 0] for timestamp in all_timestamps: session_name = str(timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] for index in range(0, len(value_dict[timestamp])): # print(index, timestamp, len(value_dict[timestamp][index])) vector = np.concatenate(value_dict[timestamp][index]) # print('inserting values :', len(value_dict[timestamp][index])) # print('vector: ', len(vector), vector.shape, vector.dtype) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() # Create time vector # TODO Share time vector for all accelerometers? timevect = np.linspace(timestamp, timestamp + len(value_dict[timestamp][index]), num=len(vector), endpoint=False, dtype=np.float64) sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() recordset = self.get_recordset(timestamp, session_name) # Update end_timestamp if required if timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = datetime.datetime.fromtimestamp(timestamp) counters[index] += len(vector) if len(vector) > 0: self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[index], sensor_timestamps, vector) # print('total samples inserted:', counters) # print('total timestamps processed:', len(all_timestamps)) # Flush DB self.db.flush() if data.__contains__('battery'): # print('battery found') # Create sensor volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery', info['Device Type'], 'Unknown', 0, 1) # Create channel volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS, DataFormat.FLOAT32, 'Battery') for epoch in data['battery']: timestamp = datetime.datetime.fromtimestamp(epoch[0]) session_name = str(timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] value = np.float32(epoch[1]) recordset = self.get_recordset(epoch[0], session_name) timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() # Update end_timestamp if required if epoch[0] > recordset.end_timestamp.timestamp(): recordset.end_timestamp = timestamp self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel, sensor_timestamps, value) # Flush to DB (ram) self.db.flush() if data.__contains__('lux'): # print('lux found') # Create sensor lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux', info['Device Type'], 'Unknown', 0, 1) # Create channel lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX, DataFormat.FLOAT32, 'Lux') for epoch in data['lux']: timestamp = datetime.datetime.fromtimestamp(epoch[0]) session_name = str(timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] value = np.float32(epoch[1]) recordset = self.get_recordset(epoch[0], session_name) timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() # Update end_timestamp if required if epoch[0] > recordset.end_timestamp.timestamp(): recordset.end_timestamp = timestamp self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel, sensor_timestamps, value) # Flush to DB (ram) self.db.flush() # Write data to file self.db.commit() self.update_progress.emit(100)
def import_activity_to_database(self, info, activity: dict): # print('activity found') # Create sensor accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', info['Device Type'], 'Activity', info['Sample Rate'], 1) accelerometer_channels = list() # Create channels accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Y')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_X')) accelerometer_channels.append( self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G, DataFormat.FLOAT32, 'Accelerometer_Z')) # Should be 1970, epoch last_timestamp = 0 all_timestamps = [] value_dict = {} # Import data for epoch in activity: # An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z current_timestamp = epoch[0] # print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1)) # Check for consecutive timestamps create_array = current_timestamp != (last_timestamp + 1) # Do not allow more than one hour of consecutive data if create_array is not True: if current_timestamp - all_timestamps[-1] >= 3600: create_array = True # Consecutive timestamps? if create_array is True: all_timestamps.append(current_timestamp) # Create list for all values for this timestamp value_dict[current_timestamp] = [list(), list(), list()] # Get data samples = epoch[1] # Separate write for each channel for index in range(0, len(accelerometer_channels)): # Using last timestamp to append data value_dict[all_timestamps[-1]][index].append(samples[:, index]) # print("samples shape", samples.shape, samples[:, index].shape) # Update timestamp last_timestamp = current_timestamp # Insert into DB as chunks of data # print('should insert records count: ', len(all_timestamps)) # print('should insert data count:', len(value_dict)) counters = [0, 0, 0] for timestamp in all_timestamps: session_name = str(timestamp) + '_' + info['Device Type'] + '_' \ + info['Subject Name'] + '_SN:' + info['Serial Number'] for index in range(0, len(value_dict[timestamp])): # print(index, timestamp, len(value_dict[timestamp][index])) vector = np.concatenate(value_dict[timestamp][index]) # print('inserting values :', len(value_dict[timestamp][index])) # print('vector: ', len(vector), vector.shape, vector.dtype) # Create sensor timestamps first sensor_timestamps = SensorTimestamps() # Create time vector # TODO Share time vector for all accelerometers? timevect = np.linspace(timestamp, timestamp + len(value_dict[timestamp][index]), num=len(vector), endpoint=False, dtype=np.float64) sensor_timestamps.timestamps = timevect sensor_timestamps.update_timestamps() recordset = self.get_recordset(timestamp, session_name) # Update end_timestamp if required if timestamp > recordset.end_timestamp.timestamp(): recordset.end_timestamp = datetime.datetime.fromtimestamp( timestamp) counters[index] += len(vector) if len(vector) > 0: self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[index], sensor_timestamps, vector) # print('total samples inserted:', counters) # print('total timestamps processed:', len(all_timestamps)) # Flush DB self.db.flush()