예제 #1
0
    def test_add_sensor_data(self):
        manager = DBManager(filename='openimu.db', overwrite=True)

        # Create sensor in DB
        group = manager.update_group(
            Group(name='Group Name', description='Group Description'))
        participant = manager.update_participant(
            Participant(name='Participant Name',
                        description='Participant Description',
                        group=group))

        sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name',
                                    'Hardware Name', 'Wrist', 30.0, 1)
        channel = manager.add_channel(sensor, Units.GRAVITY_G,
                                      DataFormat.FLOAT32, 'Accelerometer_X')

        timestamps = SensorTimestamps()
        timestamps.timestamps = np.zeros(40, dtype=np.float64)
        # will set start and end
        timestamps.update_timestamps()

        recordset = manager.add_recordset(participant, 'My Record',
                                          timestamps.start_timestamp,
                                          timestamps.end_timestamp)

        data = np.zeros(40, dtype=np.float32)

        sensordata = manager.add_sensor_data(recordset, sensor, channel,
                                             timestamps, data)
        manager.commit()

        sensordata2 = manager.get_sensor_data(sensordata.id_sensor_data)
        self.assertEqual(sensordata, sensordata2)

        manager.close()
예제 #2
0
    def import_power_to_database(self, timestamp, sensors, channels, recordset,
                                 data: dict):

        print('import_imu_to_database')
        values = np.array(data['values'], dtype=np.float32)

        if len(values) == 0:
            return False

        # Create sensor timestamps first
        sensor_timestamps = SensorTimestamps()
        sensor_timestamps.timestamps = data['times']
        sensor_timestamps.update_timestamps()

        self.add_sensor_data_to_db(recordset, sensors['battery'],
                                   channels['battery'], sensor_timestamps,
                                   values[:, 0])

        self.add_sensor_data_to_db(recordset, sensors['current'],
                                   channels['current'], sensor_timestamps,
                                   values[:, 1])

        self.db.commit()

        return True
예제 #3
0
    def import_gps_to_database(self, timestamp, sensors, channels, recordset,
                               data: dict):

        print('import_imu_to_database')
        values = np.array(data['values'], dtype=np.float32)

        if len(values) == 0:
            return False

        # Regenerate GPS data to be stored in the DB as SIRF data
        # TODO Better GPS solution?

        # We have one sample per second of GPS data
        for i, val in enumerate(values):
            geo = GPSGeodetic()
            # Discard invalid data
            if math.isnan(val[1]) or math.isnan(val[2]):
                continue

            geo.latitude = val[1] * 1e7
            geo.longitude = val[2] * 1e7
            # altitude = val[3] * 1e7

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = data['times'][i:i + 1]
            sensor_timestamps.update_timestamps()

            self.add_sensor_data_to_db(recordset, sensors['gps'],
                                       channels['gps'], sensor_timestamps, geo)

        # Commit to file
        self.db.commit()

        return True
예제 #4
0
    def import_coordinates_to_database(self, sample_rate, coordinates: dict):
        # DL Oct. 17 2018, New import to database
        coordinates_sensor = self.add_sensor_to_db(SensorType.GPS,
                                                   'Coordinates', 'AppleWatch',
                                                   'Wrist', sample_rate, 1)
        coordinates_channel = self.add_channel_to_db(coordinates_sensor,
                                                     Units.NONE,
                                                     DataFormat.UINT8,
                                                     'Coordinates')

        for timestamp in coordinates:
            # print('coordinates', timestamp, len(coordinates[timestamp]['times']),
            #      len(coordinates[timestamp]['values']))

            # Create time array as float64
            timesarray = np.asarray(coordinates[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                self.last_error = "Aucune données temporelles."
                return

            # Other values are float32
            valuesarray = np.asarray(coordinates[timestamp]['values'],
                                     dtype=np.float32)

            # Create one entry per timestamp ?
            # Could we store a vector instead ?
            for i, value in enumerate(valuesarray):
                # Build gps data
                geo = GPSGeodetic()
                geo.latitude = value[0] * 1e7
                geo.longitude = value[1] * 1e7

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = timesarray[i:i + 1]
                sensor_timestamps.update_timestamps()

                # Calculate recordset
                recordset = self.get_recordset(
                    sensor_timestamps.start_timestamp.timestamp(),
                    session_name=self.session_name)

                # Update timestamps in recordset
                # This should not happen, recordset is initialized at the beginning of the hour
                if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                    recordset.start_timestamp = sensor_timestamps.start_timestamp
                # This can occur though
                if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                    recordset.end_timestamp = sensor_timestamps.end_timestamp

                # Store
                self.add_sensor_data_to_db(recordset, coordinates_sensor,
                                           coordinates_channel,
                                           sensor_timestamps, geo)
                self.update_progress.emit(50 + np.floor(i / len(valuesarray) /
                                                        2 * 100))
예제 #5
0
    def create_sensor_timestamps(times, recordset):
        # Create sensor timestamps first
        sensor_timestamps = SensorTimestamps()
        sensor_timestamps.timestamps = times
        sensor_timestamps.update_timestamps()

        # Update timestamps in recordset
        # This should not happen, recordset is initialized at the beginning of the hour
        if sensor_timestamps.start_timestamp < recordset.start_timestamp:
            recordset.start_timestamp = sensor_timestamps.start_timestamp
        # This can occur though
        if sensor_timestamps.end_timestamp > recordset.end_timestamp:
            recordset.end_timestamp = sensor_timestamps.end_timestamp

        return sensor_timestamps
예제 #6
0
    def import_battery_to_database(self, sampling_rate, battery: dict):
        # DL Oct. 16 2018, New import to database
        battery_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery',
                                               'AppleWatch', 'Wrist',
                                               sampling_rate, 1)

        battery_channel = self.add_channel_to_db(battery_sensor, Units.VOLTS,
                                                 DataFormat.FLOAT32,
                                                 'Battery Percentage')

        # Data is already hour-aligned iterate through hours
        for timestamp in battery:
            # print('battery', timestamp, len(battery[timestamp]['times']),
            #      len(battery[timestamp]['values']))

            # Calculate recordset
            recordset = self.get_recordset(timestamp.timestamp())

            # Import to database
            # Create time array as float64
            timesarray = np.asarray(battery[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are float32
            valuesarray = np.asarray(battery[timestamp]['values'],
                                     dtype=np.float32)

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timesarray
            sensor_timestamps.update_timestamps()

            # Update timestamps in recordset
            # This should not happen, recordset is initialized at the beginning of the hour
            if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                recordset.start_timestamp = sensor_timestamps.start_timestamp
            # This can occur though
            if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                recordset.end_timestamp = sensor_timestamps.end_timestamp

            self.add_sensor_data_to_db(recordset, battery_sensor,
                                       battery_channel, sensor_timestamps,
                                       valuesarray[:, 0])
예제 #7
0
    def import_heartrate_to_database(self, sample_rate, heartrate: dict):
        # DL Oct. 17 2018, New import to database
        heartrate_sensor = self.add_sensor_to_db(SensorType.HEARTRATE,
                                                 'Heartrate', 'AppleWatch',
                                                 'Wrist', sample_rate, 1)

        heartrate_channel = self.add_channel_to_db(heartrate_sensor, Units.BPM,
                                                   DataFormat.UINT8,
                                                   'Heartrate')

        for timestamp in heartrate:
            # print('heartrate', timestamp, len(heartrate[timestamp]['times']),
            #       len(heartrate[timestamp]['values']))

            # Calculate recordset
            recordset = self.get_recordset(timestamp.timestamp())

            # Create time array as float64
            timesarray = np.asarray(heartrate[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are float32
            valuesarray = np.asarray(heartrate[timestamp]['values'],
                                     dtype=np.uint8)

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timesarray
            sensor_timestamps.update_timestamps()

            # Update timestamps in recordset
            # This should not happen, recordset is initialized at the beginning of the hour
            if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                recordset.start_timestamp = sensor_timestamps.start_timestamp
            # This can occur though
            if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                recordset.end_timestamp = sensor_timestamps.end_timestamp

            # Store data
            self.add_sensor_data_to_db(recordset, heartrate_sensor,
                                       heartrate_channel, sensor_timestamps,
                                       valuesarray[:, 0])
예제 #8
0
    def import_baro_to_database(self,  timestamp, sensors, channels, recordset, data: list):
        # print('import_imu_to_database')
        values = np.array(data['values'], dtype=np.float32)

        if len(values) == 0:
            return False

        # Create sensor timestamps first
        sensor_timestamps = SensorTimestamps()
        sensor_timestamps.timestamps = data['times']
        sensor_timestamps.update_timestamps()

        self.add_sensor_data_to_db(recordset, sensors['baro'], channels['baro'],
                                   sensor_timestamps, values[:, 1])

        # Commit to file
        self.db.commit()

        return True
예제 #9
0
    def import_imu_to_database(self, timestamp, sample_rate, sensors, channels,
                               recordset, data: dict):
        print('import_imu_to_database')
        values = np.array(data['values'], dtype=np.float32)

        if len(values) == 0:
            return False

        # print("Values shape: ", values.shape)
        end_timestamp = np.floor(data['end_time'])

        # Update end_timestamp if required
        if end_timestamp > recordset.end_timestamp.timestamp():
            recordset.end_timestamp = datetime.datetime.fromtimestamp(
                end_timestamp)

        # Create sensor timestamps first
        sensor_timestamps = SensorTimestamps()
        sensor_timestamps.timestamps = data['times']
        sensor_timestamps.update_timestamps()

        # Acc
        for i in range(len(channels['acc'])):
            self.add_sensor_data_to_db(recordset, sensors['acc'],
                                       channels['acc'][i], sensor_timestamps,
                                       values[:, i])

        # Gyro
        for i in range(len(channels['gyro'])):
            self.add_sensor_data_to_db(recordset, sensors['gyro'],
                                       channels['gyro'][i], sensor_timestamps,
                                       values[:, i + 3])

        # Magnetometer
        for i in range(len(channels['mag'])):
            self.add_sensor_data_to_db(recordset, sensors['mag'],
                                       channels['mag'][i], sensor_timestamps,
                                       values[:, i + 6])

        self.db.commit()

        return True
예제 #10
0
    def import_battery_to_database(self, info, battery: dict):
        # Create sensor
        volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery',
                                            info['Device Type'], 'General', 10,
                                            1)

        # Create channel
        volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS,
                                              DataFormat.FLOAT32, 'Battery')

        for epoch in battery:
            timestamp = datetime.datetime.fromtimestamp(epoch[0])
            session_name = str(timestamp) + '_' + info['Device Type'] + '_' + info['Subject Name'] \
                                          + '_SN:' + info['Serial Number']
            value = np.float32(epoch[1])

            recordset = self.get_recordset(epoch[0], session_name)

            timevect = np.linspace(epoch[0],
                                   epoch[0] + 1,
                                   num=1,
                                   endpoint=False,
                                   dtype=np.float64)
            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timevect
            sensor_timestamps.update_timestamps()

            # Update end_timestamp if required
            if epoch[0] > recordset.end_timestamp.timestamp():
                recordset.end_timestamp = timestamp

            self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel,
                                       sensor_timestamps, value)

        # Flush to DB (ram)
        self.db.flush()
예제 #11
0
    def import_lux_to_database(self, info, lux: dict):
        # Create sensor
        lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux',
                                           info['Device Type'], 'Unknown', 0,
                                           1)

        # Create channel
        lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX,
                                             DataFormat.FLOAT32, 'Lux')

        for epoch in lux:
            timestamp = datetime.datetime.fromtimestamp(epoch[0])
            session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
                           + info['Subject Name'] + '_SN:' + info['Serial Number']
            value = np.float32(epoch[1])

            recordset = self.get_recordset(epoch[0], session_name)

            timevect = np.linspace(epoch[0],
                                   epoch[0] + 1,
                                   num=1,
                                   endpoint=False,
                                   dtype=np.float64)
            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timevect
            sensor_timestamps.update_timestamps()

            # Update end_timestamp if required
            if epoch[0] > recordset.end_timestamp.timestamp():
                recordset.end_timestamp = timestamp

            self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel,
                                       sensor_timestamps, value)

        # Flush to DB (ram)
        self.db.flush()
예제 #12
0
    def import_motion_to_database(self, sampling_rate, motion: dict):
        # DL Oct. 16 2018, New import to database

        # Create channels and sensors
        accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
                                                     'Accelerometer',
                                                     'AppleWatch', 'Wrist',
                                                     sampling_rate, 1)

        accelerometer_channels = list()

        # Create channels
        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_X'))

        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Y'))

        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Z'))

        # Create sensor
        gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER, 'Gyro',
                                            'AppleWatch', 'Wrist',
                                            sampling_rate, 1)

        gyro_channels = list()

        # Create channels
        gyro_channels.append(
            self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_X'))

        gyro_channels.append(
            self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Y'))

        gyro_channels.append(
            self.add_channel_to_db(gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Z'))

        # Data is already hour-aligned iterate through hours
        for timestamp in motion:
            # print('motion', timestamp, len(motion[timestamp]['times']),
            #     len(motion[timestamp]['values']))

            # Calculate recordset
            recordset = self.get_recordset(timestamp.timestamp())

            # Add motion data to database

            # Create time array as float64
            timesarray = np.asarray(motion[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are float32
            valuesarray = np.asarray(motion[timestamp]['values'],
                                     dtype=np.float32)

            # Motion contains in this order
            # acceleration(x, y, z)
            # gravity vector(x, y, z)
            # gyroscope(x, y, z)
            # attitude quaternion(w, x, y, z)

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timesarray
            sensor_timestamps.update_timestamps()

            # Update timestamps in recordset
            # This should not happen, recordset is initialized at the beginning of the hour
            if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                recordset.start_timestamp = sensor_timestamps.start_timestamp
            # This can occur though
            if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                recordset.end_timestamp = sensor_timestamps.end_timestamp

            # Acc
            for i in range(len(accelerometer_channels)):
                self.add_sensor_data_to_db(recordset, accelerometer_sensor,
                                           accelerometer_channels[i],
                                           sensor_timestamps, valuesarray[:,
                                                                          i])

            # Gyro
            for i in range(len(gyro_channels)):
                self.add_sensor_data_to_db(recordset, gyro_sensor,
                                           gyro_channels[i], sensor_timestamps,
                                           valuesarray[:, i + 6])
예제 #13
0
    def import_beacons_to_database(self, sample_rate, beacons: dict):
        # DL Oct. 17 2018, New import to database
        beacons_sensor = self.add_sensor_to_db(SensorType.BEACON, 'Beacons',
                                               'Kontact', 'Environment',
                                               sample_rate, 1)
        channel_values = dict()

        # Data is already hour-aligned iterate through hours
        for timestamp in beacons:
            # print('beacons', timestamp, len(beacons[timestamp]['times']),
            #      len(beacons[timestamp]['values']))

            # Create time array as float64
            timesarray = np.asarray(beacons[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are int8
            valuesarray = np.asarray(beacons[timestamp]['values'],
                                     dtype=np.int8)

            # Iterate through each entry to generate data for each beacon_id
            for i in range(0, len(timesarray)):
                name = [
                    str(format(x, 'x')).rjust(2, '0')
                    for x in beacons[timestamp]['values'][i][0:16]
                ]
                beacon_id = ''.join(name[0:10]) + '_' + ''.join(name[10:])

                # Create channel if it does not exist
                if not channel_values.__contains__(beacon_id):
                    channel_values[beacon_id] = []

                channel_values[beacon_id].append(
                    (timesarray[i], valuesarray[i][14], valuesarray[i][15]))

            # Store each beacon_id in separate channels
            for key in channel_values:
                timevect = np.asarray([x[0] for x in channel_values[key]],
                                      dtype=np.float64)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = timevect
                sensor_timestamps.update_timestamps()

                # Calculate recordset
                recordset = self.get_recordset(
                    sensor_timestamps.start_timestamp.timestamp())

                # Update timestamps in recordset
                # This should not happen, recordset is initialized at the beginning of the hour
                if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                    recordset.start_timestamp = sensor_timestamps.start_timestamp
                # This can occur though
                if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                    recordset.end_timestamp = sensor_timestamps.end_timestamp

                # Create channel
                channel_txPower = self.add_channel_to_db(
                    beacons_sensor, Units.NONE, DataFormat.SINT8,
                    key + '_TxPower')

                channel_RSSI = self.add_channel_to_db(beacons_sensor,
                                                      Units.NONE,
                                                      DataFormat.SINT8,
                                                      key + '_RSSI')

                tx_power_vect = np.asarray([x[1] for x in channel_values[key]],
                                           dtype=np.int8)
                rssi_vect = np.asarray([x[2] for x in channel_values[key]],
                                       dtype=np.int8)

                # Add data
                self.add_sensor_data_to_db(recordset, beacons_sensor,
                                           channel_txPower, sensor_timestamps,
                                           tx_power_vect)

                self.add_sensor_data_to_db(recordset, beacons_sensor,
                                           channel_RSSI, sensor_timestamps,
                                           rssi_vect)
예제 #14
0
    def import_sensoria_to_database(self, sample_rate, sensoria: dict):
        # DL Oct. 17 2018, New import to database
        sensoria_acc_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
                                                    'Accelerometer',
                                                    'Sensoria', 'Foot',
                                                    sample_rate, 1)

        sensoria_acc_channels = list()
        sensoria_acc_channels.append(
            self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_X'))
        sensoria_acc_channels.append(
            self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Y'))
        sensoria_acc_channels.append(
            self.add_channel_to_db(sensoria_acc_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Z'))

        sensoria_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER,
                                                     'Gyrometer', 'Sensoria',
                                                     'Foot', sample_rate, 1)
        sensoria_gyro_channels = list()
        sensoria_gyro_channels.append(
            self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_X'))
        sensoria_gyro_channels.append(
            self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Y'))
        sensoria_gyro_channels.append(
            self.add_channel_to_db(sensoria_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Z'))

        sensoria_mag_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER,
                                                    'Magnetometer', 'Sensoria',
                                                    'Foot', sample_rate, 1)
        sensoria_mag_channels = list()
        sensoria_mag_channels.append(
            self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
                                   DataFormat.FLOAT32, 'Mag_X'))
        sensoria_mag_channels.append(
            self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
                                   DataFormat.FLOAT32, 'Mag_Y'))
        sensoria_mag_channels.append(
            self.add_channel_to_db(sensoria_mag_sensor, Units.GAUSS,
                                   DataFormat.FLOAT32, 'Mag_Z'))

        sensoria_fsr_sensor = self.add_sensor_to_db(SensorType.FSR, 'FSR',
                                                    'Sensoria', 'Foot',
                                                    sample_rate, 1)
        sensoria_fsr_channels = list()
        sensoria_fsr_channels.append(
            self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
                                   DataFormat.FLOAT32, 'META-1'))
        sensoria_fsr_channels.append(
            self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
                                   DataFormat.FLOAT32, 'META-5'))
        sensoria_fsr_channels.append(
            self.add_channel_to_db(sensoria_fsr_sensor, Units.NONE,
                                   DataFormat.FLOAT32, 'HEEL'))
        for timestamp in sensoria:
            # print('sensoria', timestamp, len(sensoria[timestamp]['times']),
            #      len(sensoria[timestamp]['values']))

            # Calculate recordset
            recordset = self.get_recordset(timestamp.timestamp())

            # Create time array as float64
            timesarray = np.asarray(sensoria[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are float32
            valuesarray = np.asarray(sensoria[timestamp]['values'],
                                     dtype=np.float32)

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timesarray
            sensor_timestamps.update_timestamps()

            # Update timestamps in recordset
            # This should not happen, recordset is initialized at the beginning of the hour
            if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                recordset.start_timestamp = sensor_timestamps.start_timestamp
            # This can occur though
            if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                recordset.end_timestamp = sensor_timestamps.end_timestamp

            # Store FSR
            for i in range(len(sensoria_fsr_channels)):
                self.add_sensor_data_to_db(recordset, sensoria_fsr_sensor,
                                           sensoria_fsr_channels[i],
                                           sensor_timestamps,
                                           valuesarray[:, i + 1])

            # Store Acc
            for i in range(len(sensoria_acc_channels)):
                self.add_sensor_data_to_db(recordset, sensoria_acc_sensor,
                                           sensoria_acc_channels[i],
                                           sensor_timestamps,
                                           valuesarray[:, i + 4])

            # Store Gyro
            for i in range(len(sensoria_gyro_channels)):
                self.add_sensor_data_to_db(recordset, sensoria_gyro_sensor,
                                           sensoria_gyro_channels[i],
                                           sensor_timestamps,
                                           valuesarray[:, i + 7])

            # Magneto
            for i in range(len(sensoria_mag_channels)):
                self.add_sensor_data_to_db(recordset, sensoria_mag_sensor,
                                           sensoria_mag_channels[i],
                                           sensor_timestamps,
                                           valuesarray[:, i + 10])
예제 #15
0
    def import_raw_gyro_to_database(self, sample_rate, raw_gyro: dict):
        # DL Oct. 17 2018, New import to database
        # Create sensor
        raw_gyro_sensor = self.add_sensor_to_db(SensorType.GYROMETER,
                                                'Raw Gyro', 'AppleWatch',
                                                'Wrist', sample_rate, 1)

        raw_gyro_channels = list()

        # Create channels
        raw_gyro_channels.append(
            self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_X'))

        raw_gyro_channels.append(
            self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Y'))

        raw_gyro_channels.append(
            self.add_channel_to_db(raw_gyro_sensor, Units.DEG_PER_SEC,
                                   DataFormat.FLOAT32, 'Gyro_Z'))

        # Data is already hour-aligned iterate through hours
        for timestamp in raw_gyro:
            # print('raw_gyro', timestamp, len(raw_gyro[timestamp]['times']),
            #       len(raw_gyro[timestamp]['values']))

            # Calculate recordset
            recordset = self.get_recordset(timestamp.timestamp())

            # Create time array as float64
            timesarray = np.asarray(raw_gyro[timestamp]['times'],
                                    dtype=np.float64)

            if len(timesarray) is 0:
                print('Empty data, returning')
                return

            # Other values are float32
            valuesarray = np.asarray(raw_gyro[timestamp]['values'],
                                     dtype=np.float32)

            # raw_accelero contains in this order
            # gyro(x, y, z)

            # Create sensor timestamps first
            sensor_timestamps = SensorTimestamps()
            sensor_timestamps.timestamps = timesarray
            sensor_timestamps.update_timestamps()

            # Update timestamps in recordset
            # This should not happen, recordset is initialized at the beginning of the hour
            if sensor_timestamps.start_timestamp < recordset.start_timestamp:
                recordset.start_timestamp = sensor_timestamps.start_timestamp
            # This can occur though
            if sensor_timestamps.end_timestamp > recordset.end_timestamp:
                recordset.end_timestamp = sensor_timestamps.end_timestamp

            # Gyro
            for i in range(len(raw_gyro_channels)):
                self.add_sensor_data_to_db(recordset, raw_gyro_sensor,
                                           raw_gyro_channels[i],
                                           sensor_timestamps, valuesarray[:,
                                                                          i])
예제 #16
0
    def test_get_all_sensor_data_with_args(self):
        manager = DBManager(filename='openimu.db', overwrite=True, echo=False)

        # Create sensor in DB
        group = manager.update_group(
            Group(name='Group Name', description='Group Description'))
        participant = manager.update_participant(
            Participant(name='Participant Name',
                        description='Participant Description',
                        group=group))
        sensor = manager.add_sensor(SensorType.ACCELEROMETER, 'Sensor Name',
                                    'Hardware Name', 'Wrist', 30.0, 1)
        sensor2 = manager.add_sensor(SensorType.GYROMETER, 'Sensor Name',
                                     'Hardware Name', 'Wrist', 30.0, 1)
        channel1 = manager.add_channel(sensor, Units.GRAVITY_G,
                                       DataFormat.FLOAT32, 'Accelerometer_X')
        channel2 = manager.add_channel(sensor, Units.GRAVITY_G,
                                       DataFormat.FLOAT32, 'Accelerometer_Y')

        timestamps = SensorTimestamps()
        timestamps.timestamps = np.zeros(40, dtype=np.float64)
        # will set start and end
        timestamps.update_timestamps()

        recordset = manager.add_recordset(participant, 'My Record',
                                          timestamps.start_timestamp,
                                          timestamps.end_timestamp)

        data = np.zeros(40, dtype=np.float32)
        sensordata = manager.add_sensor_data(recordset, sensor, channel1,
                                             timestamps, data)
        sensordata = manager.add_sensor_data(recordset, sensor, channel2,
                                             timestamps, data)
        manager.commit()

        # Test with no args, return everything in the recordset
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True)
        self.assertEqual(len(sensordata_res), 2)
        for sensor_data in sensordata_res:
            self.assertEqual(len(sensor_data.data), len(data))

        # Test with a valid sensor arg
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     sensor=sensor)
        self.assertEqual(len(sensordata_res), 2)
        for sensor_data in sensordata_res:
            self.assertEqual(len(sensor_data.data), len(data))

        # Test with not the right sensor arg
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     sensor=sensor2)
        self.assertEqual(len(sensordata_res), 0)

        # Testing with invalid sensor arg
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     sensor=Sensor())
        self.assertEqual(len(sensordata_res), 0)

        # Testing with channel1
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     channel=channel1)
        self.assertEqual(len(sensordata_res), 1)
        for sensor_data in sensordata_res:
            self.assertEqual(len(sensor_data.data), len(data))

        # Testing with channel2
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     channel=channel2)
        self.assertEqual(len(sensordata_res), 1)
        for sensor_data in sensordata_res:
            self.assertEqual(len(sensor_data.data), len(data))

        # Testing with invalid channel
        sensordata_res = manager.get_all_sensor_data(recordset=recordset,
                                                     convert=True,
                                                     channel=Channel())
        self.assertEqual(len(sensordata_res), 0)

        manager.close()
예제 #17
0
    def import_sensor_data_to_database(self, info, sensor_data: dict):
        if len(sensor_data) == 0:
            return

        accelerometer_sensor = None
        gyroscope_sensor = None
        magneto_sensor = None
        temp_sensor = None

        accelerometer_channels = list()
        gyroscope_channels = list()
        magneto_channels = list()
        temp_channel = None

        # Create channels for sensors
        sensor_names = sensor_data[0][1].keys()
        for sensor in sensor_names:
            if sensor == 'Accelerometer':
                # Create sensor
                accelerometer_sensor = self.add_sensor_to_db(
                    SensorType.ACCELEROMETER, 'Accelerometer',
                    info['Device Type'], 'Other', info['Sample Rate'], 1)

                # Create channels
                accelerometer_channels.append(
                    self.add_channel_to_db(accelerometer_sensor,
                                           Units.GRAVITY_G, DataFormat.FLOAT32,
                                           'Accelerometer_X'))

                accelerometer_channels.append(
                    self.add_channel_to_db(accelerometer_sensor,
                                           Units.GRAVITY_G, DataFormat.FLOAT32,
                                           'Accelerometer_Y'))

                accelerometer_channels.append(
                    self.add_channel_to_db(accelerometer_sensor,
                                           Units.GRAVITY_G, DataFormat.FLOAT32,
                                           'Accelerometer_Z'))
            if sensor == 'Gyroscope':
                # Create sensor
                gyroscope_sensor = self.add_sensor_to_db(
                    SensorType.GYROMETER, 'Gyroscope', info['Device Type'],
                    'Other', info['Sample Rate'], 1)

                # Create channels
                gyroscope_channels.append(
                    self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
                                           DataFormat.FLOAT32, 'Gyroscope_X'))

                gyroscope_channels.append(
                    self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
                                           DataFormat.FLOAT32, 'Gyroscope_Y'))

                gyroscope_channels.append(
                    self.add_channel_to_db(gyroscope_sensor, Units.DEG_PER_SEC,
                                           DataFormat.FLOAT32, 'Gyroscope_Z'))
            if sensor == 'Magnetometer':
                # Create sensor
                magneto_sensor = self.add_sensor_to_db(SensorType.MAGNETOMETER,
                                                       'Magnetometer',
                                                       info['Device Type'],
                                                       'Other',
                                                       info['Sample Rate'], 1)

                # Create channels
                magneto_channels.append(
                    self.add_channel_to_db(magneto_sensor, Units.UTESLA,
                                           DataFormat.FLOAT32,
                                           'Magnetometer_X'))

                magneto_channels.append(
                    self.add_channel_to_db(magneto_sensor, Units.UTESLA,
                                           DataFormat.FLOAT32,
                                           'Magnetometer_Y'))

                magneto_channels.append(
                    self.add_channel_to_db(magneto_sensor, Units.UTESLA,
                                           DataFormat.FLOAT32,
                                           'Magnetometer_Z'))
            if sensor == 'Temperature':
                # Create sensor
                temp_sensor = self.add_sensor_to_db(SensorType.TEMPERATURE,
                                                    'Temperature',
                                                    info['Device Type'],
                                                    'Other',
                                                    info['Sample Rate'], 1)

                temp_channel = self.add_channel_to_db(temp_sensor,
                                                      Units.CELCIUS,
                                                      DataFormat.FLOAT32,
                                                      'Temperature')

        # Reshape data vectors
        data_times = [data[0] for data in sensor_data]

        acc_data = []
        gyro_data = []
        mag_data = []
        temp_data = []
        if 'Accelerometer' in sensor_names:
            values = [data[1]['Accelerometer'] for data in sensor_data]

            for index, value in enumerate(values):
                timevect = np.linspace(data_times[index],
                                       data_times[index] + 1,
                                       num=len(value),
                                       endpoint=False,
                                       dtype=np.float64)
                shaped_values = np.reshape(value, [-1, 3])
                acc_data.extend(list(zip(timevect, shaped_values)))

        if 'Gyroscope' in sensor_names:
            values = [data[1]['Gyroscope'] for data in sensor_data]

            for index, value in enumerate(values):
                timevect = np.linspace(data_times[index],
                                       data_times[index] + 1,
                                       num=len(value),
                                       endpoint=False,
                                       dtype=np.float64)
                shaped_values = np.reshape(value, [-1, 3])
                gyro_data.extend(list(zip(timevect, shaped_values)))

        if 'Magnetometer' in sensor_names:
            values = [data[1]['Magnetometer'] for data in sensor_data]

            for index, value in enumerate(values):
                timevect = np.linspace(data_times[index],
                                       data_times[index] + 1,
                                       num=len(value),
                                       endpoint=False,
                                       dtype=np.float64)
                shaped_values = np.reshape(value, [-1, 3])
                mag_data.extend(list(zip(timevect, shaped_values)))

        if 'Temperature' in sensor_names:
            values = [data[1]['Temperature'] for data in sensor_data]

            for index, value in enumerate(values):
                timevect = np.linspace(data_times[index],
                                       data_times[index] + 1,
                                       num=len(value),
                                       endpoint=False,
                                       dtype=np.float64)
                temp_data.extend(list(zip(timevect, value)))

        # Find holes in the recording
        split_indexes = [0]
        split_indexes.extend([
            index + 1 for index, value in enumerate(np.diff(data_times))
            if value != 1
        ])
        split_indexes.append(len(data_times) - 1)

        # Find chunks of data longer than 1h
        indexes_to_add = []
        for index in range(1, len(split_indexes)):
            if (split_indexes[index] - split_indexes[index - 1]) > 3600:
                current_index = split_indexes[index - 1]
                while current_index < split_indexes[index]:
                    current_index += 3600
                    if current_index < split_indexes[-1]:
                        indexes_to_add.append(current_index)

        split_indexes.extend(indexes_to_add)
        split_indexes.sort()

        if 'Accelerometer' in sensor_names:
            # Detect hours transition to prevent recordset from being longer than 1h
            for record_index in range(1, len(split_indexes)):
                if record_index == len(split_indexes):
                    next_cut_timestamp = acc_data[-1][0] + 1
                else:
                    next_cut_timestamp = data_times[
                        split_indexes[record_index]]
                current_timestamp = data_times[split_indexes[record_index - 1]]
                data_recordset = [
                    data for data in acc_data
                    if current_timestamp <= data[0] < next_cut_timestamp
                ]

                base_timestamp = data_recordset[0][0]
                session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
                                                   + info['Subject Name'] + '_SN:' + info['Serial Number']

                recordset = self.get_recordset(base_timestamp, session_name)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = np.asarray(
                    [row[0] for row in data_recordset], dtype=np.float64)
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if base_timestamp > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = base_timestamp

                for index in range(0, len(accelerometer_channels)):
                    values = np.asarray(
                        [row[1][index] for row in data_recordset],
                        dtype=np.float32)
                    self.add_sensor_data_to_db(recordset, accelerometer_sensor,
                                               accelerometer_channels[index],
                                               sensor_timestamps, values)

        if 'Gyroscope' in sensor_names:
            # Detect hours transition to prevent recordset from being longer than 1h
            for record_index in range(1, len(split_indexes)):
                if record_index == len(split_indexes):
                    next_cut_timestamp = gyro_data[-1][0] + 1
                else:
                    next_cut_timestamp = data_times[
                        split_indexes[record_index]]
                current_timestamp = data_times[split_indexes[record_index - 1]]
                data_recordset = [
                    data for data in gyro_data
                    if current_timestamp <= data[0] < next_cut_timestamp
                ]

                base_timestamp = data_recordset[0][0]
                session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']

                recordset = self.get_recordset(base_timestamp, session_name)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = np.asarray(
                    [row[0] for row in data_recordset], dtype=np.float64)
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if base_timestamp > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = base_timestamp

                for index in range(0, len(gyroscope_channels)):
                    values = np.asarray(
                        [row[1][index] for row in data_recordset],
                        dtype=np.float32)
                    self.add_sensor_data_to_db(recordset, gyroscope_sensor,
                                               gyroscope_channels[index],
                                               sensor_timestamps, values)

                next_cut_timestamp += 3600

        if 'Magnetometer' in sensor_names:
            # Detect hours transition to prevent recordset from being longer than 1h
            for record_index in range(1, len(split_indexes)):
                if record_index == len(split_indexes):
                    next_cut_timestamp = mag_data[-1][0] + 1
                else:
                    next_cut_timestamp = data_times[
                        split_indexes[record_index]]
                current_timestamp = data_times[split_indexes[record_index - 1]]
                data_recordset = [
                    data for data in mag_data
                    if current_timestamp <= data[0] < next_cut_timestamp
                ]

                base_timestamp = data_recordset[0][0]
                session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']

                recordset = self.get_recordset(base_timestamp, session_name)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = np.asarray(
                    [row[0] for row in data_recordset], dtype=np.float64)
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if base_timestamp > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = base_timestamp

                for index in range(0, len(magneto_channels)):
                    values = np.asarray(
                        [row[1][index] for row in data_recordset],
                        dtype=np.float32)
                    self.add_sensor_data_to_db(recordset, magneto_sensor,
                                               magneto_channels[index],
                                               sensor_timestamps, values)

                next_cut_timestamp += 3600

        if 'Temperature' in sensor_names:
            # Detect hours transition to prevent recordset from being longer than 1h
            for record_index in range(1, len(split_indexes)):
                if record_index == len(split_indexes):
                    next_cut_timestamp = temp_data[-1][0] + 1
                else:
                    next_cut_timestamp = data_times[
                        split_indexes[record_index]]
                current_timestamp = data_times[split_indexes[record_index - 1]]
                data_recordset = [
                    data for data in temp_data
                    if current_timestamp <= data[0] < next_cut_timestamp
                ]
                base_timestamp = data_recordset[0][0]
                session_name = str(base_timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']

                recordset = self.get_recordset(base_timestamp, session_name)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = np.asarray(
                    [row[0] for row in data_recordset], dtype=np.float64)
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if base_timestamp > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = base_timestamp

                values = np.asarray([row[1] for row in data_recordset],
                                    dtype=np.float32)
                self.add_sensor_data_to_db(recordset, temp_sensor,
                                           temp_channel, sensor_timestamps,
                                           values)

                next_cut_timestamp += 3600

        # Flush DB
        self.db.flush()
예제 #18
0
    def import_to_database(self, results):
        [info, data] = results

        # print(info)

        # Creating recordset
        # print(info['Start Date'], info['Last Sample Time'])
        # start = int(info['Start Date'])
        # stop = int(info['Last Sample Time'])

        # start_timestamp = ticksconverter(start)
        # end_timestamp = ticksconverter(stop)

        # all_counts = [0, 0, 0]
        if data.__contains__('activity'):

            # print('activity found')
            # Create sensor
            accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER, 'Accelerometer', info['Device Type'],
                                                         'Unknown', info['Sample Rate'], 1)

            accelerometer_channels = list()

            # Create channels
            accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                                                 DataFormat.FLOAT32, 'Accelerometer_Y'))

            accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                                                 DataFormat.FLOAT32, 'Accelerometer_X'))

            accelerometer_channels.append(self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                                                 DataFormat.FLOAT32, 'Accelerometer_Z'))

            # Should be 1970, epoch
            last_timestamp = 0
            all_timestamps = []
            value_dict = {}

            # Import data
            for epoch in data['activity']:
                # An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z

                current_timestamp = epoch[0]
                # print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1))

                # Check for consecutive timestamps
                create_array = current_timestamp != (last_timestamp + 1)

                # Do not allow more than one hour of consecutive data
                if create_array is not True:
                    if current_timestamp - all_timestamps[-1] >= 3600:
                        create_array = True

                # Consecutive timestamps?
                if create_array is True:
                    all_timestamps.append(current_timestamp)
                    # Create list for all values for this timestamp
                    value_dict[current_timestamp] = [list(), list(), list()]

                # Get data
                samples = epoch[1]

                # Separate write for each channel
                for index in range(0, len(accelerometer_channels)):
                    # Using last timestamp to append data
                    value_dict[all_timestamps[-1]][index].append(samples[:, index])
                    # print("samples shape", samples.shape, samples[:, index].shape)

                # Update timestamp
                last_timestamp = current_timestamp

            # Insert into DB as chunks of data
            # print('should insert records count: ', len(all_timestamps))
            # print('should insert data count:', len(value_dict))
            counters = [0, 0, 0]

            for timestamp in all_timestamps:
                session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']

                for index in range(0, len(value_dict[timestamp])):
                    # print(index, timestamp, len(value_dict[timestamp][index]))
                    vector = np.concatenate(value_dict[timestamp][index])
                    # print('inserting values :', len(value_dict[timestamp][index]))
                    # print('vector: ', len(vector), vector.shape, vector.dtype)

                    # Create sensor timestamps first
                    sensor_timestamps = SensorTimestamps()

                    # Create time vector
                    # TODO Share time vector for all accelerometers?
                    timevect = np.linspace(timestamp, timestamp + len(value_dict[timestamp][index]),
                                           num=len(vector), endpoint=False, dtype=np.float64)
                    sensor_timestamps.timestamps = timevect
                    sensor_timestamps.update_timestamps()

                    recordset = self.get_recordset(timestamp, session_name)

                    # Update end_timestamp if required
                    if timestamp > recordset.end_timestamp.timestamp():
                        recordset.end_timestamp = datetime.datetime.fromtimestamp(timestamp)

                    counters[index] += len(vector)
                    if len(vector) > 0:
                        self.add_sensor_data_to_db(recordset, accelerometer_sensor, accelerometer_channels[index],
                                                   sensor_timestamps, vector)

            # print('total samples inserted:', counters)
            # print('total timestamps processed:', len(all_timestamps))

            # Flush DB
            self.db.flush()

        if data.__contains__('battery'):
            # print('battery found')
            # Create sensor
            volt_sensor = self.add_sensor_to_db(SensorType.BATTERY, 'Battery', info['Device Type'], 'Unknown',
                                                0, 1)

            # Create channel
            volt_channel = self.add_channel_to_db(volt_sensor, Units.VOLTS, DataFormat.FLOAT32, 'Battery')

            for epoch in data['battery']:
                timestamp = datetime.datetime.fromtimestamp(epoch[0])
                session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']
                value = np.float32(epoch[1])

                recordset = self.get_recordset(epoch[0], session_name)

                timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64)
                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = timevect
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if epoch[0] > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = timestamp

                self.add_sensor_data_to_db(recordset, volt_sensor, volt_channel, sensor_timestamps, value)

            # Flush to DB (ram)
            self.db.flush()

        if data.__contains__('lux'):
            # print('lux found')
            # Create sensor
            lux_sensor = self.add_sensor_to_db(SensorType.LUX, 'Lux', info['Device Type'], 'Unknown', 0, 1)

            # Create channel
            lux_channel = self.add_channel_to_db(lux_sensor, Units.LUX, DataFormat.FLOAT32, 'Lux')

            for epoch in data['lux']:
                timestamp = datetime.datetime.fromtimestamp(epoch[0])
                session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
                               + info['Subject Name'] + '_SN:' + info['Serial Number']
                value = np.float32(epoch[1])

                recordset = self.get_recordset(epoch[0], session_name)

                timevect = np.linspace(epoch[0], epoch[0] + 1, num=1, endpoint=False, dtype=np.float64)
                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()
                sensor_timestamps.timestamps = timevect
                sensor_timestamps.update_timestamps()

                # Update end_timestamp if required
                if epoch[0] > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = timestamp

                self.add_sensor_data_to_db(recordset, lux_sensor, lux_channel, sensor_timestamps, value)

            # Flush to DB (ram)
            self.db.flush()

        # Write data to file
        self.db.commit()

        self.update_progress.emit(100)
예제 #19
0
    def import_activity_to_database(self, info, activity: dict):
        # print('activity found')
        # Create sensor
        accelerometer_sensor = self.add_sensor_to_db(SensorType.ACCELEROMETER,
                                                     'Accelerometer',
                                                     info['Device Type'],
                                                     'Activity',
                                                     info['Sample Rate'], 1)

        accelerometer_channels = list()

        # Create channels
        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Y'))

        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_X'))

        accelerometer_channels.append(
            self.add_channel_to_db(accelerometer_sensor, Units.GRAVITY_G,
                                   DataFormat.FLOAT32, 'Accelerometer_Z'))

        # Should be 1970, epoch
        last_timestamp = 0
        all_timestamps = []
        value_dict = {}

        # Import data
        for epoch in activity:
            # An epoch will contain a timestamp and array with each acc_x, acc_y, acc_z

            current_timestamp = epoch[0]
            # print('current_timestamp', current_timestamp, current_timestamp == (last_timestamp + 1))

            # Check for consecutive timestamps
            create_array = current_timestamp != (last_timestamp + 1)

            # Do not allow more than one hour of consecutive data
            if create_array is not True:
                if current_timestamp - all_timestamps[-1] >= 3600:
                    create_array = True

            # Consecutive timestamps?
            if create_array is True:
                all_timestamps.append(current_timestamp)
                # Create list for all values for this timestamp
                value_dict[current_timestamp] = [list(), list(), list()]

            # Get data
            samples = epoch[1]

            # Separate write for each channel
            for index in range(0, len(accelerometer_channels)):
                # Using last timestamp to append data
                value_dict[all_timestamps[-1]][index].append(samples[:, index])
                # print("samples shape", samples.shape, samples[:, index].shape)

            # Update timestamp
            last_timestamp = current_timestamp

        # Insert into DB as chunks of data
        # print('should insert records count: ', len(all_timestamps))
        # print('should insert data count:', len(value_dict))
        counters = [0, 0, 0]

        for timestamp in all_timestamps:
            session_name = str(timestamp) + '_' + info['Device Type'] + '_' \
                           + info['Subject Name'] + '_SN:' + info['Serial Number']

            for index in range(0, len(value_dict[timestamp])):
                # print(index, timestamp, len(value_dict[timestamp][index]))
                vector = np.concatenate(value_dict[timestamp][index])
                # print('inserting values :', len(value_dict[timestamp][index]))
                # print('vector: ', len(vector), vector.shape, vector.dtype)

                # Create sensor timestamps first
                sensor_timestamps = SensorTimestamps()

                # Create time vector
                # TODO Share time vector for all accelerometers?
                timevect = np.linspace(timestamp,
                                       timestamp +
                                       len(value_dict[timestamp][index]),
                                       num=len(vector),
                                       endpoint=False,
                                       dtype=np.float64)
                sensor_timestamps.timestamps = timevect
                sensor_timestamps.update_timestamps()

                recordset = self.get_recordset(timestamp, session_name)

                # Update end_timestamp if required
                if timestamp > recordset.end_timestamp.timestamp():
                    recordset.end_timestamp = datetime.datetime.fromtimestamp(
                        timestamp)

                counters[index] += len(vector)
                if len(vector) > 0:
                    self.add_sensor_data_to_db(recordset, accelerometer_sensor,
                                               accelerometer_channels[index],
                                               sensor_timestamps, vector)

        # print('total samples inserted:', counters)
        # print('total timestamps processed:', len(all_timestamps))

        # Flush DB
        self.db.flush()