def __init__(self, parent=None):
self.parent = parent
self.fs = main_config['fs']
self.n_channels = main_config['n_channels']
self.t0 = 0
self.last_ts = 0
# Game log reader (separate thread)
self.game_logs_path = game_config['game_logs_path']
self.game_log_reader = None
self._expected_action = (0, 'Rest')
self.thread_log = QtCore.QThreadPool()
# Game window (separate process)
clean_log_directory(self.game_logs_path)
self.game = None
# Port event sender
self.micro_path = main_config['micro_path']
self.port_sender = None
# Game player
self.game_path = game_config['game_path']
self.player_idx = game_config['player_idx']
self.game_start_time = None
# Chronogram
self.chrono_source = ColumnDataSource(dict(ts=[],
y_true=[],
y_pred=[]))
self.pred_decoding = main_config['pred_decoding']
# LSL stream reader
self.lsl_reader = None
self.lsl_start_time = None
self.thread_lsl = QtCore.QThreadPool()
self.channel_source = ColumnDataSource(dict(ts=[], eeg=[]))
self._lsl_data = (None, None)
# LSL stream recorder
if not os.path.isdir(main_config['record_path']):
os.mkdir(main_config['record_path'])
self.record_path = main_config['record_path']
self.record_name = game_config['record_name']
self.lsl_recorder = None
# Predictor
self.models_path = main_config['models_path']
self.input_signal = np.zeros((self.n_channels, 4 * self.fs))
self.predictor = None
self.thread_pred = QtCore.QThreadPool()
self._pred_action = (0, 'Rest')
def __init__(self, files, pxSize, crop, gaussSigma, intThres, cArgs):
super().__init__()
self.files = files
self.pxSize = pxSize
self.crop = crop
self.gaussSigma = gaussSigma / self.pxSize
self.intThres = intThres
self.cArgs = cArgs
self.pool = QtCore.QThreadPool()
self.pool.setMaxThreadCount(1)
self.nfiles = len(files)
self.subimgPxSize = 1000 / self.pxSize
# Get data shape and derivates
im = Image.open(self.files[0])
inputData = np.array(im).astype(np.float64)
initShape = inputData.shape
self.bound = (np.array(inputData.shape) - self.crop).astype(np.int)
inputData = inputData[self.crop:self.bound[0], self.crop:self.bound[1]]
dataShape = inputData.shape
self.n = (np.array(dataShape) / self.subimgPxSize).astype(int)
self.mag = np.array(dataShape) / self.n
self.corrArray = np.zeros((self.nfiles, self.n[0], self.n[1]))
self.path = os.path.split(self.files[0])[0]
self.corrExp = np.empty((self.nfiles, initShape[0], initShape[1]),
dtype=np.single)
self.corrExp[:] = np.nan
self.ringsExp = np.empty((self.nfiles, initShape[0], initShape[1]),
dtype=np.single)
self.ringsExp[:] = np.nan
def __init__(self, *args, **kwargs):
super(Sim, self).__init__()
self.cp = get_cmap('inferno')
self.cam = cv2.VideoCapture(0)
self.win = pg.GraphicsLayoutWidget()
self.win.show()
self.view = self.win.addViewBox()
ret, frame = self.cam.read()
self.view.setAspectLocked(True)
self.img = pg.ImageItem(border='w')
self.view.addItem(self.img)
self.view.setRange(QtCore.QRectF(0, 0, 640, 480))
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2) #n of threads running
##modify when add the marker feature
self.loc = [2, 2]
self.cam_frame = np.zeros([480, 640, 3]).astype(np.uint8)
self.roach_frame = np.zeros([480, 640, 3]).astype(np.uint8)
self.superposition = np.zeros([480, 640, 3]).astype(np.uint8)
self.Update()
def __init__(self, parent=None):
self.parent = parent
self.fs = main_config['fs']
self.n_channels = main_config['n_channels']
self.t0 = 0
self.last_ts = 0
self.game_is_on = False
# Chronogram
self.chrono_source = ColumnDataSource(dict(ts=[], y_pred=[]))
self.pred_decoding = main_config['pred_decoding']
# LSL stream reader
self.lsl_reader = None
self.lsl_start_time = None
self._lsl_data = (None, None)
self.thread_lsl = QtCore.QThreadPool()
self.channel_source = ColumnDataSource(dict(ts=[], eeg=[]))
self.buffer_size_s = 10
# LSL stream recorder
if not os.path.isdir(main_config['record_path']):
os.mkdir(main_config['record_path'])
self.record_path = main_config['record_path']
self.record_name = warmup_config['record_name']
self.lsl_recorder = None
# Predictor
self.models_path = main_config['models_path']
self.input_signal = np.zeros((self.n_channels, 4 * self.fs))
self.predictor = None
self.thread_pred = QtCore.QThreadPool()
self._pred_action = (0, 'Rest')
# Feedback images
self.static_folder = warmup_config['static_folder']
self.action2image = warmup_config['action2image']
def __init__(self, player_idx):
# Send commands using a separate thread
self.thread_game = QtCore.QThreadPool()
def __init__(self, *args, **kwargs):
super(SensorApplication, self).__init__(*args, **kwargs)
# logger properties setup
self.folder_name = "logs/"
traces_name = self.folder_name + dt.datetime.now().strftime(
"%d-%m-%Y %Hh%Mm%Ss")
self.cvs_name = traces_name + ".csv"
self.logfile_name = traces_name + ".txt"
os.makedirs(os.path.dirname(self.logfile_name), exist_ok=True)
logging.basicConfig(filename=self.logfile_name,
level=logging.DEBUG,
datefmt="%H:%M:%S",
format=LOG_FORMAT)
self.logger1 = logging.getLogger('RawData')
self.logger2 = logging.getLogger('ProgFlow')
# Thread poll for getting serial line
self.thread_pool = QtCore.QThreadPool()
self.thread_pool.setMaxThreadCount(20)
self.logger2.info("Multithreading with maximum %d threads" %
self.thread_pool.maxThreadCount())
# Serial properties setup
self.serial_dev = serial.Serial()
self.serial_dev.port = 'COM4'
self.serial_dev.baudrate = 115200
self.serial_dev.timeout = 2
# graphic properties setup
self.win = QtGui.QMainWindow()
self.area = DockArea()
self.win.setCentralWidget(self.area)
self.win.resize(1200, 650)
self.win.setWindowTitle('IMU signal viewer')
# Note that size arguments are only a suggestion; docks will still have to
# fill the entire dock area and obey the limits of their internal widgets
self.d1 = Dock("Accelerometer", size=(400, 400))
self.d2 = Dock("Magnetometer", size=(400, 400))
self.d3 = Dock("Gyroscope", size=(400, 400))
self.d4 = Dock("LinearAcc", size=(800, 250))
self.d5 = Dock("Position", size=(800, 400))
self.d6 = Dock("Euler", size=(800, 250))
self.d7 = Dock("Menu", size=(50, 50))
self.d8 = Dock("Calibration", size=(400, 400))
self.d9 = Dock("Graph", size=(CANVAS_LENGTH, CANVAS_HEIGHT))
self.d10 = Dock("Word", size=(800, 250))
self.d11 = Dock("Clasification", size=(400, 400))
self.area.addDock(
self.d4, 'left') # place self.d6 at left edge of dock self.area
self.area.addDock(self.d10, 'above', self.d4)
self.area.addDock(self.d6, 'above', self.d10)
self.area.addDock(self.d5, 'bottom',
self.d6) # place self.d9 bottom edge of self.d6
self.area.addDock(self.d9, 'above', self.d5)
self.area.addDock(self.d1, 'right')
self.area.addDock(self.d3, 'bottom', self.d1)
self.area.addDock(self.d8, 'bottom', self.d3)
self.area.addDock(self.d11, 'above', self.d8)
self.area.addDock(self.d2, 'above', self.d11)
self.area.addDock(self.d7, 'bottom', self.d2)
plot_acc = pg.PlotWidget(title="Accelerometer m/s^2")
plot_acc.addLegend()
plot_acc.setLabel('bottom', 'Time', units='s')
plot_acc.setYRange(-60, 60)
self.acc_x = plot_acc.plot(pen=pg.mkPen('r', width=2), name='x')
self.acc_y = plot_acc.plot(pen=pg.mkPen('b', width=2), name='y')
self.acc_z = plot_acc.plot(pen=pg.mkPen('g', width=2), name='z')
self.d1.addWidget(plot_acc)
plot_mag = pg.PlotWidget(title="Magnetometer uT")
plot_mag.addLegend()
plot_mag.setLabel('bottom', 'Time', units='s')
plot_mag.setYRange(-90, 90)
self.mag_x = plot_mag.plot(pen=pg.mkPen('r', width=2), name='x')
self.mag_y = plot_mag.plot(pen=pg.mkPen('b', width=2), name='y')
self.mag_z = plot_mag.plot(pen=pg.mkPen('g', width=2), name='z')
self.d2.addWidget(plot_mag)
plot_gyr = pg.PlotWidget(title="Gyroscope rad/s")
plot_gyr.addLegend()
plot_gyr.setLabel('bottom', 'Time', units='s')
plot_gyr.setYRange(-1500, 1500)
self.gyr_x = plot_gyr.plot(pen=pg.mkPen('r', width=2), name='x')
self.gyr_y = plot_gyr.plot(pen=pg.mkPen('b', width=2), name='y')
self.gyr_z = plot_gyr.plot(pen=pg.mkPen('g', width=2), name='z')
self.d3.addWidget(plot_gyr)
plot_lin = pg.PlotWidget(title="Linear Accelerometer m/s^2")
plot_lin.addLegend()
plot_lin.setLabel('bottom', 'Time', units='s')
plot_lin.setYRange(-60, 60)
self.lin_x = plot_lin.plot(pen=pg.mkPen('r', width=2), name='x')
self.lin_y = plot_lin.plot(pen=pg.mkPen('b', width=2), name='y')
self.lin_z = plot_lin.plot(pen=pg.mkPen('g', width=2), name='z')
self.d4.addWidget(plot_lin)
plot_eul = pg.PlotWidget(title="Euler degrees")
plot_eul.addLegend()
plot_eul.setLabel('bottom', 'Time', units='s')
plot_eul.setYRange(-200, 200)
self.eul_x = plot_eul.plot(pen=pg.mkPen('r', width=2), name='x')
self.eul_y = plot_eul.plot(pen=pg.mkPen('b', width=2), name='y')
self.eul_z = plot_eul.plot(pen=pg.mkPen('g', width=2), name='z')
self.d6.addWidget(plot_eul)
# canvas dock
self.label = QtWidgets.QLabel()
self.canvas = QtGui.QPixmap(800, 400)
self.canvas.fill(QtCore.Qt.white)
self.label.setPixmap(self.canvas)
self.d9.addWidget(self.label)
self.last_x, self.last_y = CANVAS_LENGTH / 2, CANVAS_HEIGHT / 2
self.last_ang_x, self.last_ang_y = 0.0, 0.0
# display word dock
layout_word = pg.LayoutWidget()
self.label_word = QtGui.QLabel()
layout_word.addWidget(self.label_word, row=0, col=0)
self.d10.addWidget(layout_word)
# plot position dock
self.plot_position_wg = pg.PlotWidget(title="Character Display")
self.plot_position_wg.setYRange(-0.50, 0.50)
self.plot_position_wg.setXRange(-0.50, 0.50)
self.plot_position_wg.addLegend()
self.position_coord = self.plot_position_wg.plot(
pen=pg.mkPen('r', width=5))
self.d5.addWidget(self.plot_position_wg)
# calibration window dock
layout_calibration = pg.LayoutWidget()
label_calib = QtGui.QLabel("Calibration:")
self.label_calib_acc = QtGui.QLabel("ACC: 0")
self.label_calib_gyr = QtGui.QLabel("GYR: 0")
self.label_calib_mag = QtGui.QLabel("MAG: 0")
self.label_calib_sys = QtGui.QLabel("SYS: 0")
label_quat = QtGui.QLabel("Quaternion:")
label_eul = QtGui.QLabel("Euler:")
self.label_quaternion = QtGui.QLabel("W: 0.0\tX: 0.0\tY: 0.0\tZ: 0.0")
self.label_euler = QtGui.QLabel("X: 0.0\tY: 0.0\tZ: 0.0")
layout_calibration.addWidget(label_calib, row=0, col=0)
layout_calibration.addWidget(self.label_calib_acc, row=1, col=0)
layout_calibration.addWidget(self.label_calib_gyr, row=2, col=0)
layout_calibration.addWidget(self.label_calib_mag, row=3, col=0)
layout_calibration.addWidget(self.label_calib_sys, row=4, col=0)
layout_calibration.addWidget(label_quat, row=5, col=0)
layout_calibration.addWidget(self.label_quaternion, row=5, col=1)
layout_calibration.addWidget(label_eul, row=6, col=0)
layout_calibration.addWidget(self.label_euler, row=6, col=1)
self.d8.addWidget(layout_calibration)
# classification window dock
layout_letter = pg.LayoutWidget()
self.label_letter = QtGui.QLabel("")
self.label_letter.setFont(QtGui.QFont('Arial', 50))
layout_letter.addWidget(self.label_letter, row=0, col=0)
self.d11.addWidget(layout_letter)
# Menu window dock
layout_buttons = pg.LayoutWidget()
connect_b = QtGui.QPushButton(
"Connect") # To start listening to Serial port
self.record_b = QtGui.QPushButton("Record") # To start Recording data
self.stop_b = QtGui.QPushButton("Stop") # To start Recording data
self.disconnect_b = QtGui.QPushButton(
"Disconnect Device") # To start Recording data
self.change_name_b = QtGui.QPushButton(
"Set log name") # To start Recording data
recenter_b = QtGui.QPushButton(
"Re-Center") # Re-center pointer position
self.textbox = QtGui.QLineEdit()
label_1 = QtGui.QLabel("""Orientation Mode: """)
self.mode_box = QtGui.QComboBox()
self.mode_box.addItems(
["complementary_filter", "Madgwick", "sensor_ahrs"])
label_2 = QtGui.QLabel("""Word Mode: """)
self.word_box = QtGui.QComboBox()
self.word_box.addItems(["OFF", "ON"])
label_3 = QtGui.QLabel("""Word command: """)
self.start_b = QtGui.QPushButton(
"Start") # To start listening to Serial port
self.next_b = QtGui.QPushButton("Next") # To start Recording data
self.end_b = QtGui.QPushButton("End") # To start Recording data
layout_buttons.addWidget(connect_b, row=0, col=0)
layout_buttons.addWidget(self.record_b, row=0, col=1)
layout_buttons.addWidget(self.stop_b, row=0, col=2)
layout_buttons.addWidget(self.disconnect_b, row=0, col=3)
layout_buttons.addWidget(self.change_name_b, row=1, col=0)
layout_buttons.addWidget(self.textbox, row=1, col=1, colspan=3)
layout_buttons.addWidget(label_1, row=2, col=0)
layout_buttons.addWidget(self.mode_box, row=2, col=1, colspan=2)
layout_buttons.addWidget(recenter_b, row=2, col=3)
layout_buttons.addWidget(label_2, row=3, col=0)
layout_buttons.addWidget(self.word_box, row=3, col=1, colspan=2)
layout_buttons.addWidget(label_3, row=4, col=0)
layout_buttons.addWidget(self.start_b, row=4, col=1)
layout_buttons.addWidget(self.next_b, row=4, col=2)
layout_buttons.addWidget(self.end_b, row=4, col=3)
connect_b.pressed.connect(self.listen)
self.record_b.pressed.connect(self.record)
self.stop_b.pressed.connect(self.stop_recording)
self.disconnect_b.pressed.connect(self.disconnect)
self.change_name_b.pressed.connect(self.change_name)
self.start_b.pressed.connect(self.start_letter)
self.next_b.pressed.connect(self.next_letter)
self.end_b.pressed.connect(self.end_letter)
recenter_b.pressed.connect(self.recenter_pointer)
self.mode_box.currentIndexChanged.connect(self.mode_change)
self.word_box.currentIndexChanged.connect(self.word_mode_change)
self.start_b.setEnabled(False)
self.next_b.setEnabled(False)
self.end_b.setEnabled(False)
self.d7.addWidget(layout_buttons)
self.mutex = QtCore.QMutex() # mutex to control raw_data access
self.mutex_line = QtCore.QMutex() # mutex to control raw_line access
self.raw_line = [] # This list will keep the incoming lines to process
self.imu_struct = {
"TIME": [],
"ELAPSED_SECONDS": [],
"PSTATE": [],
"acc_x": [],
"acc_y": [],
"acc_z": [],
"mag_x": [],
"mag_y": [],
"mag_z": [],
"gyr_x": [],
"gyr_y": [],
"gyr_z": [],
"lin_x": [],
"lin_y": [],
"lin_z": [],
"eul_x": [],
"eul_y": [],
"eul_z": [],
"qua_w": [],
"qua_x": [],
"qua_y": [],
"qua_z": []
}
# Prettier and better solution would be to include this values to
# imu_struct but that would imply REFACTORING the code to calculate
# the x & y position in the fill table thread instead of the plotting thread
self.empty_xy_points = {"x": [], "y": []}
self.canvas_xy_points = copy.deepcopy(self.empty_xy_points)
##########################################################################################
# The rate of receiving RAW data depends on how much data is being transmitted
# ########################################################################################
# Data received | Possible Calculation Methods | speed
# --------------------------------------------------------------------------------
# acc + gyr + quat | comp. filter | Hz = 33 period = 30ms
##########################################################################################
# Orientation Modes:
# complementary_filter Not working correctlty, can be improved or removed
# Madgwick Using Madgwick's Sensor Fusion Algorithm
# sensor_ahrs Using the Orientation provided by the sensor
##########################################################################################
self.mode = "sensor_ahrs"
self.period = 0.035 # The period depends on the transmission rate from the finger wearable
self.rate = 1 / self.period
# this is the structure where the las 50 samples are stored for displaying on the real-time
# plots on screen.
self.raw_data = copy.deepcopy(self.imu_struct)
# empty structure, must be keep that way
self.data_struct = copy.deepcopy(self.imu_struct)
# This is the structure where the data is stored for storing the online data after completing
# the finger-movement.
self.to_save_data = copy.deepcopy(self.raw_data)
# Display the GUI
self.win.show()
# Calibration from BNO055 IMU sensor
self.accel_calibration = 0
self.gyro_calibration = 0
self.magnet_calibration = 0
self.system_calibration = 0
# PyQt timer
self.timer = pg.QtCore.QTimer()
self.timer.timeout.connect(self.update_graph)
#flags to control de state machine (see Thesis Document chapter 3.4)
self.first_angle_measure = True
self.first_line = True
self.first_update = True
self.first_graph_update = True
self.stop_threads = False
self.init_time = 0
self.recording = False
self.word_mode = False
self.recording_word = False
self.word_letter_number = 0
# Madgwick variable
# By Increasing the value of Beta(magnetic & acc correction to gyro) is faster but more unstable
self.madgwick_ahrs = skin.imus.Madgwick(rate=self.rate, Beta=0.10)
self.prediction_model = None
self.load_model()
self.word_prediction = ""
