def update(self, line): if not True in [line.startswith(f) for f in self.filter]: if not line.startswith('<'): try: code = line.split('\t')[0] decoded = self.code_map[code] line = '\t'.join([decoded, line.split('\t')[1]]) except: utils.printer("Error dealing with line %s" % line, 'error') pass # TODO deal with the history functionality history_len = 100 # FIXME expose this property? or remove it. for now for debugging if len(self.lines) < history_len: self.lines.append(line) else: self.lines.append(line) self.lines = self.lines[1:] # print lines in window sb = self.TextBrowser.verticalScrollBar() sb_prev_value = sb.value() self.TextBrowser.setPlainText('\n'.join(self.lines)) # scroll to end sb.setValue(sb.maximum())
def __init__(self, parent, config, task_config): super(BonsaiController, self).__init__(parent=parent) self.name = "BonsaiController" self.config = config self.task_config = task_config utils.printer("init bonsai controller", "debug")
def update(self, line): self.lines.append(line) decoded = self.decode(line) if decoded is not None: # the event that separates the stream of data into chunks of trials if decoded == self.new_trial_event: # parse lines TrialMetricsDf = None try: TrialDf = bhv.parse_lines(self.lines, code_map=self.code_map, parse_var=True) TrialMetricsDf = bhv.parse_trial(TrialDf, self.Metrics) except ValueError: # important TODO - investigate this! this was added with cue on reach and no mistakes utils.printer('failed parse of lines into TrialDf', 'error') # utils.debug_trace() pass if TrialMetricsDf is not None: if self.SessionDf is None: # on first self.SessionDf = TrialMetricsDf else: self.SessionDf = self.SessionDf.append(TrialMetricsDf) self.SessionDf = self.SessionDf.reset_index(drop=True) # emit data self.trial_data_available.emit(TrialDf, TrialMetricsDf) # restart lines with current line self.lines = [line]
def send_raw(self, bytestr): """ sends bytestring """ if hasattr(self, 'connection'): if self.connection.is_open: self.connection.write(bytestr) else: utils.printer("Arduino is not connected", 'error')
def use_vars(self, Df): # check if possible if not np.all(Df['name'].sort_values().values == self.Df['name'].sort_values().values): utils.printer( "unequal variable names between last session and this session", 'error') else: self.VariableEditWidget.set_entries(Df)
def init_counters(self): if 'OnlineAnalysis' in dict(self.task_config).keys(): if 'counters' in dict(self.task_config['OnlineAnalysis']).keys(): counters = [c.strip() for c in self.task_config['OnlineAnalysis']['counters'].split(',')] for counter in counters: mod = importlib.import_module('Visualizers.Counters') C = getattr(mod, counter) self.Counters.append(C(self)) utils.printer("initializing counter: %s" % counter, 'msg')
def send(self, command): """ sends string command interface to arduino, interface compatible """ if hasattr(self, 'connection'): cmd = '<' + command + '>' bytestr = str.encode(cmd) if self.connection.is_open: self.connection.write(bytestr) else: utils.printer("Arduino is not connected", 'error')
def check(self, A, B): """ check consistency of all clock pulses and if possible fixes them """ if self.data[A].shape[0] == 0 or self.data[A].shape[0] == 0: utils.printer("sync failed - %s is empty" % A, 'error') return False elif self.data[B].shape[0] == 0: utils.printer("sync failed - %s is empty" % B, 'error') return False elif self.data[A].shape[0] != self.data[B].shape[0]: # Decide which is the reference to cut to if self.data[A].shape[0] > self.data[B].shape[0]: bigger = 'A' print("Clock A has more pulses") t_bigger = self.data[A] t_smaller = self.data[B] else: print("Clock B has more pulses") bigger = 'B' t_bigger = self.data[B] t_smaller = self.data[A] utils.printer( "sync problem - unequal number, %s has more sync signals" % bigger, 'warning') utils.printer("Number in %s: %i" % (A, self.data[A].shape[0]), 'warning') utils.printer("Number in %s: %i" % (B, self.data[B].shape[0]), 'warning') # Compute the difference offset = np.argmax( np.correlate(np.diff(t_bigger), np.diff(t_smaller), mode='valid')) # Cut the initial timestamps from the argument with more clock pulses t_bigger = t_bigger[offset:t_smaller.shape[0] + offset] if bigger == 'A': self.data[A] = t_bigger self.data[B] = t_smaller else: self.data[B] = t_bigger self.data[A] = t_smaller return True else: return True
def Run(self): """ ask for weight initialize folder structure runs all controllers """ # flags self.running = True # UI related self.RunBtn.setEnabled(False) self.DoneBtn.setEnabled(True) self.online_vis_btn.setEnabled(True) self.TaskChoiceWidget.setEnabled(False) self.AnimalChoiceWidget.setEnabled(False) # animal popup self.RunInfo = RunInfoPopup(self) utils.printer("RUN", 'task') utils.printer("Task: %s" % self.task, 'msg') utils.printer("Animal: %s - body weight: %s%%" % (self.Animal.display(), self.Animal.weight_ratio()),'msg') # make folder structure date_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") # underscores in times bc colons kill windows paths ... self.run_folder = self.Animal.folder / '_'.join([date_time, self.task]) os.makedirs(self.run_folder, exist_ok=True) for Controller in self.Controllers: utils.printer("running controller: %s" % Controller.name, 'msg') Controller.Run(self.run_folder) # reset and start the counters for Counter in self.Counters: Counter.init()
def animal_changed(self): current_id = self.AnimalChoiceWidget.get_value().split(' - ')[0] self.config['current']['animal'] = current_id self.Animal, = [Animal for Animal in self.Animals if Animal.ID == current_id] # TODO bring back via a button # # displaying previous sessions info # if hasattr(self,'AnimalInfoWidget'): # self.AnimalInfoWidget.close() # self.Children.remove(self.AnimalInfoWidget) # self.AnimalInfoWidget = AnimalInfoWidget(self, self.config, self.Animal) # self.Children.append(self.AnimalInfoWidget) utils.printer("Animal: %s" % self.Animal.display(),'msg')
def Run(self, folder): """ folder is the logging folder """ utils.printer("running bonsai controller", "debug") # animal = self.config['current']['animal'] task = self.config['current']['task'] task_folder = Path(self.config['paths']['tasks_folder']) / task save_path = folder / 'bonsai_' # this needs to be fixed in bonsai # FIXME TODO # constructing the bonsai exe string parameters = "-p:save_path=\"" + str(save_path) + "\"" # com port for firmata if 'firmata_arduino_port' in dict(self.config['connections']).keys(): parameters = parameters + " -p:com_port=" + self.config[ 'connections']['firmata_arduino_port'] # com port for load cell if 'harp_loadcell_port' in dict(self.config['connections']).keys(): parameters = parameters + " -p:LC_com_port=" + self.config[ 'connections']['harp_loadcell_port'] # getting other manually set params variables_path = task_folder / "Bonsai" / "interface_variables.ini" if variables_path.exists(): with open(variables_path, 'r') as fH: params = fH.readlines() params = [p.strip() for p in params] for line in params: parameters = parameters + " -p:%s" % line bonsai_exe = Path(self.config['system']['bonsai_cmd']) bonsai_workflow = task_folder / 'Bonsai' / self.task_config[ 'workflow_fname'] command = ' '.join([ str(bonsai_exe), str(bonsai_workflow), "--start", parameters, "&" ]) utils.printer("bonsai command: %s " % command, 'msg') log = open(save_path.with_name('bonsai_log.txt'), 'w') theproc = subprocess.Popen(command, shell=True, stdout=log, stderr=log)
def set_entry(self, name, value): """ controller function - update both view and model """ if name not in list(self.Df['name']): utils.printer( "trying to set variable %s, but is not part of model" % name, "warning") # self.Df.loc[name,'value'] = value else: # get index ix = list(self.Df['name']).index(name) # update model # dtype = self.FormLayout.itemAt(ix,1).widget().dtype dtype = self.Df.loc[ix, 'dtype'] # dtype is part of the model self.Df.loc[ix, 'value'] = np.array(value, dtype=dtype) # explicit cast # update view self.FormLayout.itemAt(ix, 1).widget().set_value(value)
def __init__(self, *args, config_path=None): super(TaskControlApp, self).__init__(*args) # read config.ini self.config_path = Path(config_path) self.config = configparser.ConfigParser() self.config.read(self.config_path) print(" --- this is TaskControl --- ") utils.printer("using config: %s" % config_path, 'msg') # launch GUI self.Settings_Widget = SettingsWidget(self, self.config) # hack - store default box settings self.default_box_config = self.config['box'] # on close - TODO check if obsolete with proper QT parent child structure self.setQuitOnLastWindowClosed(False) self.lastWindowClosed.connect(self.onLastClosed) self.exec_()
def connect(self): """ establish serial connection with the arduino board """ com_port = self.config['connections']['FSM_arduino_port'] baud_rate = self.config['connections']['arduino_baud_rate'] try: utils.printer("initializing serial port: " + com_port, 'message') # ser = serial.Serial(port=com_port, baudrate=baud_rate, timeout=2) connection = serial.Serial(port=com_port, baudrate=baud_rate, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE, timeout=1, xonxoff=0, rtscts=0) self.reset_arduino(connection) return connection except: utils.printer("failed to connect to the FSM arduino", 'error') sys.exit()
def send_variables(self): """ sends all current variables to arduino """ if hasattr( self.parent(), 'connection' ): # TODO check if this can attempt to write on a closed connection Df = self.VariableEditWidget.get_entries() for i, row in Df.iterrows(): # this is the hardcoded command sending definition cmd = ' '.join(['SET', str(row['name']), str(row['value'])]) cmd = '<' + cmd + '>' bytestr = str.encode(cmd) # reading and writing from different threads apparently threadsafe # https://stackoverflow.com/questions/8796800/pyserial-possible-to-write-to-serial-port-from-thread-a-do-blocking-reads-fro # self.parent().connection.write(bytestr) self.parent().send_raw(bytestr) time.sleep( 0.05 ) # to fix incomplete sends? verify if this really works ... else: utils.printer("Arduino is not connected", 'error')
def load_last_vars(self): """ try to get arduino variables from last run for the task only loads, does not send! """ config = self.parent().config folder = Path( config['paths']['animals_folder']) / config['current']['animal'] SessionsDf = utils.get_sessions(folder) try: previous_sessions = SessionsDf.groupby('task').get_group( config['current']['task']) except KeyError: utils.printer( "trying to use last vars, but animal has not been run on this task before.", 'error') return None # to allow for this functionalty while task is running if self.parent().parent().running: ix = -2 else: ix = -1 try: prev_session_path = Path(previous_sessions.iloc[ix]['path']) prev_vars_path = prev_session_path / config['current'][ 'task'] / "Arduino" / "src" / "interface_variables.h" if prev_vars_path.exists(): prev_vars = utils.parse_arduino_vars(prev_vars_path) return prev_vars else: utils.printer( "found variables from last session, but can't set them", "error") return None except IndexError: # thrown when there is no previous session return None
def parse_arduino_log(log_path, code_map=None, parse_var=True, return_check=False): """ create a DataFrame representation of an arduino log. If a code map is passed a corresponding decoded column will be created for offline use """ with open(log_path, 'r') as fH: lines = fH.readlines() lines = [line.strip() for line in lines] lines = [line for line in lines if line != ''] # test for validity valid_lines = [] invalid_lines = [] for i, line in enumerate(lines): if len(line.split('\t')) == 2 or line.startswith('<'): valid_lines.append(line) else: invalid_lines.append(line) utils.printer("bad line in log: %i: %s" % (i, line), "error") return parse_lines(valid_lines, code_map=code_map, parse_var=parse_var) if return_check == True: if all_good == True: return parse_lines(valid_lines, code_map=code_map, parse_var=parse_var) else: return None else: return parse_lines(valid_lines, code_map=code_map, parse_var=parse_var)
def task_changed(self): # first check if task is running, if yes, don't do anything if self.running == True: utils.printer("trying to change a running task", 'error') return None else: # update current task self.config['current']['task'] = self.TaskChoiceWidget.get_value() self.task = self.config['current']['task'] self.task_folder = Path(self.config['paths']['tasks_folder']) / self.task utils.printer("selected Task: %s" % self.task, 'msg') # parse task config file self.task_config = configparser.ConfigParser() self.task_config.read(self.task_folder / 'task_config.ini') # take down all currently open controllers for Controller in self.Controllers: Controller.stop() Controller.close() self.Controllers = [] for Counter in self.Counters: Counter.stop() Counter.close() self.Counters = [] # run each controller present in task config for section in self.task_config.sections(): utils.printer("initializing %s" % section, 'msg') if section == 'Arduino': self.ArduinoController = ArduinoController(self, self.config, self.task_config['Arduino']) self.Controllers.append(self.ArduinoController) if section == 'Bonsai': self.BonsaiController = BonsaiController(self, self.config, self.task_config['Bonsai']) self.Controllers.append(self.BonsaiController) # if section == 'LoadCell': # self.LoadCellController = LoadCellController(self, self.config, self.task_config['LoadCell']) # self.Controllers.append(self.LoadCellController) # if section == 'Display': # self.DisplayController = HardwareWidgets.DisplayController(self) # self.Controllers.append(self.DisplayController) # after controllers, reinit counter self.init_counters()
# %% # trial selection SDf = bhv.groupby_dict(SessionDf, dict(outcome='correct', correct_side='right')) SessionDf.loc[SessionDf['choice_rt'] < 500] TrialDf = TrialDfs[22] # 1 ms choice RT Df = bhv.event_slice(TrialDf,'TRIAL_ENTRY_EVENT','CHOICE_EVENT') t_on = Df.iloc[0]['t'] - 250 t_off = Df.iloc[-1]['t'] + 2000 make_annotated_video(Vid, t_on, t_off, LogDf, DlcDf) # %% slice entire video from Utils import utils for i, row in SessionDf.iloc[:3].iterrows(): utils.printer("slicing video: Trial %i/%i" % (i, SessionDf.shape[0])) TrialDf = TrialDfs[i] outpath = session_folder / 'plots' / 'video_sliced' os.makedirs(outpath, exist_ok=True) try: Df = bhv.event_slice(TrialDf,'TRIAL_ENTRY_EVENT','ITI_STATE') t_on = Df.iloc[0]['t'] - 250 t_off = Df.iloc[-1]['t'] + 2000 side = row['correct_side'] outcome = row['outcome'] fname = outpath / ("Trial_%i_%s_%s.mp4" % (i, side, outcome)) make_annotated_video(Vid, t_on, t_off, LogDf, DlcDf, fps=10, save=fname) except IndexError: utils.printer("not able to process trial %i" % i,'error')
def upload(self): """ uploads the sketch specified in platformio.ini which is in turn specified in the task_config.ini """ # building interface utils.printer("generating interface.cpp", 'task') try: # catch this exception for downward compatibility utils.printer("generating interface from: %s" % self.vars_path, 'msg') utils.printer( "using as template: %s" % self.task_config['interface_template_fname'], 'msg') interface_template_fname = self.task_config[ 'interface_template_fname'] interface_generator.run(self.vars_path, interface_template_fname) except KeyError: utils.printer("generating interface based on %s" % self.vars_path, 'msg') interface_generator.run(self.vars_path) # uploading code onto arduino # replace whatever com port is in the platformio.ini with the one from task config self.pio_config_path = self.task_folder / "Arduino" / "platformio.ini" pio_config = configparser.ConfigParser() pio_config.read(self.pio_config_path) # get upload port upload_port = self.config['connections']['FSM_arduino_port'] for section in pio_config.sections(): if section.split(":")[0] == "env": pio_config.set(section, "upload_port", upload_port) # write it with open(self.pio_config_path, 'w') as fH: pio_config.write(fH) # get current UI arduino variables, backup defaults, # write the UI derived and upload those, revert after upload # this workaround is necessary to use the get previous variables # functionality ... # backing up original values shutil.copy(self.vars_path, self.vars_path.with_suffix('.default')) # setting the valve calibration factor utils.printer("setting valve calibration factors", 'task') valves = [ key for key in dict(self.config['box']).keys() if key.startswith('valve_') ] for valve in valves: try: utils.printer( 'setting calibration factor of valve: %s = %s' % (valve, self.config['box'][valve]), 'msg') self.VariableController.VariableEditWidget.set_entry( valve, self.config['box'][valve]) except: utils.printer( "can't set valve calibration factors of valve %s" % valve, 'error') # overwriting vars self.VariableController.write_variables(self.vars_path) # upload utils.printer("uploading code on arduino", 'task') prev_dir = Path.cwd() os.chdir(self.task_folder / 'Arduino') fH = open(self.run_folder / 'platformio_build_log.txt', 'w') platformio_cmd = self.config['system']['platformio_cmd'] cmd = ' '.join([platformio_cmd, 'run', '--target', 'upload']) proc = subprocess.Popen(cmd, shell=True, stdout=fH) # ,stderr=fH) proc.communicate() fH.close() os.chdir(prev_dir) # restoring original variables shutil.copy(self.vars_path.with_suffix('.default'), self.vars_path) os.remove(self.vars_path.with_suffix('.default')) utils.printer("done", 'msg')
def log_task(self, folder): """ copy the entire arduino folder to the logging folder """ utils.printer("logging arduino code", 'task') src = self.task_folder target = folder / self.config['current']['task'] shutil.copytree(src, target)
def Run(self, folder): """ folder is the logging folder """ # the folder that is used for storage self.run_folder = folder # needs to be stored for access # logging the code self.log_task(self.run_folder) # upload if self.reprogramCheckBox.checkState() == 2: # true when checked self.upload() else: utils.printer("reusing previously uploaded sketch", 'msg') # last vars if self.VariableController.LastVarsCheckBox.checkState( ) == 2: # true when checked last_vars = self.VariableController.load_last_vars() if last_vars is not None: self.VariableController.use_vars(last_vars) utils.printer("reusing variables from last session", 'msg') else: utils.printer("using default variables from last session", 'msg') self.VariableController.VariableEditWidget.setEnabled(True) # connect to serial port self.connection = self.connect() # start up the online data analyzer if hasattr(self, 'OnlineDataAnalyser'): utils.printer("starting online data analyser", 'msg') self.OnlineDataAnalyser.run() # external logging fH = open(self.run_folder / 'arduino_log.txt', 'w') def read_from_port(ser): while ser.is_open: try: line = ser.readline().decode('utf-8').strip() except AttributeError: line = '' except TypeError: line = '' except serial.serialutil.SerialException: line = '' except UnicodeDecodeError: line = '' if line is not '': # filtering out empty reads fH.write(line + os.linesep) # external logging self.serial_data_available.emit( line) # internal publishing self.thread = threading.Thread(target=read_from_port, args=(self.connection, )) self.thread.start() utils.printer( "listening to FSM arduino on serial port %s" % self.config['connections']['FSM_arduino_port'], 'msg') # potentially this ... # FIXME remove hardcode, check for type? for counter in self.parent().Counters: if hasattr(counter, 'timer'): counter.start()
def stop(self): """ """ utils.printer("stopping bonsai controller", "debug") pass
def closeEvent(self, event): """ """ utils.printer("closing bonsai controller", "debug") self.close()
""" Nicknames = ['Lifeguard', 'Lumberjack', 'Teacher', 'Plumber', 'Poolboy', 'Policeman', 'Therapist'] # Nicknames = ['Therapist'] task_name = 'learn_to_choose_v2' # get animals by Nickname Animals_folder = Path("/media/georg/htcondor/shared-paton/georg/Animals_reaching") Animals = utils.get_Animals(Animals_folder) Animals = [a for a in Animals if a.Nickname in Nicknames] overwrite = False for i, Animal in enumerate(Animals): utils.printer("processing animal %s" % Animal, 'msg') SessionsDf = utils.get_sessions(Animal.folder).groupby('task').get_group(task_name) SessionsDf = SessionsDf.reset_index() for i, row in SessionsDf.iterrows(): session_folder = Path(row['path']) Session = utils.Session(session_folder) # session overview if 0: outpath = Animal.folder / 'plots' / 'session_overviews' / ('session_overview_%s_%s_day_%s.png' % (Session.date, Session.time, Session.day)) if not outpath.exists() or overwrite: plot_session_overview(session_folder, save=outpath) # init histograms if 1:
def plot_init_hist(session_folder, save=None): LogDf = bhv.get_LogDf_from_path(session_folder / "arduino_log.txt") # Sync first loadcell_sync_event = sync.parse_harp_sync(session_folder / 'bonsai_harp_sync.csv', trig_len=100, ttol=5) arduino_sync_event = sync.get_arduino_sync(session_folder / 'arduino_log.txt') Sync = sync.Syncer() Sync.data['arduino'] = arduino_sync_event['t'].values Sync.data['loadcell'] = loadcell_sync_event['t'].values success = Sync.sync('arduino', 'loadcell') # abort if sync fails if not success: utils.printer( "trying to plot_init_hist, but failed to sync in file %s, - aborting" % session_folder) return None LogDf['t_orig'] = LogDf['t'] LogDf['t'] = Sync.convert(LogDf['t'].values, 'arduino', 'loadcell') LoadCellDf = bhv.parse_bonsai_LoadCellData(session_folder / 'bonsai_LoadCellData.csv') # preprocessing samples = 10000 # 10s buffer: harp samples at 1khz, arduino at 100hz, LC controller has 1000 samples in buffer LoadCellDf['x'] = LoadCellDf['x'] - LoadCellDf['x'].rolling(samples).mean() LoadCellDf['y'] = LoadCellDf['y'] - LoadCellDf['y'].rolling(samples).mean() # smoothing forces F = LoadCellDf[['x', 'y']].values w = np.ones(100) F[:, 0] = np.convolve(F[:, 0], w, mode='same') F[:, 1] = np.convolve(F[:, 1], w, mode='same') # detect pushes th = 500 L = F < -th events = np.where(np.diff(np.logical_and(L[:, 0], L[:, 1])) == 1)[0] times = [LoadCellDf.iloc[int(i)]['t'] for i in events] # histogram of pushes pre vs pushes post trial available trial_times = bhv.get_events_from_name(LogDf, 'TRIAL_AVAILABLE_EVENT')['t'].values post = [] pre = [] for t in trial_times: dt = times - t try: post.append(np.min(dt[dt > 0])) except ValueError: # thrown when no more pushes after last init pass try: pre.append(np.min(-1 * dt[dt < 0])) except ValueError: # thrown when no pushes before first init pass fig, axes = plt.subplots() bins = np.linspace(0, 5000, 25) axes.hist(pre, bins=bins, alpha=0.5, label='pre') axes.hist(post, bins=bins, alpha=0.5, label='post') axes.set_xlabel('time (ms)') axes.set_ylabel('count') axes.legend() Session = utils.Session(session_folder) Animal = utils.Animal(session_folder.parent) title = ' - '.join( [Animal.display(), Session.date, 'day: %s' % Session.day]) sns.despine(fig) fig.suptitle(title) fig.tight_layout() fig.subplots_adjust(top=0.85) if save is not None: os.makedirs(save.parent, exist_ok=True) plt.savefig(save, dpi=600) plt.close(fig)