Ejemplo n.º 1
0
    def onTimer(self, event):
        ''' Main timer 
        for processing messages from modules
        and updating running time on the main window
        '''
        if DEBUG: print('CATOSFrame.onTimer()')

        ### processing message
        msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)

        ### update log file path, if necessary
        lfd = path.basename(self.log_file_path)[-6:-4]  # date of the current
        # log file name
        _d = '%.2i' % (datetime.now().day)  # current date
        if lfd != _d:
            # reset the log file path
            self.log_file_path = get_log_file_path(self.output_folder)

        ### update several running time
        e_time = time() - self.program_start_time
        self.sTxt_pr_time.SetLabel(
            str(timedelta(seconds=e_time)).split('.')[0])
        if self.session_start_time != -1:
            e_time = time() - self.session_start_time
            self.sTxt_s_time.SetLabel(
                str(timedelta(seconds=e_time)).split('.')[0])
        if self.last_play_time != -1:
            e_time = time() - self.last_play_time
            self.sTxt_time.SetLabel(
                str(timedelta(seconds=e_time)).split('.')[0])
Ejemplo n.º 2
0
    def onTimer(self, event):
        ''' Main timer 
        for processing messages from modules
        and updating running time on the main window
        '''
        ### processing message
        msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)

        ### update log file path, if necessary
        lfd = path.basename(self.log_file_path)[-6:-4] # date of the current log file name
        _d = '%.2i'%(datetime.now().day) # current date
        if lfd != _d: self.log_file_path = get_log_file_path(self.output_folder) # reset the log file path

        ### update several running time
        e_time = time() - self.program_start_time
        self.sTxt_pr_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
        if self.session_start_time != -1:
            e_time = time() - self.session_start_time
            self.sTxt_s_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
        if self.last_play_time != -1:
            e_time = time() - self.last_play_time
            self.sTxt_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
Ejemplo n.º 3
0
    def run(self, flag_chk_cam_view=False, flag_feeder=False):
        fSz = self.fSize # frame size
        msg = ''
        fps=0; prev_fps=[]; prev_fps_time=time()
        mod_name = 'videoIn-%i'%(self.cam_idx)
        first_run = True
        recent_imgs = [] # buffer to store 60 recent frames
        recent_m = [] # storing whether meaningful movements 
          # happened in the recent 60 frames
        recent_m_time = -1 # time when movements were enough 
          # to start video recording
        log = "%s, [%s],"%(get_time_stamp(), mod_name)
        log += " webcam %i starts."%(self.cam_idx)
        log += " Frame-size: %s\n"%(str(fSz))
        writeFile(self.parent.log_file_path, log)
        sleep(1)
        for i in range(10):
            ret, frame_arr = self.cap_cam.read() # retrieve some images 
              # giving some time to camera to adjust
        ### find ROI with red color 
        ###   (red tape is attached on bottom of side monitors)
        r = (0, 0) + fSz # rect to find the color
        HSV_min = (175,100,90)
        HSV_max = (180,255,255)
        red_col = self.find_color(r, frame_arr, HSV_min, HSV_max, (0,0,0))
        wr, rects = self.chk_contours(red_col, self.contour_threshold)
        if wr == (-1,-1,0,0):
            writeFile(self.parent.log_file_path, "%s, [%s], Red color detection failed.\n"%(get_time_stamp(), mod_name))
            redY = -1
        else:
            redY = int(wr[1]+wr[3]/2) # middle y position of red tape
        bgImg = frame_arr.copy() # store background image
        while True:
            fps, prev_fps, prev_fps_time = chk_fps(mod_name, 
                                                   fps, 
                                                   prev_fps, 
                                                   prev_fps_time, 
                                                   self.parent.log_file_path)
            ret, frame_arr = self.cap_cam.read() # get a new frame
            if ret == False: sleep(0.1); continue

            recent_imgs.append(frame_arr)
            if len(recent_imgs) > 60: recent_imgs.pop(0)
            recent_m.append(False)
            if len(recent_m) > 60: recent_m.pop(0)

            if flag_chk_cam_view == False:
                ### extract subject image by obtaining difference image 
                ###   between the frame_arr and bgImg
                diff = cv2.absdiff(frame_arr, bgImg) 
                diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY)
                __, diff = cv2.threshold(diff, 50, 255, cv2.THRESH_BINARY)
                kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
                diff = cv2.morphologyEx(diff, 
                                        cv2.MORPH_OPEN, 
                                        kernel, 
                                        iterations=1) # decrease noise and
                                                      # minor features
                #M = cv2.moments(diff)
                #print self.cam_idx, M['m00']/255
                diff = cv2.Canny(diff, 150, 150)
                sbr, rects = self.chk_contours(diff.copy(), 20) # sbr = subject
                                                                # bounding rect
                if sbr != (-1,-1,0,0):
                    cv2.rectangle(frame_arr, 
                                  sbr[:2], 
                                  (sbr[0]+sbr[2],sbr[1]+sbr[3]), 
                                  (0,255,0), 
                                  2)
                    dist_to_s = sbr[1]-redY # distance from red tape(screen) 
                      # to the subject 
                    msg = None
                    if self.cam_idx == 1: # center screen
                        if dist_to_s < 10: msg = 'center'
                        else:
                            sMid = int(sbr[0] + sbr[2]/2)
                            if sMid < int(fSz[0]/6): msg='left'
                            elif sMid > int(fSz[0]-fSz[0]/6): msg='right'
                    else:
                        if dist_to_s < 100: # close to the screen
                            if self.cam_idx == 0: msg='left'
                            else: msg='right'
                            
                    if msg != None and self.parent.mods["session_mngr"] != None:
                        self.parent.mods["session_mngr"].msg_q.put(
                                        "%s/close_to_screen/%s"%(mod_name,msg), 
                                        True, 
                                        None
                                        )
                # red color bottom line
                cv2.line(frame_arr, (0,redY), (640,redY), (0,255,255), 2) 
            else: # chk_cam_view
                pass
                                
            if self.flagWindow:
                if flag_chk_cam_view:
                    cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
                else:
                    cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
            
            cv2.waitKey(5)
            # listen to a message
            msg_src, msg_body, msg_details = chk_msg_q(self.msg_q) 
            if msg_body == 'quit': break
            
        self.cap_cam.release()
        if self.flagWindow: cv2.destroyWindow("CATOS_CAM%.2i"%(self.cam_idx))
        log = "%s, [%s],"%(get_time_stamp(), mod_name)
        log += " webcam %i stopped.\n"%(self.cam_idx)
        writeFile(self.parent.log_file_path, log)
        if self.video_rec != None: self.video_rec.release()
Ejemplo n.º 4
0
    def run(self, flag_chk_cam_view=False, flag_feeder=False):
        msg = ''
        fps=0; prev_fps=[]; prev_fps_time=time()
        mod_name = 'videoIn'
        first_run = True
        # average NMC ( Number of Movement Contours, Image moment is not used. Image moment will be much bigger if a monkey is closer to the webcam )
        # average distance between movement contours
        # average CMC ( center of movement contours : (int(sum(contours.X) / NMC), int(sum(contours.Y) / NMC)) )
        # length of video
        nmc = 0 # Number of movement contours
        dist_b_mc = [] # average Distance between movement contours
        cmcX = [] # X-pos of average Center of movement contours
        cmcY = [] # Y-pos of average Center of movement contours
        movLog_fp = path.join( self.parent.output_folder, '%s_MovLog.txt'%(get_time_stamp()) ) # movement log file
        if flag_chk_cam_view == False:
            f = open(movLog_fp, 'w')
            f.write('timestamp, sum.NMC, avg.Dist_b_MC, avg.CMC-X, avg.CMC-Y\n')
            f.close()
        writeFile(self.parent.log_file_path, '%s, [%s], webcam %i starts. Frame-size: %s\n'%(get_time_stamp(), mod_name, self.cam_idx, str(self.fSize)))

        # Wait for a few seconds while retreiving webcam images
        # (When webcam is initialized, retreived images change at the beginning, 
        # and it's recognized as movements.)
        func_init_time = time()
        while time()-func_init_time < 1:
            ret, frame_arr = self.cap_cam.read() # get a new frame
            cv2.waitKey(100)
        last_mov_log_time = time()
        while True:
            fps, prev_fps, prev_fps_time = chk_fps(mod_name, fps, prev_fps, prev_fps_time, self.parent.log_file_path)

            ret, frame_arr = self.cap_cam.read() # get a new frame
            if ret == False: sleep(0.1); continue

            if flag_chk_cam_view == False:
                grey_img = cv2.cvtColor(frame_arr, cv2.COLOR_RGB2GRAY) # grey image
                grey_img = self.preprocessing(grey_img) # preprocess the grey image
                
                ### leave only the area surrounded by three screens
                mask = np.zeros( (grey_img.shape[0], grey_img.shape[1]) , dtype=np.uint8 )
                cv2.fillConvexPoly(mask, np.asarray(self.roi_pts), 255)
                grey_img = cv2.bitwise_and( grey_img, grey_img, mask=mask )
                
                ### processing of motion around screens
                if first_run == True:
                    first_run = False
                    grey_avg = cv2.convertScaleAbs(grey_img)
                    grey_avg = grey_avg.astype(np.float32)
                else:
                    cv2.accumulateWeighted(grey_img, grey_avg, 0.8)

                ### contour of movements
                grey_tmp = cv2.convertScaleAbs(grey_avg)
                grey_diff = cv2.absdiff(grey_img, grey_tmp)
                grey_diff = cv2.Canny(grey_diff, 10, 15)
                wrect, rects = self.chk_contours(grey_diff, self.contour_threshold)
                
                if (self.cam_idx in self.parent.cam_idx) and (rects != []) and (self.m_wrectTh[0] < wrect[2]+wrect[3] < self.m_wrectTh[1]):
                # if this is a cam for watching subject and there's a meaningful movement
                    nmc += len(rects)
                    sumX = 0; sumY = 0
                    sum_dist_b_mc = 0
                    for ri in range(len(rects)):
                        _r = rects[ri]
                        _x = _r[0]+_r[2]/2; _y = _r[1]+_r[3]/2
                        cv2.circle(grey_img, (_x,_y), 5, 200, 2)
                        if ri > 0:
                            _pr = rects[ri-1]
                            _x2 = _pr[0]+_pr[2]/2; _y2 = _pr[1]+_pr[3]/2
                            cv2.line(grey_img, (_x,_y), (_x2,_y2), 200, 1)
                            sum_dist_b_mc += np.sqrt( abs(_x-_x2)**2 + abs(_y-_y2)**2 ) 
                        sumX += _x; sumY += _y
                        #cv2.rectangle(grey_img, (_r[0],_r[1]), (_r[0]+_r[2],_r[1]+_r[3]), 255, 1)
                    avgX = sumX/len(rects); avgY = sumY/len(rects)
                    cmcX.append(avgX); cmcY.append(avgY)
                    dist_b_mc.append(sum_dist_b_mc/len(rects))
                else: # there's no meaningful movement
                    pass 
                if time()-last_mov_log_time > 10: # every 10 seconds
                    ### record the movement data
                    f = open(movLog_fp, 'a')
                    if nmc > 0:
                        f.write( '%s, %i, %i, %i, %i\n'%(get_time_stamp(), nmc, int(np.average(dist_b_mc)), int(np.average(cmcX)), int(np.average(cmcY))) )
                    else:
                        f.write( '%s, 0, 0, 0, 0\n'%(get_time_stamp()) )
                    f.close()
                    nmc=0; dist_b_mc=[]; cmcX=[]; cmcY=[] # init
                    last_mov_log_time = time()
                            
            else: # chk_cam_view
                ### draw ROI lines
                for i in range(len(self.roi_pts)):
                    pt1 = self.roi_pts[(i-1)]
                    pt2 = self.roi_pts[i]
                    cv2.line(frame_arr, pt1, pt2, (0,0,255), 2)
            
            if flag_window == True:
                if flag_chk_cam_view == True: cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
                else: cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), grey_img)
            cv2.waitKey(5)
            msg_src, msg_body, msg_details = chk_msg_q(self.msg_q) # listen to a message
            if msg_body == 'quit': break
            
        self.cap_cam.release()
        if flag_window == True: cv2.destroyWindow("CATOS_CAM%.2i"%(self.cam_idx))
        log_ = '%s, [%s], webcam %i stopped.\n'%(get_time_stamp(), mod_name, self.cam_idx)
        writeFile(self.parent.log_file_path, log_)
Ejemplo n.º 5
0
    def onTimer(self, event):
        ''' Timer for checking message and processing with the current state
        '''
        ### retrieve messages
        flag_foMove = False
        flag_stim_touched = False
        close_to_rScreen = False
        close_to_cScreen = False
        close_to_lScreen = False
        while not self.msg_q.empty():
            # listen to a message
            msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)
            if msg_body == 'foMove':
                flag_foMove = True
                fo_pos = [
                    int(msg_details[0]),
                    int(msg_details[1]),
                    int(msg_details[2])
                ]
            elif msg_body == 'close_to_screen':
                # movement happened around a screen
                if msg_details[0] == 'left': close_to_lScreen = True
                if msg_details[0] == 'center': close_to_cScreen = True
                if msg_details[0] == 'right': close_to_rScreen = True
            elif msg_body == 'stim_touched':
                flag_stim_touched = True

        ### processing with the current state and received messages
        if self.state == 'inTrial':
            if flag_foMove == True:
                # group FOs is located in a differect section of the screen,
                # sound source should move accordingly
                if self.session_type == 'immersion':
                    # move sound source position
                    self.parent.mods["audioOut"].move(fo_pos)

            if flag_stim_touched == True:  # stimulus was touched
                if self.session_type == 'immersion':
                    self.parent.mods["audioOut"].stop()
                    # play positive feedback sound
                    self.parent.mods["audioOut"].play(1, False)
                if self.parent.mods["videoOut"].timer != None:
                    self.parent.mods["videoOut"].timer.Stop()
                self.parent.mods["videoOut"].flag_trial = False
                self.parent.mods["videoOut"].panel.Refresh()
                #self.parent.stop_mods(mod='videoIn') # stop webcam
                writeFile(
                    self.parent.log_file_path,
                    '%s, [session_mngr], Trial finished.\n' %
                    (get_time_stamp()))
                self.parent.mods["arduino"].send("feed".encode())
                writeFile(
                    self.parent.log_file_path,
                    '%s, [session_mngr], Feed message sent.\n' %
                    (get_time_stamp()))
                self.state = 'pause'  # pause for ITI
                wx.CallLater(self.ITI, self.init_trial)

            if self.session_type == 'feed':
                if time() - self.last_feed_time >= self.feed_intv:
                    self.parent.mods["arduino"].send("feed".encode())
                    log = "%s, [session_mngr]," % (get_time_stamp())
                    log += "Feed message sent.\n"
                    writeFile(self.parent.log_file_path, log)
                    self.state = 'pause'  # pause for ITI
                    wx.CallLater(self.ITI, self.init_trial)
                    self.last_feed_time = time()

            if self.session_type == 'immersion':
                if close_to_lScreen or close_to_cScreen or close_to_rScreen:
                    # subject is close to a screen
                    voMod = self.parent.mods["videoOut"]
                    screenCtr = voMod.wSize[0] / 2
                    dest = (randint(voMod.ctr_rect[0], voMod.ctr_rect[2]),
                            randint(voMod.ctr_rect[1], voMod.ctr_rect[3]))
                    #print close_to_lScreen, close_to_cScreen, close_to_rScreen
                    if (close_to_rScreen and \
                            voMod.gctr[0] > (voMod.s_w[0]+voMod.s_w[1])) or \
                       (close_to_lScreen and \
                            voMod.gctr[0] < voMod.s_w[0]):
                        steps = randint(30, 40)  # determine steps to travel
                        # (shorter = faster movement)
                        ### set a new destination which should be
                        ###   in the center screen / line, not a curve
                        for i in xrange(len(voMod.fo)):
                            orig_ = (voMod.fo[i]['track'][0][0],
                                     voMod.fo[i]['track'][0][1])
                            dest_ = (randint(dest[0] - 50, dest[0] + 50),
                                     randint(dest[1] - 50, dest[1] + 50))
                            tl = []
                            xstep = int(abs(dest_[0] - orig_[0]) / steps)
                            ystep = int(abs(dest_[1] - orig_[1]) / steps)
                            for j in xrange(steps):
                                if orig_[0] < dest_[0]:
                                    x = orig_[0] + xstep * j
                                else:
                                    x = orig_[0] - xstep * j
                                if orig_[1] < dest_[1]:
                                    y = orig_[1] + ystep * j
                                else:
                                    y = orig_[1] - ystep * j
                                tl.append((x, y))
                            voMod.fo[i]['track'] = tl
                    elif close_to_cScreen and \
                      (voMod.s_w[0] <= voMod.gctr[0] <= voMod.s_w[0]+voMod.s_w[1]):
                        # subject is close to the center screen and
                        # the stimulus is already on the screen.
                        if self.ctrFOResetTime == -1 or \
                          (time()-self.ctrFOResetTime)>1:
                            # FO track was set before a half second ago.
                            steps = randint(70, 120)  # steps to travel
                            # (shorter = faster movement)
                            h1 = (randint(voMod.ctr_rect[0] - 50,
                                          voMod.ctr_rect[2] + 50),
                                  randint(voMod.ctr_rect[1] - 50,
                                          voMod.ctr_rect[3] + 50))
                            h2 = (randint(voMod.ctr_rect[0] - 50,
                                          voMod.ctr_rect[2] + 50),
                                  randint(voMod.ctr_rect[1] - 50,
                                          voMod.ctr_rect[3] + 50))
                            for i in xrange(len(voMod.fo)):
                                orig_ = (voMod.fo[i]['track'][0][0],
                                         voMod.fo[i]['track'][0][1])
                                dest_ = (randint(dest[0] - 50, dest[0] + 50),
                                         randint(dest[1] - 50, dest[1] + 50))
                                h1_ = (randint(h1[0] - 50, h1[0] + 50),
                                       randint(h1[1] - 50, h1[1] + 50))
                                h2_ = (randint(h2[0] - 50, h2[0] + 50),
                                       randint(h2[1] - 50, h2[1] + 50))
                                voMod.fo[i]['track'] = make_b_curve_coord(
                                    orig_, h1_, h2_, dest_, steps)
                                self.ctrFOResetTime = time()
Ejemplo n.º 6
0
    def run(self):
        aDataBuff = []  # buffer for audio data
        r_cnt = 0  # counting how many new frames were appended after last writing to WAV
        last_valid_time = -1  # last time when data was valid to record
        # writing to WAV file occurs once per second
        snd_file = None
        is_recording = False
        fps = 0
        prev_fps = []
        prev_fps_time = time()
        stream = self.open_mic_stream()
        writeFile(self.parent.log_file_path, "%s, [audioIn], 'run' starts.\n" % (get_time_stamp()))
        num_of_IOErr = 0
        while True:
            fps, prev_fps, prev_fps_time = chk_fps("audioIn", fps, prev_fps, prev_fps_time, self.parent.log_file_path)

            msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)  # listen to a message
            if msg_src == "main":
                if msg_body == "quit":
                    if is_recording == True and r_cnt > 0:
                        snd_file = self.finish_rec(aDataBuff, snd_file, r_cnt, prev_fps)
                        is_recording = False
                    break

            try:
                ### get audio data
                aDataBuff.append(
                    np.fromstring(
                        stream.read(self.rp["input_frames_per_block"], exception_on_overflow=False), dtype=np.short
                    ).tolist()
                )
                if len(aDataBuff) > self.buff_sz:
                    aDataBuff.pop(0)

                ### record to file
                if is_recording == True:
                    r_cnt += 1
                    if r_cnt > (self.fps * 2):
                        snd_file.writeframes(np.array(aDataBuff[-(self.fps * 2) :], dtype=np.int16).tostring())
                        r_cnt = 0

                ### check data to record
                _fData = np.asarray(abs(np.fft.fft(aDataBuff[-1]))[: self.rp["input_frames_per_block"] / 2])
                _d = _fData / self.dMax * 100  # data range 0~100
                _d = _d[self.cutoff_hz / self.rp["freq_res"] :]  # cut off low frequency data
                if np.sum(_d) > _d.shape[0] and np.average(_d) > (np.median(_d) * 1.5):
                    # Sum of data is bigger than the length of data : each data is bigger than 1 on average
                    # Average is bigger than median*1.5 : amplitude is more concentrated in some areas
                    last_valid_time = time()
                    if is_recording == False:  # not recording
                        ### start recording
                        is_recording = True
                        r_cnt = 0
                        n_ = datetime.now()
                        folder = path.join(self.parent.output_folder, "%.4i_%.2i_%.2i" % (n_.year, n_.month, n_.day))
                        if path.isdir(folder) == False:
                            mkdir(folder)
                        wav_fp = path.join(folder, "%s.wav" % (get_time_stamp()))
                        snd_file = wave.open(wav_fp, "wb")
                        snd_file.setparams(
                            (
                                self.rp["channels"],
                                self.rp["sampWidth"],
                                self.rp["sampleRate"],
                                0,
                                "NONE",
                                "noncompressed",
                            )
                        )
                        snd_file.writeframes(np.array(aDataBuff[-(self.fps * 2) :], dtype=np.int16).tostring())
                        writeFile(
                            self.parent.log_file_path,
                            "%s, [audioIn], start to write WAV, %s.\n" % (get_time_stamp(), wav_fp),
                        )
                else:
                    if is_recording == True:  # currently recording
                        if (
                            time() - last_valid_time > self.stop_latency
                        ):  # there was no valid data to record for some time
                            ### stop recording
                            is_recording = False
                            snd_file = self.finish_rec(aDataBuff, snd_file, r_cnt, prev_fps)

            except IOError, e:
                if num_of_IOErr < 10:
                    msg_ = "%s, [audioIn], IOError : %s\n" % (get_time_stamp(), e)
                    writeFile(self.parent.log_file_path, msg_)
                num_of_IOErr += 1
                sleep(self.input_block_time / 2)