def onTimer(self, event):
''' Main timer
for processing messages from modules
and updating running time on the main window
'''
if DEBUG: print('CATOSFrame.onTimer()')
### processing message
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)
### update log file path, if necessary
lfd = path.basename(self.log_file_path)[-6:-4] # date of the current
# log file name
_d = '%.2i' % (datetime.now().day) # current date
if lfd != _d:
# reset the log file path
self.log_file_path = get_log_file_path(self.output_folder)
### update several running time
e_time = time() - self.program_start_time
self.sTxt_pr_time.SetLabel(
str(timedelta(seconds=e_time)).split('.')[0])
if self.session_start_time != -1:
e_time = time() - self.session_start_time
self.sTxt_s_time.SetLabel(
str(timedelta(seconds=e_time)).split('.')[0])
if self.last_play_time != -1:
e_time = time() - self.last_play_time
self.sTxt_time.SetLabel(
str(timedelta(seconds=e_time)).split('.')[0])
def onTimer(self, event):
''' Main timer
for processing messages from modules
and updating running time on the main window
'''
### processing message
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)
### update log file path, if necessary
lfd = path.basename(self.log_file_path)[-6:-4] # date of the current log file name
_d = '%.2i'%(datetime.now().day) # current date
if lfd != _d: self.log_file_path = get_log_file_path(self.output_folder) # reset the log file path
### update several running time
e_time = time() - self.program_start_time
self.sTxt_pr_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
if self.session_start_time != -1:
e_time = time() - self.session_start_time
self.sTxt_s_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
if self.last_play_time != -1:
e_time = time() - self.last_play_time
self.sTxt_time.SetLabel( str(timedelta(seconds=e_time)).split('.')[0] )
def run(self, flag_chk_cam_view=False, flag_feeder=False):
fSz = self.fSize # frame size
msg = ''
fps=0; prev_fps=[]; prev_fps_time=time()
mod_name = 'videoIn-%i'%(self.cam_idx)
first_run = True
recent_imgs = [] # buffer to store 60 recent frames
recent_m = [] # storing whether meaningful movements
# happened in the recent 60 frames
recent_m_time = -1 # time when movements were enough
# to start video recording
log = "%s, [%s],"%(get_time_stamp(), mod_name)
log += " webcam %i starts."%(self.cam_idx)
log += " Frame-size: %s\n"%(str(fSz))
writeFile(self.parent.log_file_path, log)
sleep(1)
for i in range(10):
ret, frame_arr = self.cap_cam.read() # retrieve some images
# giving some time to camera to adjust
### find ROI with red color
### (red tape is attached on bottom of side monitors)
r = (0, 0) + fSz # rect to find the color
HSV_min = (175,100,90)
HSV_max = (180,255,255)
red_col = self.find_color(r, frame_arr, HSV_min, HSV_max, (0,0,0))
wr, rects = self.chk_contours(red_col, self.contour_threshold)
if wr == (-1,-1,0,0):
writeFile(self.parent.log_file_path, "%s, [%s], Red color detection failed.\n"%(get_time_stamp(), mod_name))
redY = -1
else:
redY = int(wr[1]+wr[3]/2) # middle y position of red tape
bgImg = frame_arr.copy() # store background image
while True:
fps, prev_fps, prev_fps_time = chk_fps(mod_name,
fps,
prev_fps,
prev_fps_time,
self.parent.log_file_path)
ret, frame_arr = self.cap_cam.read() # get a new frame
if ret == False: sleep(0.1); continue
recent_imgs.append(frame_arr)
if len(recent_imgs) > 60: recent_imgs.pop(0)
recent_m.append(False)
if len(recent_m) > 60: recent_m.pop(0)
if flag_chk_cam_view == False:
### extract subject image by obtaining difference image
### between the frame_arr and bgImg
diff = cv2.absdiff(frame_arr, bgImg)
diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY)
__, diff = cv2.threshold(diff, 50, 255, cv2.THRESH_BINARY)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
diff = cv2.morphologyEx(diff,
cv2.MORPH_OPEN,
kernel,
iterations=1) # decrease noise and
# minor features
#M = cv2.moments(diff)
#print self.cam_idx, M['m00']/255
diff = cv2.Canny(diff, 150, 150)
sbr, rects = self.chk_contours(diff.copy(), 20) # sbr = subject
# bounding rect
if sbr != (-1,-1,0,0):
cv2.rectangle(frame_arr,
sbr[:2],
(sbr[0]+sbr[2],sbr[1]+sbr[3]),
(0,255,0),
2)
dist_to_s = sbr[1]-redY # distance from red tape(screen)
# to the subject
msg = None
if self.cam_idx == 1: # center screen
if dist_to_s < 10: msg = 'center'
else:
sMid = int(sbr[0] + sbr[2]/2)
if sMid < int(fSz[0]/6): msg='left'
elif sMid > int(fSz[0]-fSz[0]/6): msg='right'
else:
if dist_to_s < 100: # close to the screen
if self.cam_idx == 0: msg='left'
else: msg='right'
if msg != None and self.parent.mods["session_mngr"] != None:
self.parent.mods["session_mngr"].msg_q.put(
"%s/close_to_screen/%s"%(mod_name,msg),
True,
None
)
# red color bottom line
cv2.line(frame_arr, (0,redY), (640,redY), (0,255,255), 2)
else: # chk_cam_view
pass
if self.flagWindow:
if flag_chk_cam_view:
cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
else:
cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
cv2.waitKey(5)
# listen to a message
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)
if msg_body == 'quit': break
self.cap_cam.release()
if self.flagWindow: cv2.destroyWindow("CATOS_CAM%.2i"%(self.cam_idx))
log = "%s, [%s],"%(get_time_stamp(), mod_name)
log += " webcam %i stopped.\n"%(self.cam_idx)
writeFile(self.parent.log_file_path, log)
if self.video_rec != None: self.video_rec.release()
def run(self, flag_chk_cam_view=False, flag_feeder=False):
msg = ''
fps=0; prev_fps=[]; prev_fps_time=time()
mod_name = 'videoIn'
first_run = True
# average NMC ( Number of Movement Contours, Image moment is not used. Image moment will be much bigger if a monkey is closer to the webcam )
# average distance between movement contours
# average CMC ( center of movement contours : (int(sum(contours.X) / NMC), int(sum(contours.Y) / NMC)) )
# length of video
nmc = 0 # Number of movement contours
dist_b_mc = [] # average Distance between movement contours
cmcX = [] # X-pos of average Center of movement contours
cmcY = [] # Y-pos of average Center of movement contours
movLog_fp = path.join( self.parent.output_folder, '%s_MovLog.txt'%(get_time_stamp()) ) # movement log file
if flag_chk_cam_view == False:
f = open(movLog_fp, 'w')
f.write('timestamp, sum.NMC, avg.Dist_b_MC, avg.CMC-X, avg.CMC-Y\n')
f.close()
writeFile(self.parent.log_file_path, '%s, [%s], webcam %i starts. Frame-size: %s\n'%(get_time_stamp(), mod_name, self.cam_idx, str(self.fSize)))
# Wait for a few seconds while retreiving webcam images
# (When webcam is initialized, retreived images change at the beginning,
# and it's recognized as movements.)
func_init_time = time()
while time()-func_init_time < 1:
ret, frame_arr = self.cap_cam.read() # get a new frame
cv2.waitKey(100)
last_mov_log_time = time()
while True:
fps, prev_fps, prev_fps_time = chk_fps(mod_name, fps, prev_fps, prev_fps_time, self.parent.log_file_path)
ret, frame_arr = self.cap_cam.read() # get a new frame
if ret == False: sleep(0.1); continue
if flag_chk_cam_view == False:
grey_img = cv2.cvtColor(frame_arr, cv2.COLOR_RGB2GRAY) # grey image
grey_img = self.preprocessing(grey_img) # preprocess the grey image
### leave only the area surrounded by three screens
mask = np.zeros( (grey_img.shape[0], grey_img.shape[1]) , dtype=np.uint8 )
cv2.fillConvexPoly(mask, np.asarray(self.roi_pts), 255)
grey_img = cv2.bitwise_and( grey_img, grey_img, mask=mask )
### processing of motion around screens
if first_run == True:
first_run = False
grey_avg = cv2.convertScaleAbs(grey_img)
grey_avg = grey_avg.astype(np.float32)
else:
cv2.accumulateWeighted(grey_img, grey_avg, 0.8)
### contour of movements
grey_tmp = cv2.convertScaleAbs(grey_avg)
grey_diff = cv2.absdiff(grey_img, grey_tmp)
grey_diff = cv2.Canny(grey_diff, 10, 15)
wrect, rects = self.chk_contours(grey_diff, self.contour_threshold)
if (self.cam_idx in self.parent.cam_idx) and (rects != []) and (self.m_wrectTh[0] < wrect[2]+wrect[3] < self.m_wrectTh[1]):
# if this is a cam for watching subject and there's a meaningful movement
nmc += len(rects)
sumX = 0; sumY = 0
sum_dist_b_mc = 0
for ri in range(len(rects)):
_r = rects[ri]
_x = _r[0]+_r[2]/2; _y = _r[1]+_r[3]/2
cv2.circle(grey_img, (_x,_y), 5, 200, 2)
if ri > 0:
_pr = rects[ri-1]
_x2 = _pr[0]+_pr[2]/2; _y2 = _pr[1]+_pr[3]/2
cv2.line(grey_img, (_x,_y), (_x2,_y2), 200, 1)
sum_dist_b_mc += np.sqrt( abs(_x-_x2)**2 + abs(_y-_y2)**2 )
sumX += _x; sumY += _y
#cv2.rectangle(grey_img, (_r[0],_r[1]), (_r[0]+_r[2],_r[1]+_r[3]), 255, 1)
avgX = sumX/len(rects); avgY = sumY/len(rects)
cmcX.append(avgX); cmcY.append(avgY)
dist_b_mc.append(sum_dist_b_mc/len(rects))
else: # there's no meaningful movement
pass
if time()-last_mov_log_time > 10: # every 10 seconds
### record the movement data
f = open(movLog_fp, 'a')
if nmc > 0:
f.write( '%s, %i, %i, %i, %i\n'%(get_time_stamp(), nmc, int(np.average(dist_b_mc)), int(np.average(cmcX)), int(np.average(cmcY))) )
else:
f.write( '%s, 0, 0, 0, 0\n'%(get_time_stamp()) )
f.close()
nmc=0; dist_b_mc=[]; cmcX=[]; cmcY=[] # init
last_mov_log_time = time()
else: # chk_cam_view
### draw ROI lines
for i in range(len(self.roi_pts)):
pt1 = self.roi_pts[(i-1)]
pt2 = self.roi_pts[i]
cv2.line(frame_arr, pt1, pt2, (0,0,255), 2)
if flag_window == True:
if flag_chk_cam_view == True: cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), frame_arr)
else: cv2.imshow("CATOS_CAM%.2i"%(self.cam_idx), grey_img)
cv2.waitKey(5)
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q) # listen to a message
if msg_body == 'quit': break
self.cap_cam.release()
if flag_window == True: cv2.destroyWindow("CATOS_CAM%.2i"%(self.cam_idx))
log_ = '%s, [%s], webcam %i stopped.\n'%(get_time_stamp(), mod_name, self.cam_idx)
writeFile(self.parent.log_file_path, log_)
def onTimer(self, event):
''' Timer for checking message and processing with the current state
'''
### retrieve messages
flag_foMove = False
flag_stim_touched = False
close_to_rScreen = False
close_to_cScreen = False
close_to_lScreen = False
while not self.msg_q.empty():
# listen to a message
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q)
if msg_body == 'foMove':
flag_foMove = True
fo_pos = [
int(msg_details[0]),
int(msg_details[1]),
int(msg_details[2])
]
elif msg_body == 'close_to_screen':
# movement happened around a screen
if msg_details[0] == 'left': close_to_lScreen = True
if msg_details[0] == 'center': close_to_cScreen = True
if msg_details[0] == 'right': close_to_rScreen = True
elif msg_body == 'stim_touched':
flag_stim_touched = True
### processing with the current state and received messages
if self.state == 'inTrial':
if flag_foMove == True:
# group FOs is located in a differect section of the screen,
# sound source should move accordingly
if self.session_type == 'immersion':
# move sound source position
self.parent.mods["audioOut"].move(fo_pos)
if flag_stim_touched == True: # stimulus was touched
if self.session_type == 'immersion':
self.parent.mods["audioOut"].stop()
# play positive feedback sound
self.parent.mods["audioOut"].play(1, False)
if self.parent.mods["videoOut"].timer != None:
self.parent.mods["videoOut"].timer.Stop()
self.parent.mods["videoOut"].flag_trial = False
self.parent.mods["videoOut"].panel.Refresh()
#self.parent.stop_mods(mod='videoIn') # stop webcam
writeFile(
self.parent.log_file_path,
'%s, [session_mngr], Trial finished.\n' %
(get_time_stamp()))
self.parent.mods["arduino"].send("feed".encode())
writeFile(
self.parent.log_file_path,
'%s, [session_mngr], Feed message sent.\n' %
(get_time_stamp()))
self.state = 'pause' # pause for ITI
wx.CallLater(self.ITI, self.init_trial)
if self.session_type == 'feed':
if time() - self.last_feed_time >= self.feed_intv:
self.parent.mods["arduino"].send("feed".encode())
log = "%s, [session_mngr]," % (get_time_stamp())
log += "Feed message sent.\n"
writeFile(self.parent.log_file_path, log)
self.state = 'pause' # pause for ITI
wx.CallLater(self.ITI, self.init_trial)
self.last_feed_time = time()
if self.session_type == 'immersion':
if close_to_lScreen or close_to_cScreen or close_to_rScreen:
# subject is close to a screen
voMod = self.parent.mods["videoOut"]
screenCtr = voMod.wSize[0] / 2
dest = (randint(voMod.ctr_rect[0], voMod.ctr_rect[2]),
randint(voMod.ctr_rect[1], voMod.ctr_rect[3]))
#print close_to_lScreen, close_to_cScreen, close_to_rScreen
if (close_to_rScreen and \
voMod.gctr[0] > (voMod.s_w[0]+voMod.s_w[1])) or \
(close_to_lScreen and \
voMod.gctr[0] < voMod.s_w[0]):
steps = randint(30, 40) # determine steps to travel
# (shorter = faster movement)
### set a new destination which should be
### in the center screen / line, not a curve
for i in xrange(len(voMod.fo)):
orig_ = (voMod.fo[i]['track'][0][0],
voMod.fo[i]['track'][0][1])
dest_ = (randint(dest[0] - 50, dest[0] + 50),
randint(dest[1] - 50, dest[1] + 50))
tl = []
xstep = int(abs(dest_[0] - orig_[0]) / steps)
ystep = int(abs(dest_[1] - orig_[1]) / steps)
for j in xrange(steps):
if orig_[0] < dest_[0]:
x = orig_[0] + xstep * j
else:
x = orig_[0] - xstep * j
if orig_[1] < dest_[1]:
y = orig_[1] + ystep * j
else:
y = orig_[1] - ystep * j
tl.append((x, y))
voMod.fo[i]['track'] = tl
elif close_to_cScreen and \
(voMod.s_w[0] <= voMod.gctr[0] <= voMod.s_w[0]+voMod.s_w[1]):
# subject is close to the center screen and
# the stimulus is already on the screen.
if self.ctrFOResetTime == -1 or \
(time()-self.ctrFOResetTime)>1:
# FO track was set before a half second ago.
steps = randint(70, 120) # steps to travel
# (shorter = faster movement)
h1 = (randint(voMod.ctr_rect[0] - 50,
voMod.ctr_rect[2] + 50),
randint(voMod.ctr_rect[1] - 50,
voMod.ctr_rect[3] + 50))
h2 = (randint(voMod.ctr_rect[0] - 50,
voMod.ctr_rect[2] + 50),
randint(voMod.ctr_rect[1] - 50,
voMod.ctr_rect[3] + 50))
for i in xrange(len(voMod.fo)):
orig_ = (voMod.fo[i]['track'][0][0],
voMod.fo[i]['track'][0][1])
dest_ = (randint(dest[0] - 50, dest[0] + 50),
randint(dest[1] - 50, dest[1] + 50))
h1_ = (randint(h1[0] - 50, h1[0] + 50),
randint(h1[1] - 50, h1[1] + 50))
h2_ = (randint(h2[0] - 50, h2[0] + 50),
randint(h2[1] - 50, h2[1] + 50))
voMod.fo[i]['track'] = make_b_curve_coord(
orig_, h1_, h2_, dest_, steps)
self.ctrFOResetTime = time()
def run(self):
aDataBuff = [] # buffer for audio data
r_cnt = 0 # counting how many new frames were appended after last writing to WAV
last_valid_time = -1 # last time when data was valid to record
# writing to WAV file occurs once per second
snd_file = None
is_recording = False
fps = 0
prev_fps = []
prev_fps_time = time()
stream = self.open_mic_stream()
writeFile(self.parent.log_file_path, "%s, [audioIn], 'run' starts.\n" % (get_time_stamp()))
num_of_IOErr = 0
while True:
fps, prev_fps, prev_fps_time = chk_fps("audioIn", fps, prev_fps, prev_fps_time, self.parent.log_file_path)
msg_src, msg_body, msg_details = chk_msg_q(self.msg_q) # listen to a message
if msg_src == "main":
if msg_body == "quit":
if is_recording == True and r_cnt > 0:
snd_file = self.finish_rec(aDataBuff, snd_file, r_cnt, prev_fps)
is_recording = False
break
try:
### get audio data
aDataBuff.append(
np.fromstring(
stream.read(self.rp["input_frames_per_block"], exception_on_overflow=False), dtype=np.short
).tolist()
)
if len(aDataBuff) > self.buff_sz:
aDataBuff.pop(0)
### record to file
if is_recording == True:
r_cnt += 1
if r_cnt > (self.fps * 2):
snd_file.writeframes(np.array(aDataBuff[-(self.fps * 2) :], dtype=np.int16).tostring())
r_cnt = 0
### check data to record
_fData = np.asarray(abs(np.fft.fft(aDataBuff[-1]))[: self.rp["input_frames_per_block"] / 2])
_d = _fData / self.dMax * 100 # data range 0~100
_d = _d[self.cutoff_hz / self.rp["freq_res"] :] # cut off low frequency data
if np.sum(_d) > _d.shape[0] and np.average(_d) > (np.median(_d) * 1.5):
# Sum of data is bigger than the length of data : each data is bigger than 1 on average
# Average is bigger than median*1.5 : amplitude is more concentrated in some areas
last_valid_time = time()
if is_recording == False: # not recording
### start recording
is_recording = True
r_cnt = 0
n_ = datetime.now()
folder = path.join(self.parent.output_folder, "%.4i_%.2i_%.2i" % (n_.year, n_.month, n_.day))
if path.isdir(folder) == False:
mkdir(folder)
wav_fp = path.join(folder, "%s.wav" % (get_time_stamp()))
snd_file = wave.open(wav_fp, "wb")
snd_file.setparams(
(
self.rp["channels"],
self.rp["sampWidth"],
self.rp["sampleRate"],
0,
"NONE",
"noncompressed",
)
)
snd_file.writeframes(np.array(aDataBuff[-(self.fps * 2) :], dtype=np.int16).tostring())
writeFile(
self.parent.log_file_path,
"%s, [audioIn], start to write WAV, %s.\n" % (get_time_stamp(), wav_fp),
)
else:
if is_recording == True: # currently recording
if (
time() - last_valid_time > self.stop_latency
): # there was no valid data to record for some time
### stop recording
is_recording = False
snd_file = self.finish_rec(aDataBuff, snd_file, r_cnt, prev_fps)
except IOError, e:
if num_of_IOErr < 10:
msg_ = "%s, [audioIn], IOError : %s\n" % (get_time_stamp(), e)
writeFile(self.parent.log_file_path, msg_)
num_of_IOErr += 1
sleep(self.input_block_time / 2)