def setup(self):
self.input_signal: ArrayAttribute = None
self.integration_window_width = None
psd_return_size = self.nperseg // 2 + 1
self.frequencies = ArrayAttribute('psd_frequency', (psd_return_size, ),
ArrayType.float64)
self.power = ArrayAttribute('psd_power', (psd_return_size, ),
ArrayType.float64)
def setup(self):
require_camera_device(self.camera_device_id)
# Get camera specs
camera_config = config.CONF_CAMERA_DEVICES.get(self.camera_device_id)
self.res_x = camera_config['width']
self.res_y = camera_config['height']
# Set up parameter variables (accessible externally)
self.rois = dict()
self.thresh = None
self.min_size = None
self.saccade_threshold = None
# Set up attributes
# self.data = []
for id in range(self.roi_maxnum):
roi = {}
# Rectangle
extracted_roi_name = f'{self.extracted_rect_prefix}{id}'
roi[extracted_roi_name] = ObjectAttribute(extracted_roi_name)
# Position
# LE
le_pos_name = f'{self.ang_le_pos_prefix}{id}'
roi[le_pos_name] = ArrayAttribute(le_pos_name, (1,), ArrayType.float64)
# RE
re_pos_name = f'{self.ang_re_pos_prefix}{id}'
roi[re_pos_name] = ArrayAttribute(re_pos_name, (1,), ArrayType.float64)
# Velocity
# LE
le_vel_name = f'{self.ang_le_vel_prefix}{id}'
roi[le_vel_name] = ArrayAttribute(le_vel_name, (1,), ArrayType.float64)
# RE
re_vel_name = f'{self.ang_re_vel_prefix}{id}'
roi[re_vel_name] = ArrayAttribute(re_vel_name, (1,), ArrayType.float64)
# Saccade detection
# LE
le_sacc_name = f'{self.le_sacc_prefix}{id}'
roi[le_sacc_name] = ArrayAttribute(le_sacc_name, (1,), ArrayType.float64)
# RE
re_sacc_name = f'{self.re_sacc_prefix}{id}'
roi[re_sacc_name] = ArrayAttribute(re_sacc_name, (1,), ArrayType.float64)
# self.data.append(roi)
# Set frame buffer
# self.frame = ArrayAttribute('eyeposdetect_frame', (self.res_y, self.res_x), ArrayType.uint8)
self.frame = ArrayAttribute('eyeposdetect_frame', (self.res_x, self.res_y), ArrayType.uint8)
# Set saccade trigger buffer
self.saccade_trigger = ArrayAttribute('eyeposdetect_saccade_trigger', (1, ), ArrayType.bool)
class SinesAddedWhiteNoise(IoRoutine):
def __init__(self, *args):
IoRoutine.__init__(self, *args)
def setup(self):
self.whitenoise = ArrayAttribute('test_sines_whitenoise', (1,), ArrayType.float64)
def initialize(self):
register_with_plotter('test_sines_whitenoise', axis='Sines+Noise')
self.whitenoise.add_to_file()
def main(self, *args, **kwargs):
t = get_time()
self.whitenoise.write(3. * np.random.normal() + np.sin(t * 2. * np.pi * 20) + np.sin(t * 2. * np.pi * 180))
class RandomOnOff(IoRoutine):
def __init__(self, *args):
IoRoutine.__init__(self, *args)
def setup(self):
self.test_onoff = ArrayAttribute('test_onoff', (1,), ArrayType.bool)
def initialize(self):
set_digital_output('onoff_out', 'test_onoff')
def main(self, *args, **kwargs):
t = get_time()
sig = int(t / 2) % 2
self.test_onoff.write(sig)
class WriteProtocolSync(IoRoutine):
def __init__(self):
super(WriteProtocolSync, self).__init__()
def setup(self):
self.protocol_active = ArrayAttribute('protocol_active', (1, ), ArrayType.bool)
self.phase_active = ArrayAttribute('phase_active', (1, ), ArrayType.bool)
def initialize(self):
register_with_plotter('protocol_active', axis=WriteProtocolSync.__name__)
register_with_plotter('phase_active', axis=WriteProtocolSync.__name__)
set_digital_output('protocol_active', 'protocol_active')
set_digital_output('phase_active', 'phase_active')
def main(self, *args, **kwargs):
self.phase_active.write(ipc.Process.phase_is_active)
def setup(self):
# Read all pins
self.pin_configs: Dict[str, Dict] = {}
for pin_id, pin_config in config.CONF_IO_PINS.items():
self.pin_configs[pin_id] = pin_config
# Set up buffer attributes
self.attributes: Dict[str, ArrayAttribute] = {}
for pin_id, pin_config in self.pin_configs.items():
if pin_config['type'] in ('do', 'ao'):
continue
if pin_config['type'] == 'di':
attr = ArrayAttribute(pin_id, (1,), ArrayType.uint8)
else:
attr = ArrayAttribute(pin_id, (1,), ArrayType.float64)
self.attributes[pin_id] = attr
def setup(self):
self.device_list: List[Tuple[str, int, int]] = []
for device_id, device_config in config.CONF_CAMERA_DEVICES.items():
self.device_list.append(
(device_id, device_config['width'], device_config['height']))
# Set one array attribute per camera device
self.frames: Dict[str, ArrayAttribute] = {}
for device_id, res_x, res_y in self.device_list:
# self.frames[device_id] = ArrayAttribute(f'{device_id}_frame', (res_y, res_x), ArrayType.uint8)
self.frames[device_id] = ArrayAttribute(f'{device_id}_frame',
(res_x, res_y),
ArrayType.uint8)
class WriteRandoms(IoRoutine):
def __init__(self):
super(WriteRandoms, self).__init__()
def setup(self):
self.sawtooth_attr = ArrayAttribute('test_sawtooth', (1, ), ArrayType.float32)
self.poisson_1p20_attr = ArrayAttribute('test_poisson_1p200', (1, ), ArrayType.bool)
def initialize(self):
register_with_plotter('test_sawtooth', axis=WriteRandoms.__name__)
register_with_plotter('test_poisson_1p200', axis=WriteRandoms.__name__)
set_digital_output('ch_do01', 'test_poisson_1p200')
self.poisson_high_end = 0.
def main(self, *args, **kwargs):
t = get_time()
# Update sawtooth
a = 1.
p = 2.
sawtooth = - 2 * a / np.pi * np.arctan(1. / np.tan(np.pi * t / p))
self.sawtooth_attr.write(sawtooth)
# Update poisson_1p200
if self.poisson_high_end >= get_time():
new_state = True
else:
self.poisson_high_end = 0.
new_state = np.random.randint(200) == 0
if new_state:
# Show high state for 5ms
self.poisson_high_end = get_time() + 0.005
self.poisson_1p20_attr.write(new_state)
def setup(self):
self.whitenoise = ArrayAttribute('test_sines_whitenoise', (1,), ArrayType.float64)
def setup(self):
self.test_onoff = ArrayAttribute('test_onoff', (1,), ArrayType.bool)
def setup(self):
self.sawtooth_attr = ArrayAttribute('test_sawtooth', (1, ), ArrayType.float32)
self.poisson_1p20_attr = ArrayAttribute('test_poisson_1p200', (1, ), ArrayType.bool)
class EyePositionDetection(CameraRoutine):
# Set required device
camera_device_id = 'fish_embedded'
routine_prefix = 'eyepos_'
extracted_rect_prefix = f'{routine_prefix}extracted_rect_'
ang_le_pos_prefix = f'{routine_prefix}angular_le_pos_'
ang_re_pos_prefix = f'{routine_prefix}angular_re_pos_'
ang_le_vel_prefix = f'{routine_prefix}angular_le_vel_'
ang_re_vel_prefix = f'{routine_prefix}angular_re_vel_'
le_sacc_prefix = f'{routine_prefix}le_saccade_'
re_sacc_prefix = f'{routine_prefix}re_saccade_'
roi_maxnum = 10
def setup(self):
require_camera_device(self.camera_device_id)
# Get camera specs
camera_config = config.CONF_CAMERA_DEVICES.get(self.camera_device_id)
self.res_x = camera_config['width']
self.res_y = camera_config['height']
# Set up parameter variables (accessible externally)
self.rois = dict()
self.thresh = None
self.min_size = None
self.saccade_threshold = None
# Set up attributes
# self.data = []
for id in range(self.roi_maxnum):
roi = {}
# Rectangle
extracted_roi_name = f'{self.extracted_rect_prefix}{id}'
roi[extracted_roi_name] = ObjectAttribute(extracted_roi_name)
# Position
# LE
le_pos_name = f'{self.ang_le_pos_prefix}{id}'
roi[le_pos_name] = ArrayAttribute(le_pos_name, (1,), ArrayType.float64)
# RE
re_pos_name = f'{self.ang_re_pos_prefix}{id}'
roi[re_pos_name] = ArrayAttribute(re_pos_name, (1,), ArrayType.float64)
# Velocity
# LE
le_vel_name = f'{self.ang_le_vel_prefix}{id}'
roi[le_vel_name] = ArrayAttribute(le_vel_name, (1,), ArrayType.float64)
# RE
re_vel_name = f'{self.ang_re_vel_prefix}{id}'
roi[re_vel_name] = ArrayAttribute(re_vel_name, (1,), ArrayType.float64)
# Saccade detection
# LE
le_sacc_name = f'{self.le_sacc_prefix}{id}'
roi[le_sacc_name] = ArrayAttribute(le_sacc_name, (1,), ArrayType.float64)
# RE
re_sacc_name = f'{self.re_sacc_prefix}{id}'
roi[re_sacc_name] = ArrayAttribute(re_sacc_name, (1,), ArrayType.float64)
# self.data.append(roi)
# Set frame buffer
# self.frame = ArrayAttribute('eyeposdetect_frame', (self.res_y, self.res_x), ArrayType.uint8)
self.frame = ArrayAttribute('eyeposdetect_frame', (self.res_x, self.res_y), ArrayType.uint8)
# Set saccade trigger buffer
self.saccade_trigger = ArrayAttribute('eyeposdetect_saccade_trigger', (1, ), ArrayType.bool)
def initialize(self):
self.add_trigger('saccade_trigger')
# Set accessible methods
self.exposed.append(EyePositionDetection.set_threshold)
self.exposed.append(EyePositionDetection.set_threshold_range)
self.exposed.append(EyePositionDetection.set_threshold_iterations)
self.exposed.append(EyePositionDetection.set_max_im_value)
self.exposed.append(EyePositionDetection.set_min_particle_size)
self.exposed.append(EyePositionDetection.set_roi)
self.exposed.append(EyePositionDetection.set_saccade_threshold)
set_digital_output('saccade_trigger_out', 'eyeposdetect_saccade_trigger')
def set_threshold(self, thresh):
self.thresh = thresh
def set_threshold_range(self, range):
self.thresh_range = range
def set_threshold_iterations(self, n):
self.thresh_iter = n
def set_max_im_value(self, value):
self.maxvalue = value
def set_min_particle_size(self, size):
self.min_size = size
def set_roi(self, id, params):
if id not in self.rois:
log.info(f'Create new ROI at {params}')
# Send buffer attributes to plotter
# Position
register_with_plotter(f'{self.ang_le_pos_prefix}{id}', name=f'eye_pos(LE {id})', axis='eye_pos', units='deg')
register_with_plotter(f'{self.ang_re_pos_prefix}{id}', name=f'eye_pos(RE {id})', axis='eye_pos', units='deg')
# Velocity
register_with_plotter(f'{self.ang_le_vel_prefix}{id}', name=f'eye_vel(LE {id})', axis='eye_vel', units='deg/s')
register_with_plotter(f'{self.ang_re_vel_prefix}{id}', name=f'eye_vel(RE {id})', axis='eye_vel', units='deg/s')
# Saccade trigger
register_with_plotter(f'{self.le_sacc_prefix}{id}', name=f'sacc(LE {id})', axis='sacc')
register_with_plotter(f'{self.re_sacc_prefix}{id}', name=f'sacc(RE {id})', axis='sacc')
# Add attributes to save-to-file list:
write_to_file(self, f'{self.ang_le_pos_prefix}{id}')
write_to_file(self, f'{self.ang_re_pos_prefix}{id}')
write_to_file(self, f'{self.ang_le_vel_prefix}{id}')
write_to_file(self, f'{self.ang_re_vel_prefix}{id}')
write_to_file(self, f'{self.le_sacc_prefix}{id}')
write_to_file(self, f'{self.re_sacc_prefix}{id}')
self.rois[id] = params
def set_saccade_threshold(self, thresh):
self.saccade_threshold = thresh
@staticmethod
def tuple_o_ints(t):
return tuple((int(i) for i in t))
def from_ellipse(self, rect):
# Formatting for drawing
line_thickness = np.ceil(np.mean(rect.shape) / 50).astype(int)
line_thickness = 1 if line_thickness == 0 else line_thickness
marker_size = line_thickness * 5
# Set rect center
rect_center = (rect.shape[1] // 2, rect.shape[0] // 2)
# Apply threshold
_, thresh = cv2.threshold(rect[:,:], self.thresh, 255, cv2.THRESH_BINARY_INV)
# Detect contours
cnts, hierarchy = cv2.findContours(thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
# Make RGB
thresh = np.stack((thresh, thresh, thresh), axis=-1)
# Collect contour parameters and filter contours
areas = list()
barycenters = list()
hulls = list()
feret_points = list()
thetas = list()
axes = list()
dists = list()
i = 0
while i < len(cnts):
cnt = cnts[i]
M = cv2.moments(cnt)
A = M['m00']
# Discard if contour has no area
if A < self.min_size:
del cnts[i]
continue
# Particle center
center = (M['m10']/A, M['m01']/A)
# Hull of particle
hull = cv2.convexHull(cnt).squeeze()
# Ellipse axes lengths
a = M['m20'] / M['m00'] - center[0] ** 2
b = 2 * (M['m11'] / M['m00'] - center[0] * center[1])
c = M['m02'] / M['m00'] - center[1] ** 2
# Avoid divisions by zero
if (a - c) == 0.:
del cnts[i]
continue
# Ellipse's major axis angle
theta = (1 / 2 * np.arctan(b / (a - c)) + (a < c) - 1) / np.pi * 180 #* np.pi / 2
W = np.sqrt(8 * (a + c - np.sqrt(b ** 2 + (a - c) ** 2))) / 2
L = np.sqrt(8 * (a + c + np.sqrt(b ** 2 + (a - c) ** 2))) / 2
thetas.append(theta)
barycenters.append(center)
axes.append((W, L))
areas.append(A)
hulls.append(hull)
dists.append(distance.euclidean(center, rect_center))
feret_points.append((hull[np.argmin(hull[:,1])], hull[np.argmax(hull[:,1])]))
i += 1
# Additional filtering of particles to idenfity both eyes if more than 2
if len(cnts) > 2:
dists, areas, barycenters, hulls, feret_points, thetas, axes = \
list(zip(*sorted(list(zip(dists, areas, barycenters, hulls, feret_points, thetas, axes)))[:2]))
forward_vec = np.array([0,-1])
forward_vec_norm = forward_vec / np.linalg.norm(forward_vec)
# Draw rect center and midline marker for debugging
# (Important: this has to happen AFTER detection of contours,
# as it alters the tresh'ed image)
cv2.drawMarker(thresh, rect_center, (0, 255, 0), cv2.MARKER_DIAMOND, marker_size * 2, line_thickness)
cv2.arrowedLine(thresh,
tuple(rect_center), tuple((rect_center + rect.shape[0]/3 * forward_vec_norm).astype(int)),
(0, 255, 0), line_thickness, tipLength=0.3)
# Draw hull contours for debugging (before possible premature return)
cv2.drawContours(thresh, hulls, -1, (128,128,0), line_thickness)
# If less than two particles, return
if len(cnts) < 2:
return [np.nan, np.nan], thresh
# At this point there should only be 2 particles left
le_idx = 0 if (barycenters[0][0] < rect_center[0]) else 1
re_idx = 1 if (barycenters[0][0] < rect_center[0]) else 0
try:
for center, axis, theta in zip(barycenters, axes, thetas):
center = self.tuple_o_ints(center)
axis = self.tuple_o_ints(axis)
angle = float(theta)
start_angle = 0.
end_angle = 360.
color = (255, 0, 0)
cv2.ellipse(thresh,
center,
axis,
angle, start_angle, end_angle, color, line_thickness)
except Exception as exc:
import traceback
traceback.print_exc()
return [thetas[le_idx], thetas[re_idx]], thresh
def main(self, **frames):
# Read frame
frame = frames.get(self.camera_device_id)
# Check if frame was returned
if frame is None:
return
# Reduce to mono
if frame.ndim > 2:
frame = frame[:,:,0]
# Write frame to buffer
self.frame.write(frame.T)
# Do eye detection and angular position estimation
if not bool(self.rois):
return
# If eyes were marked: iterate over ROIs and extract eye positions
saccade_happened = False
for id, rect_params in self.rois.items():
####
# Extract rectanglular ROI
# Convert from pyqtgraph image coordinates to openCV
# rect_params = (tuple(self.coord_transform_pg2cv(rect_params[0])), tuple(rect_params[1]), -rect_params[2],)
# Get rect and frame parameters
center, size, angle = rect_params[0], rect_params[1], rect_params[2]
center, size = tuple(map(int, center)), tuple(map(int, size))
height, width = frame.shape[0], frame.shape[1]
# Rotate
M = cv2.getRotationMatrix2D(center, angle, 1)
rotFrame = cv2.warpAffine(frame, M, (width, height))
# Crop rect from frame
cropRect = cv2.getRectSubPix(rotFrame, size, center)
# Rotate rect so that "up" direction in image corresponds to "foward" for the fish
center = (size[0]/2, size[1]/2)
width, height = size
M = cv2.getRotationMatrix2D(center, 90, 1)
absCos = abs(M[0, 0])
absSin = abs(M[0, 1])
# New bound width/height
wBound = int(height * absSin + width * absCos)
hBound = int(height * absCos + width * absSin)
# Subtract old image center
M[0, 2] += wBound / 2 - center[0]
M[1, 2] += hBound / 2 - center[1]
# Rotate
rot_rect = cv2.warpAffine(cropRect, M, (wBound, hBound))
####
# Calculate eye angular POSITIONS
# Apply detection function on cropped rect which contains eyes
(le_pos, re_pos), new_rect = self.from_ellipse(rot_rect)
# Get shared attributes
le_pos_attr = get_attribute(f'{self.ang_le_pos_prefix}{id}')
re_pos_attr = get_attribute(f'{self.ang_re_pos_prefix}{id}')
le_vel_attr = get_attribute(f'{self.ang_le_vel_prefix}{id}')
re_vel_attr = get_attribute(f'{self.ang_re_vel_prefix}{id}')
le_sacc_attr = get_attribute(f'{self.le_sacc_prefix}{id}')
re_sacc_attr = get_attribute(f'{self.re_sacc_prefix}{id}')
rect_roi_attr = get_attribute(f'{self.extracted_rect_prefix}{id}')
# Calculate eye angular VELOCITIES
_, _, last_le_pos = le_pos_attr.read()
last_le_pos = last_le_pos[0]
_, last_time, last_re_pos = re_pos_attr.read()
last_re_pos = last_re_pos[0]
last_time = last_time[-1]
if last_time is None:
last_time = -np.inf
# Calculate time elapsed since last frame
current_time = get_time()
dt = (current_time - last_time)
# Calculate velocities
le_vel = np.abs((le_pos - last_le_pos) / dt)
re_vel = np.abs((re_pos - last_re_pos) / dt)
# Calculate saccade trigger
_, _, last_le_vel = le_vel_attr.read()
last_le_vel = last_le_vel[0]
_, _, last_re_vel = re_vel_attr.read()
last_re_vel = last_re_vel[0]
le_sacc = int(last_le_vel < self.saccade_threshold < le_vel)
re_sacc = int(last_re_vel < self.saccade_threshold < re_vel)
is_saccade = bool(le_sacc) or bool(re_sacc)
saccade_happened = saccade_happened or is_saccade
if is_saccade:
self.emit_trigger('saccade_trigger')
# Write to buffer
le_pos_attr.write(le_pos)
re_pos_attr.write(re_pos)
le_vel_attr.write(le_vel)
re_vel_attr.write(re_vel)
le_sacc_attr.write(le_sacc)
re_sacc_attr.write(re_sacc)
# Set current rect ROI data
rect_roi_attr.write(new_rect)
self.saccade_trigger.write(saccade_happened)
class CalculatePSD(WorkerRoutine):
nperseg = 2**10
def __init__(self, *args, **kwargs):
WorkerRoutine.__init__(self, *args, **kwargs)
self.exposed.append(CalculatePSD.set_input_signal)
self.exposed.append(CalculatePSD.set_integration_window_width)
def set_input_signal(self, attr_name, force_overwrite=False):
if self.input_signal is not None and not force_overwrite:
warn_context = f'Signal is already set to {self.input_signal.name}.'
else:
self.input_signal = get_attribute(attr_name)
if self.input_signal is None:
warn_context = 'Undefined attribute.'
else:
log.info(
f'Set input signal in {self.__class__.__name__} to {attr_name}.'
)
return
log.warning(
f'Failed to set input signal in {self.__class__.__name__} to {attr_name}. {warn_context}'
)
def set_integration_window_width(self, width):
if width < self.nperseg:
log.warning(
f'Failed to set integration window width in {self.__class__.__name__}. '
f'New value {width} < nperseg ({self.nperseg}). '
f'Keeping current ({self.integration_window_width})')
return
self.integration_window_width = width
def setup(self):
self.input_signal: ArrayAttribute = None
self.integration_window_width = None
psd_return_size = self.nperseg // 2 + 1
self.frequencies = ArrayAttribute('psd_frequency', (psd_return_size, ),
ArrayType.float64)
self.power = ArrayAttribute('psd_power', (psd_return_size, ),
ArrayType.float64)
def initialize(self):
pass
def main(self, *args, **kwargs):
if self.input_signal is None or self.integration_window_width is None:
return
i, t, y = self.input_signal.read(self.integration_window_width)
if t[0] is None or not isinstance(y, np.ndarray):
return
y = y.flatten()
f, p = signal.csd(y,
y,
fs=1. / np.mean(np.diff(t)),
nperseg=self.nperseg)
self.frequencies.write(f)
self.power.write(p)
def setup(self):
self.protocol_active = ArrayAttribute('protocol_active', (1, ), ArrayType.bool)
self.phase_active = ArrayAttribute('phase_active', (1, ), ArrayType.bool)