def calibrate_flatfield_gains(self, channel: LC):
"""
Calibrate the flatfield of the given channel. A reference value is needed for calculations of for example NDVI. This function computes the average value of the flatfield and saves it with the accompanying gain.
:param channel: channel of light that requires flatfield calibration
"""
# capture a photo of the appropriate channel
rgb, gain = self.capture(channel)
# save the flatfield gain to the config
self.config["ff"]["gain"][channel] = float(gain)
# cut image to size
rgb = rgb[self.settings.crop["y_min"]:self.settings.crop["y_max"], self.settings.crop["x_min"]:self.settings.crop["x_max"], :]
# turn rgb into hsv and extract the v channel as the mask
v = self.extract_value_from_rgb(channel, rgb)
# get the average intensity of the light and save for flatfielding
self.config["ff"]["value"][channel] = np.mean(v[self.settings.ground_plane["y_min"]:self.settings.ground_plane["y_max"], self.settings.ground_plane["x_min"]:self.settings.ground_plane["x_max"]])
d_print("{} ff std: ".format(channel) + str(np.std(v[self.settings.ground_plane["y_min"]:self.settings.ground_plane["y_max"], self.settings.ground_plane["x_min"]:self.settings.ground_plane["x_max"]])), 1)
# write image to file using imageio's imwrite
path_to_img = "{}/cam/cfg/{}_mask.jpg".format(self.working_directory, channel)
d_print("Writing to file...", 1)
imwrite(path_to_img, rgb.astype(np.uint8))
def calibrate(self):
"""
Calibrates the camera and the light sources.
"""
# ask user to put something white and diffuse in the kit
d_print("Assuming a white diffuse surface is placed at the bottom of the kit...", 1)
self.CALIBRATED = False
# set up the camera config dict
self.config = dict()
self.config["cam_id"] = self.CAM_ID
self.config["rotation"] = 0
self.config["wb"] = dict()
self.config["ff"] = dict()
self.config["ff"]["gain"] = dict()
self.config["ff"]["value"] = dict()
d_print("Starting calibration...", 1)
# calibrate all white balances
for channel in self.light_channels:
self.calibrate_white_balance(channel)
# calibrate the gains
for channel in self.light_channels:
if channel == LC.RED or channel == LC.NIR:
self.calibrate_flatfield_gains(channel)
# write the configuration to file
self.save_config_to_file()
self.CALIBRATED = True
def capture(self, channel: LC):
"""
Function that captures an image. Uses raspistill in a separate terminal process to take the picture. This is faster (about 4-5 seconds to take an image on average) due to the possibility to manually set the gains of the camera, something that is not possible in picamera 1.13 (but will probably be in version 1.14 or 1.15).
:param channel: channel of light in which the photo is taken, used for white balance and gain values
:return: 8 bit rgb array containing the image
"""
# check if gain information is available, if not, update first
if "d2d" not in self.config:
self.setup_d2d()
self.update()
# turn on the light
self.light_control(channel, 1)
# assemble the terminal command
path_to_bright = os.getcwd() + "/cam/tmp/bright.bmp"
path_to_dark = os.getcwd() + "/cam/tmp/dark.bmp"
gain = self.config["d2d"][channel]["analog-gain"] * self.config["d2d"][channel]["digital-gain"]
photo_cmd = "raspistill -e bmp -w {} -h {} -ss {} -t 1000 -awb off -awbg {},{} -ag {} -dg {}".format(self.settings.resolution[0], self.settings.resolution[1], self.settings.shutter_speed[channel], self.config["wb"][channel]["r"], self.config["wb"][channel]["b"], self.config["d2d"][channel]["analog-gain"], self.config["d2d"][channel]["digital-gain"])
# run command and take bright and dark picture
# start the bright image capture by spawning a clean process and executing the command, then waiting for the q
p = mp.Process(target=photo_worker, args=(photo_cmd + " -o {}".format(path_to_bright),))
try:
p.start()
p.join()
except OSError:
d_print("Could not start child process, out of memory", 3)
return (None, 0)
# turn off the light
self.light_control(channel, 0)
# start the dark image capture by spawning a clean process and executing the command, then waiting for the q
p = mp.Process(target=photo_worker, args=(photo_cmd + " -o {}".format(path_to_dark),))
try:
p.start()
p.join()
except OSError:
d_print("Could not start child process, out of memory", 3)
return (None, 0)
# load the images from file, perform dark frame subtraction and return the array
bright = Image.open(path_to_bright)
rgb = np.array(bright)
if channel != LC.GROWTH:
dark = Image.open(path_to_dark)
rgb = cv2.subtract(rgb, np.array(dark))
# if the time since last update is larger than a day, update the gains after the photo
if time.time() - self.config["d2d"]["timestamp"] > 3600*24:
self.update()
return (rgb, gain)
def light_control(channel: LC, state):
d_print(
"Setting {} camera lighting state to {}".format(channel, state), 1)
if channel == LC.WHITE:
pi.write(17, state)
elif channel == LC.RED:
pi.write(3, state)
elif channel == LC.NIR:
pi.write(4, state)
else:
d_print("no such light available", 3)
time.sleep(0.1)
def extract_value_from_rgb(self, channel: LC, rgb):
"""
Subfunction used to extract the right value matrix from the rgb image.
:param channel: channel of light of which the value matrix is needed
:param rgb: rgb matrix (original photo)
:return: value matrix
"""
if channel == LC.NIR:
# turn rgb into hsv and extract the v channel as the mask
hsv = cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV)
v = hsv[:,:,2]
elif channel == LC.RED:
# extract the red channel
v = rgb[:,:,0]
else:
d_print("unknown channel {}, cannot extract value, returning black matrix...".format(channel), 3)
v = np.zeros([10, 10])
return v
def photo(self, channel: LC):
"""
Make a photo with the specified light channel and save the image to disk
:param channel: channel of light a photo needs to be taken from
:return: path to the photo taken
"""
# capture a photo of the appropriate channel
rgb, _ = self.capture(channel)
# catch error
if rgb is None:
res = dict()
res["contains_photo"] = False
res["contains_value"] = False
res["encountered_error"] = True
res["timestamp"] = curr_time
return res
# crop the sensor readout
rgb = rgb[self.settings.crop["y_min"]:self.settings.crop["y_max"], self.settings.crop["x_min"]:self.settings.crop["x_max"], :]
# write image to file using imageio's imwrite
d_print("Writing to file...", 1)
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
path_to_img = "{}/cam/img/{}_{}.jpg".format(self.working_directory, channel, curr_time)
imwrite(path_to_img, rgb)
res = dict()
res["contains_photo"] = True
res["contains_value"] = False
res["encountered_error"] = False
res["timestamp"] = curr_time
res["photo_path"] = [path_to_img]
res["photo_kind"] = [channel]
return(res)
def do(self, command: CC):
"""
Function that directs the commands from the user to the right place. Does some preliminary checks to see if actions are allowed in the current state of the camera (uncalibrated etc.). Throws an error on the command line if actions are illegal.
:param command: (C)amera (C)ommand, what the user wants to do.
"""
if command == CC.WHITE_PHOTO and LC.WHITE in self.light_channels and self.CALIBRATED:
return self.photo(LC.WHITE)
elif command == CC.NDVI_PHOTO and self.NDVI_CAPABLE and self.CALIBRATED:
return self.ndvi.ndvi_photo()
elif command == CC.GROWTH_PHOTO and LC.GROWTH in self.light_channels and self.CALIBRATED:
return self.photo(LC.GROWTH)
elif command == CC.NDVI and self.NDVI_CAPABLE and self.CALIBRATED:
return self.ndvi.ndvi()
elif command == CC.NIR_PHOTO and LC.NIR in self.light_channels and self.CALIBRATED:
return self.photo(LC.NIR)
elif command == CC.CALIBRATE:
self.calibrate()
elif command == CC.UPDATE and self.HAS_UPDATE and self.CALIBRATED:
self.update()
else:
d_print("Camera is unable to perform command '{}', is it calibrated? ({})\n Run [cam].state() to check current camera and lighting status...\n Returning empty...".format(command, self.CALIBRATED), 3)
return ""
def __init__(self, *args, light_control = light_control_dummy, light_channels = [LC.GROWTH], settings, working_directory = os.getcwd(), **kwargs):
"""
Initialize an object that contains the visible routines.
Link the pi and gpio pins necessary and provide a function that controls the growth lighting.
:param light_control: function that allows control over the lighting. Parameters are the channel to control and either a 0 or 1 for off and on respectively
:param light_channels: list containing allowable light channels
:param settings: reference to one of the settings object contained in this file. These settings are used to control shutter speeds, resolutions, crops etc.
"""
# set up the camera super class
super().__init__(light_control = light_control, working_directory = working_directory)
# give the camera a unique ID per brand/kind/etc, software uses this ID to determine whether the
# camera is calibrated or not
self.CAM_ID = 2
# enable update function to update gains
self.HAS_UPDATE = True
# bind the settings to the camera object
self.settings = settings
# set up the light channel array
self.light_channels = []
for channel in light_channels:
if channel in self.settings.allowed_channels:
self.light_channels.append(channel)
# load config file and check if it matches the cam id, if so, assume calibrated
try:
self.load_config_from_file()
if self.config["cam_id"] == self.CAM_ID:
self.CALIBRATED = True
d_print("Succesfully loaded suitable camera configuration.", 1)
else:
self.CALIBRATED = False
d_print("Found camera configuration file, but contents are not suitable for current camera.", 3)
except (EnvironmentError, ValueError):
d_print("No suitable camera configuration file found!", 3)
self.CALIBRATED = False
# set multiprocessing to spawn (so NOT fork)
try:
mp.set_start_method('spawn')
except RuntimeError:
pass
def update(self):
"""
Function that updates the gains needed to expose the image correctly. Saves it to the configuration file.
"""
# check if gain information is available, if not, update config
if "d2d" not in self.config:
self.setup_d2d()
for channel in self.light_channels:
# turn on the light
self.light_control(channel, 1)
d_print("Letting gains settle for the {} channel...".format(channel), 1)
with picamera.PiCamera() as sensor:
# set up the sensor with all its settings
sensor.resolution = self.settings.resolution
sensor.framerate = self.settings.framerate[channel]
sensor.shutter_speed = self.settings.shutter_speed[channel]
sensor.awb_mode = "off"
sensor.awb_gains = (self.config["wb"][channel]["r"], self.config["wb"][channel]["b"])
time.sleep(30)
sensor.exposure_mode = self.settings.exposure_mode
# set the analog and digital gain
ag = float(sensor.analog_gain)
dg = float(sensor.digital_gain)
self.config["d2d"][channel]["digital-gain"] = dg
self.config["d2d"][channel]["analog-gain"] = ag
d_print("Measured ag: {} and dg: {} for channel {}".format(ag, dg, channel), 1)
d_print("Saved ag: {} and dg: {} for channel {}".format(self.config["d2d"][channel]["analog-gain"], self.config["d2d"][channel]["digital-gain"], channel), 1)
# turn the light off
self.light_control(channel, 0)
# update timestamp
self.config["d2d"]["timestamp"] = time.time()
# save the new configuration to file
self.save_config_to_file()
def calibrate_white_balance(self, channel: LC):
"""
Function that calibrates the white balance for certain lighting specified in the channel parameter. This is camera specific, so it needs to be specified for each camera.
:param channel: light channel that needs to be calibrated
"""
d_print("Warming up camera sensor...", 1)
# turn on channel light
self.light_control(channel, 1)
if channel == LC.WHITE or channel == LC.NIR:
with picamera.PiCamera() as sensor:
# set up the sensor with all its settings
sensor.resolution = (128, 80)
sensor.rotation = self.config["rotation"]
sensor.framerate = self.settings.framerate[channel]
sensor.shutter_speed = self.settings.shutter_speed[channel]
# set up the blue and red gains
sensor.awb_mode = "off"
rg, bg = (1.1, 1.1)
sensor.awb_gains = (rg, bg)
# now sleep and lock exposure
time.sleep(20)
sensor.exposure_mode = self.settings.exposure_mode
# record camera data to array and scale up a numpy array
#rgb = np.zeros((1216,1216,3), dtype=np.uint16)
with picamera.array.PiRGBArray(sensor) as output:
# capture images and analyze until convergence
for i in range(30):
output.truncate(0)
sensor.capture(output, 'rgb')
rgb = np.copy(output.array)
#crop = rgb[508:708,666:966,:]
crop = rgb[30:50, 32:96, :]
r, g, b = (np.mean(crop[..., i]) for i in range(3))
d_print(
"\trg: {:4.3f} bg: {:4.3f} --- ({:4.1f}, {:4.1f}, {:4.1f})"
.format(rg, bg, r, g, b), 1)
if abs(r - g) > 1:
if r > g:
rg -= 0.025
else:
rg += 0.025
if abs(b - g) > 1:
if b > g:
bg -= 0.025
else:
bg += 0.025
sensor.awb_gains = (rg, bg)
elif channel == LC.GROWTH:
rg = self.settings.wb[LC.GROWTH]["r"]
bg = self.settings.wb[LC.GROWTH]["b"]
else:
rg = self.settings.wb[LC.RED]["r"]
bg = self.settings.wb[LC.RED]["b"]
# turn off channel light
self.light_control(channel, 0)
self.config["wb"][channel] = dict()
self.config["wb"][channel]["r"] = rg
self.config["wb"][channel]["b"] = bg
d_print("Done.", 1)
def update(self):
"""
Function that updates the gains needed to expose the image correctly. Saves it to the configuration file.
"""
# check if gain information is available, if not, update config
if "d2d" not in self.config:
self.setup_d2d()
for channel in self.light_channels:
# turn on the light
self.light_control(channel, 1)
d_print(
"Letting gains settle for the {} channel...".format(channel),
1)
with picamera.PiCamera() as sensor:
# set up the sensor with all its settings
sensor.resolution = self.settings.resolution
sensor.framerate = self.settings.framerate[channel]
sensor.shutter_speed = self.settings.shutter_speed[channel]
sensor.awb_mode = "off"
sensor.awb_gains = (self.config["wb"][channel]["r"],
self.config["wb"][channel]["b"])
time.sleep(30)
sensor.exposure_mode = self.settings.exposure_mode
# there is some non-linearity in the camera for higher pixel values when the gain gets large.
# currently, we fix this by limiting the gain to a maximum of 1.5 times the calibration gain.
# images will be a bit underexposed, but this can be expected for the red images anyway, since they
# tend to be dark.
ag = float(sensor.analog_gain)
dg = float(sensor.digital_gain)
if self.CALIBRATED:
if channel == LC.RED or channel == LC.NIR:
if self.config["ff"]["gain"][channel] * 1.5 < ag:
self.config["d2d"][channel][
"analog-gain"] = self.config["ff"]["gain"][
channel] * 1.5
elif self.config["ff"]["gain"][channel] * 0.67 > ag:
self.config["d2d"][channel][
"analog-gain"] = self.config["ff"]["gain"][
channel] * 0.67
else:
self.config["d2d"][channel]["analog-gain"] = ag
else:
self.config["d2d"][channel]["analog-gain"] = ag
if dg > 2 and (channel == LC.RED or channel == LC.NIR):
self.config["d2d"][channel]["digital-gain"] = 2
else:
self.config["d2d"][channel]["digital-gain"] = dg
else:
self.config["d2d"][channel]["digital-gain"] = dg
self.config["d2d"][channel]["analog-gain"] = ag
d_print(
"Measured ag: {} and dg: {} for channel {}".format(
ag, dg, channel), 1)
d_print(
"Saved ag: {} and dg: {} for channel {}".format(
self.config["d2d"][channel]["analog-gain"],
self.config["d2d"][channel]["digital-gain"], channel),
1)
# turn the light off
self.light_control(channel, 0)
# update timestamp
self.config["d2d"]["timestamp"] = time.time()
# save the new configuration to file
self.save_config_to_file()
def light_control_dummy(channel: LC, state):
d_print("light_control dummmy called, passing...", 1)
time.sleep(0.1)
def ndvi_photo(self):
"""
Make a photo in the nir and the red spectrum and overlay to obtain ndvi.
:return: (path to the ndvi image, average ndvi value for >0.25 (iff the #pixels is larger than 2 percent of the total))
"""
# get the ndvi matrix
ndvi_matrix = self.ndvi_matrix()
# catch error
if ndvi_matrix is None:
res = dict()
res["contains_photo"] = False
res["contains_value"] = False
res["encountered_error"] = True
res["timestamp"] = curr_time
return res
ndvi_matrix = np.clip(ndvi_matrix, -1.0, 1.0)
if np.count_nonzero(ndvi_matrix > 0.25) > 0.02 * np.size(ndvi_matrix):
ndvi = np.mean(ndvi_matrix[ndvi_matrix > 0.25])
else:
ndvi = 0
rescaled = np.uint8(np.round(127.5 * (ndvi_matrix + 1.0)))
ndvi_plot = np.copy(ndvi_matrix)
ndvi_plot[ndvi_plot < 0.25] = np.nan
# write images to file using imageio's imwrite and matplotlibs savefig
d_print("Writing to file...", 1)
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# set multiprocessing to spawn (so NOT fork)
try:
mp.set_start_method('spawn')
except RuntimeError:
pass
# write the matplotlib part in a separate process so no memory leaks
path_to_img_2 = "{}/cam/img/{}{}_{}.jpg".format(
self.camera.working_directory, "ndvi", 2, curr_time)
p = mp.Process(target=plotter, args=(
ndvi_plot,
path_to_img_2,
))
try:
p.start()
p.join()
except OSError:
d_print("Could not start child process, out of memory", 3)
res = dict()
res["contains_photo"] = False
res["contains_value"] = False
res["encountered_error"] = True
res["timestamp"] = curr_time
return res
path_to_img_1 = "{}/cam/img/{}{}_{}.tif".format(
self.camera.working_directory, "ndvi", 1, curr_time)
imwrite(path_to_img_1, rescaled)
res = dict()
res["contains_photo"] = True
res["contains_value"] = True
res["encountered_error"] = False
res["timestamp"] = curr_time
res["photo_path"] = [path_to_img_1, path_to_img_2]
res["photo_kind"] = ["raw NDVI", "processed NDVI"]
res["value"] = [ndvi]
res["value_kind"] = ["NDVI"]
res["value_error"] = [0.0]
return res