def calibrate_flatfield_gains(self, channel: LC): """ Calibrate the flatfield of the given channel. A reference value is needed for calculations of for example NDVI. This function computes the average value of the flatfield and saves it with the accompanying gain. :param channel: channel of light that requires flatfield calibration """ # capture a photo of the appropriate channel rgb, gain = self.capture(channel) # save the flatfield gain to the config self.config["ff"]["gain"][channel] = float(gain) # cut image to size rgb = rgb[self.settings.crop["y_min"]:self.settings.crop["y_max"], self.settings.crop["x_min"]:self.settings.crop["x_max"], :] # turn rgb into hsv and extract the v channel as the mask v = self.extract_value_from_rgb(channel, rgb) # get the average intensity of the light and save for flatfielding self.config["ff"]["value"][channel] = np.mean(v[self.settings.ground_plane["y_min"]:self.settings.ground_plane["y_max"], self.settings.ground_plane["x_min"]:self.settings.ground_plane["x_max"]]) d_print("{} ff std: ".format(channel) + str(np.std(v[self.settings.ground_plane["y_min"]:self.settings.ground_plane["y_max"], self.settings.ground_plane["x_min"]:self.settings.ground_plane["x_max"]])), 1) # write image to file using imageio's imwrite path_to_img = "{}/cam/cfg/{}_mask.jpg".format(self.working_directory, channel) d_print("Writing to file...", 1) imwrite(path_to_img, rgb.astype(np.uint8))
def calibrate(self): """ Calibrates the camera and the light sources. """ # ask user to put something white and diffuse in the kit d_print("Assuming a white diffuse surface is placed at the bottom of the kit...", 1) self.CALIBRATED = False # set up the camera config dict self.config = dict() self.config["cam_id"] = self.CAM_ID self.config["rotation"] = 0 self.config["wb"] = dict() self.config["ff"] = dict() self.config["ff"]["gain"] = dict() self.config["ff"]["value"] = dict() d_print("Starting calibration...", 1) # calibrate all white balances for channel in self.light_channels: self.calibrate_white_balance(channel) # calibrate the gains for channel in self.light_channels: if channel == LC.RED or channel == LC.NIR: self.calibrate_flatfield_gains(channel) # write the configuration to file self.save_config_to_file() self.CALIBRATED = True
def capture(self, channel: LC): """ Function that captures an image. Uses raspistill in a separate terminal process to take the picture. This is faster (about 4-5 seconds to take an image on average) due to the possibility to manually set the gains of the camera, something that is not possible in picamera 1.13 (but will probably be in version 1.14 or 1.15). :param channel: channel of light in which the photo is taken, used for white balance and gain values :return: 8 bit rgb array containing the image """ # check if gain information is available, if not, update first if "d2d" not in self.config: self.setup_d2d() self.update() # turn on the light self.light_control(channel, 1) # assemble the terminal command path_to_bright = os.getcwd() + "/cam/tmp/bright.bmp" path_to_dark = os.getcwd() + "/cam/tmp/dark.bmp" gain = self.config["d2d"][channel]["analog-gain"] * self.config["d2d"][channel]["digital-gain"] photo_cmd = "raspistill -e bmp -w {} -h {} -ss {} -t 1000 -awb off -awbg {},{} -ag {} -dg {}".format(self.settings.resolution[0], self.settings.resolution[1], self.settings.shutter_speed[channel], self.config["wb"][channel]["r"], self.config["wb"][channel]["b"], self.config["d2d"][channel]["analog-gain"], self.config["d2d"][channel]["digital-gain"]) # run command and take bright and dark picture # start the bright image capture by spawning a clean process and executing the command, then waiting for the q p = mp.Process(target=photo_worker, args=(photo_cmd + " -o {}".format(path_to_bright),)) try: p.start() p.join() except OSError: d_print("Could not start child process, out of memory", 3) return (None, 0) # turn off the light self.light_control(channel, 0) # start the dark image capture by spawning a clean process and executing the command, then waiting for the q p = mp.Process(target=photo_worker, args=(photo_cmd + " -o {}".format(path_to_dark),)) try: p.start() p.join() except OSError: d_print("Could not start child process, out of memory", 3) return (None, 0) # load the images from file, perform dark frame subtraction and return the array bright = Image.open(path_to_bright) rgb = np.array(bright) if channel != LC.GROWTH: dark = Image.open(path_to_dark) rgb = cv2.subtract(rgb, np.array(dark)) # if the time since last update is larger than a day, update the gains after the photo if time.time() - self.config["d2d"]["timestamp"] > 3600*24: self.update() return (rgb, gain)
def light_control(channel: LC, state): d_print( "Setting {} camera lighting state to {}".format(channel, state), 1) if channel == LC.WHITE: pi.write(17, state) elif channel == LC.RED: pi.write(3, state) elif channel == LC.NIR: pi.write(4, state) else: d_print("no such light available", 3) time.sleep(0.1)
def extract_value_from_rgb(self, channel: LC, rgb): """ Subfunction used to extract the right value matrix from the rgb image. :param channel: channel of light of which the value matrix is needed :param rgb: rgb matrix (original photo) :return: value matrix """ if channel == LC.NIR: # turn rgb into hsv and extract the v channel as the mask hsv = cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) v = hsv[:,:,2] elif channel == LC.RED: # extract the red channel v = rgb[:,:,0] else: d_print("unknown channel {}, cannot extract value, returning black matrix...".format(channel), 3) v = np.zeros([10, 10]) return v
def photo(self, channel: LC): """ Make a photo with the specified light channel and save the image to disk :param channel: channel of light a photo needs to be taken from :return: path to the photo taken """ # capture a photo of the appropriate channel rgb, _ = self.capture(channel) # catch error if rgb is None: res = dict() res["contains_photo"] = False res["contains_value"] = False res["encountered_error"] = True res["timestamp"] = curr_time return res # crop the sensor readout rgb = rgb[self.settings.crop["y_min"]:self.settings.crop["y_max"], self.settings.crop["x_min"]:self.settings.crop["x_max"], :] # write image to file using imageio's imwrite d_print("Writing to file...", 1) curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") path_to_img = "{}/cam/img/{}_{}.jpg".format(self.working_directory, channel, curr_time) imwrite(path_to_img, rgb) res = dict() res["contains_photo"] = True res["contains_value"] = False res["encountered_error"] = False res["timestamp"] = curr_time res["photo_path"] = [path_to_img] res["photo_kind"] = [channel] return(res)
def do(self, command: CC): """ Function that directs the commands from the user to the right place. Does some preliminary checks to see if actions are allowed in the current state of the camera (uncalibrated etc.). Throws an error on the command line if actions are illegal. :param command: (C)amera (C)ommand, what the user wants to do. """ if command == CC.WHITE_PHOTO and LC.WHITE in self.light_channels and self.CALIBRATED: return self.photo(LC.WHITE) elif command == CC.NDVI_PHOTO and self.NDVI_CAPABLE and self.CALIBRATED: return self.ndvi.ndvi_photo() elif command == CC.GROWTH_PHOTO and LC.GROWTH in self.light_channels and self.CALIBRATED: return self.photo(LC.GROWTH) elif command == CC.NDVI and self.NDVI_CAPABLE and self.CALIBRATED: return self.ndvi.ndvi() elif command == CC.NIR_PHOTO and LC.NIR in self.light_channels and self.CALIBRATED: return self.photo(LC.NIR) elif command == CC.CALIBRATE: self.calibrate() elif command == CC.UPDATE and self.HAS_UPDATE and self.CALIBRATED: self.update() else: d_print("Camera is unable to perform command '{}', is it calibrated? ({})\n Run [cam].state() to check current camera and lighting status...\n Returning empty...".format(command, self.CALIBRATED), 3) return ""
def __init__(self, *args, light_control = light_control_dummy, light_channels = [LC.GROWTH], settings, working_directory = os.getcwd(), **kwargs): """ Initialize an object that contains the visible routines. Link the pi and gpio pins necessary and provide a function that controls the growth lighting. :param light_control: function that allows control over the lighting. Parameters are the channel to control and either a 0 or 1 for off and on respectively :param light_channels: list containing allowable light channels :param settings: reference to one of the settings object contained in this file. These settings are used to control shutter speeds, resolutions, crops etc. """ # set up the camera super class super().__init__(light_control = light_control, working_directory = working_directory) # give the camera a unique ID per brand/kind/etc, software uses this ID to determine whether the # camera is calibrated or not self.CAM_ID = 2 # enable update function to update gains self.HAS_UPDATE = True # bind the settings to the camera object self.settings = settings # set up the light channel array self.light_channels = [] for channel in light_channels: if channel in self.settings.allowed_channels: self.light_channels.append(channel) # load config file and check if it matches the cam id, if so, assume calibrated try: self.load_config_from_file() if self.config["cam_id"] == self.CAM_ID: self.CALIBRATED = True d_print("Succesfully loaded suitable camera configuration.", 1) else: self.CALIBRATED = False d_print("Found camera configuration file, but contents are not suitable for current camera.", 3) except (EnvironmentError, ValueError): d_print("No suitable camera configuration file found!", 3) self.CALIBRATED = False # set multiprocessing to spawn (so NOT fork) try: mp.set_start_method('spawn') except RuntimeError: pass
def update(self): """ Function that updates the gains needed to expose the image correctly. Saves it to the configuration file. """ # check if gain information is available, if not, update config if "d2d" not in self.config: self.setup_d2d() for channel in self.light_channels: # turn on the light self.light_control(channel, 1) d_print("Letting gains settle for the {} channel...".format(channel), 1) with picamera.PiCamera() as sensor: # set up the sensor with all its settings sensor.resolution = self.settings.resolution sensor.framerate = self.settings.framerate[channel] sensor.shutter_speed = self.settings.shutter_speed[channel] sensor.awb_mode = "off" sensor.awb_gains = (self.config["wb"][channel]["r"], self.config["wb"][channel]["b"]) time.sleep(30) sensor.exposure_mode = self.settings.exposure_mode # set the analog and digital gain ag = float(sensor.analog_gain) dg = float(sensor.digital_gain) self.config["d2d"][channel]["digital-gain"] = dg self.config["d2d"][channel]["analog-gain"] = ag d_print("Measured ag: {} and dg: {} for channel {}".format(ag, dg, channel), 1) d_print("Saved ag: {} and dg: {} for channel {}".format(self.config["d2d"][channel]["analog-gain"], self.config["d2d"][channel]["digital-gain"], channel), 1) # turn the light off self.light_control(channel, 0) # update timestamp self.config["d2d"]["timestamp"] = time.time() # save the new configuration to file self.save_config_to_file()
def calibrate_white_balance(self, channel: LC): """ Function that calibrates the white balance for certain lighting specified in the channel parameter. This is camera specific, so it needs to be specified for each camera. :param channel: light channel that needs to be calibrated """ d_print("Warming up camera sensor...", 1) # turn on channel light self.light_control(channel, 1) if channel == LC.WHITE or channel == LC.NIR: with picamera.PiCamera() as sensor: # set up the sensor with all its settings sensor.resolution = (128, 80) sensor.rotation = self.config["rotation"] sensor.framerate = self.settings.framerate[channel] sensor.shutter_speed = self.settings.shutter_speed[channel] # set up the blue and red gains sensor.awb_mode = "off" rg, bg = (1.1, 1.1) sensor.awb_gains = (rg, bg) # now sleep and lock exposure time.sleep(20) sensor.exposure_mode = self.settings.exposure_mode # record camera data to array and scale up a numpy array #rgb = np.zeros((1216,1216,3), dtype=np.uint16) with picamera.array.PiRGBArray(sensor) as output: # capture images and analyze until convergence for i in range(30): output.truncate(0) sensor.capture(output, 'rgb') rgb = np.copy(output.array) #crop = rgb[508:708,666:966,:] crop = rgb[30:50, 32:96, :] r, g, b = (np.mean(crop[..., i]) for i in range(3)) d_print( "\trg: {:4.3f} bg: {:4.3f} --- ({:4.1f}, {:4.1f}, {:4.1f})" .format(rg, bg, r, g, b), 1) if abs(r - g) > 1: if r > g: rg -= 0.025 else: rg += 0.025 if abs(b - g) > 1: if b > g: bg -= 0.025 else: bg += 0.025 sensor.awb_gains = (rg, bg) elif channel == LC.GROWTH: rg = self.settings.wb[LC.GROWTH]["r"] bg = self.settings.wb[LC.GROWTH]["b"] else: rg = self.settings.wb[LC.RED]["r"] bg = self.settings.wb[LC.RED]["b"] # turn off channel light self.light_control(channel, 0) self.config["wb"][channel] = dict() self.config["wb"][channel]["r"] = rg self.config["wb"][channel]["b"] = bg d_print("Done.", 1)
def update(self): """ Function that updates the gains needed to expose the image correctly. Saves it to the configuration file. """ # check if gain information is available, if not, update config if "d2d" not in self.config: self.setup_d2d() for channel in self.light_channels: # turn on the light self.light_control(channel, 1) d_print( "Letting gains settle for the {} channel...".format(channel), 1) with picamera.PiCamera() as sensor: # set up the sensor with all its settings sensor.resolution = self.settings.resolution sensor.framerate = self.settings.framerate[channel] sensor.shutter_speed = self.settings.shutter_speed[channel] sensor.awb_mode = "off" sensor.awb_gains = (self.config["wb"][channel]["r"], self.config["wb"][channel]["b"]) time.sleep(30) sensor.exposure_mode = self.settings.exposure_mode # there is some non-linearity in the camera for higher pixel values when the gain gets large. # currently, we fix this by limiting the gain to a maximum of 1.5 times the calibration gain. # images will be a bit underexposed, but this can be expected for the red images anyway, since they # tend to be dark. ag = float(sensor.analog_gain) dg = float(sensor.digital_gain) if self.CALIBRATED: if channel == LC.RED or channel == LC.NIR: if self.config["ff"]["gain"][channel] * 1.5 < ag: self.config["d2d"][channel][ "analog-gain"] = self.config["ff"]["gain"][ channel] * 1.5 elif self.config["ff"]["gain"][channel] * 0.67 > ag: self.config["d2d"][channel][ "analog-gain"] = self.config["ff"]["gain"][ channel] * 0.67 else: self.config["d2d"][channel]["analog-gain"] = ag else: self.config["d2d"][channel]["analog-gain"] = ag if dg > 2 and (channel == LC.RED or channel == LC.NIR): self.config["d2d"][channel]["digital-gain"] = 2 else: self.config["d2d"][channel]["digital-gain"] = dg else: self.config["d2d"][channel]["digital-gain"] = dg self.config["d2d"][channel]["analog-gain"] = ag d_print( "Measured ag: {} and dg: {} for channel {}".format( ag, dg, channel), 1) d_print( "Saved ag: {} and dg: {} for channel {}".format( self.config["d2d"][channel]["analog-gain"], self.config["d2d"][channel]["digital-gain"], channel), 1) # turn the light off self.light_control(channel, 0) # update timestamp self.config["d2d"]["timestamp"] = time.time() # save the new configuration to file self.save_config_to_file()
def light_control_dummy(channel: LC, state): d_print("light_control dummmy called, passing...", 1) time.sleep(0.1)
def ndvi_photo(self): """ Make a photo in the nir and the red spectrum and overlay to obtain ndvi. :return: (path to the ndvi image, average ndvi value for >0.25 (iff the #pixels is larger than 2 percent of the total)) """ # get the ndvi matrix ndvi_matrix = self.ndvi_matrix() # catch error if ndvi_matrix is None: res = dict() res["contains_photo"] = False res["contains_value"] = False res["encountered_error"] = True res["timestamp"] = curr_time return res ndvi_matrix = np.clip(ndvi_matrix, -1.0, 1.0) if np.count_nonzero(ndvi_matrix > 0.25) > 0.02 * np.size(ndvi_matrix): ndvi = np.mean(ndvi_matrix[ndvi_matrix > 0.25]) else: ndvi = 0 rescaled = np.uint8(np.round(127.5 * (ndvi_matrix + 1.0))) ndvi_plot = np.copy(ndvi_matrix) ndvi_plot[ndvi_plot < 0.25] = np.nan # write images to file using imageio's imwrite and matplotlibs savefig d_print("Writing to file...", 1) curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # set multiprocessing to spawn (so NOT fork) try: mp.set_start_method('spawn') except RuntimeError: pass # write the matplotlib part in a separate process so no memory leaks path_to_img_2 = "{}/cam/img/{}{}_{}.jpg".format( self.camera.working_directory, "ndvi", 2, curr_time) p = mp.Process(target=plotter, args=( ndvi_plot, path_to_img_2, )) try: p.start() p.join() except OSError: d_print("Could not start child process, out of memory", 3) res = dict() res["contains_photo"] = False res["contains_value"] = False res["encountered_error"] = True res["timestamp"] = curr_time return res path_to_img_1 = "{}/cam/img/{}{}_{}.tif".format( self.camera.working_directory, "ndvi", 1, curr_time) imwrite(path_to_img_1, rescaled) res = dict() res["contains_photo"] = True res["contains_value"] = True res["encountered_error"] = False res["timestamp"] = curr_time res["photo_path"] = [path_to_img_1, path_to_img_2] res["photo_kind"] = ["raw NDVI", "processed NDVI"] res["value"] = [ndvi] res["value_kind"] = ["NDVI"] res["value_error"] = [0.0] return res