def __init__(self, target_od: float, **kwargs) -> None:
super(PIDTurbidostat, self).__init__(**kwargs)
assert target_od is not None, "`target_od` must be set"
with local_persistant_storage("pump_calibration") as cache:
if "media_ml_calibration" not in cache:
raise RuntimeError("Media pump calibration must be performed first.")
elif "waste_ml_calibration" not in cache:
raise RuntimeError("Waste pump calibration must be performed first.")
self.set_target_od(target_od)
self.volume_to_cycle = None
# PID%20controller%20turbidostat.ipynb
Kp = config.getfloat("dosing_automation.pid_turbidostat", "Kp")
Ki = config.getfloat("dosing_automation.pid_turbidostat", "Ki")
Kd = config.getfloat("dosing_automation.pid_turbidostat", "Kd")
self.pid = PID(
-Kp,
-Ki,
-Kd,
setpoint=self.target_od,
sample_time=None,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="od",
)
def __init__(self,
target_growth_rate=None,
target_od=None,
volume=None,
**kwargs):
super(PIDMorbidostat, self).__init__(**kwargs)
assert target_od is not None, "`target_od` must be set"
assert target_growth_rate is not None, "`target_growth_rate` must be set"
self.set_target_growth_rate(target_growth_rate)
self.target_od = float(target_od)
Kp = config.getfloat("pid_morbidostat", "Kp")
Ki = config.getfloat("pid_morbidostat", "Ki")
Kd = config.getfloat("pid_morbidostat", "Kd")
self.pid = PID(
-Kp,
-Ki,
-Kd,
setpoint=self.target_growth_rate,
output_limits=(0, 1),
sample_time=None,
unit=self.unit,
experiment=self.experiment,
)
if volume is not None:
self.logger.info(
"Ignoring volume parameter; volume set by target growth rate and duration."
)
self.volume = round(
self.target_growth_rate * VIAL_VOLUME * (self.duration / 60), 4)
def __init__(
self,
target_rpm: float,
unit: str,
experiment: str,
rpm_calculator: Optional[RpmCalculator],
hertz: float = 150,
) -> None:
super(Stirrer, self).__init__(job_name="stirring",
unit=unit,
experiment=experiment)
self.logger.debug(f"Starting stirring with initial {target_rpm} RPM.")
self.rpm_calculator = rpm_calculator
if not hardware.is_HAT_present():
self.logger.error("Pioreactor HAT must be present.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
if (self.rpm_calculator
is not None) and not hardware.is_heating_pcb_present():
self.logger.error("Heating PCB must be present to measure RPM.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError(
"Heating PCB must be present to measure RPM.")
pin = hardware.PWM_TO_PIN[config.get("PWM_reverse", "stirring")]
self.pwm = PWM(pin, hertz)
self.pwm.lock()
self.rpm_to_dc_lookup = self.initialize_rpm_to_dc_lookup()
self.target_rpm = target_rpm
self.duty_cycle = self.rpm_to_dc_lookup(self.target_rpm)
# set up PID
self.pid = PID(
Kp=config.getfloat("stirring.pid", "Kp"),
Ki=config.getfloat("stirring.pid", "Ki"),
Kd=config.getfloat("stirring.pid", "Kd"),
setpoint=self.target_rpm,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="rpm",
output_limits=(-20, 20), # avoid whiplashing
)
# set up thread to periodically check the rpm
self.rpm_check_repeated_thread = RepeatedTimer(
17, # 17 and 5 are coprime
self.poll_and_update_dc,
job_name=self.job_name,
run_immediately=True,
run_after=5,
poll_for_seconds=
4, # technically should be a function of the RPM: lower RPM, longer to get sufficient data.
)
def voltage(self) -> float:
import random
import numpy as np
self.gr = self.growth_rate(
self._counter / config.getfloat("od_config", "samples_per_second"))
self.state *= np.exp(
self.gr / 60 / 60 /
config.getfloat("od_config", "samples_per_second") /
25 # divide by 25 from oversampling_count
)
self._counter += 1.0 / 25.0 # divide by 25 from oversampling_count
return self.state + random.normalvariate(0, sigma=self.state * 0.001)
def initialize_extended_kalman_filter(self) -> CultureGrowthEKF:
import numpy as np
initial_state = np.array(
[
self.initial_od,
self.initial_growth_rate,
self.initial_acc,
]
)
initial_covariance = 1e-5 * np.eye(
3
) # empirically selected - TODO: this should probably scale with `expected_dt`
self.logger.debug(f"Initial covariance matrix:\n{str(initial_covariance)}")
acc_std = config.getfloat("growth_rate_kalman", "acc_std")
acc_process_variance = (acc_std * self.expected_dt) ** 2
od_std = config.getfloat("growth_rate_kalman", "od_std")
od_process_variance = (od_std * self.expected_dt) ** 2
rate_std = config.getfloat("growth_rate_kalman", "rate_std")
rate_process_variance = (rate_std * self.expected_dt) ** 2
process_noise_covariance = np.zeros((3, 3))
process_noise_covariance[0, 0] = od_process_variance
process_noise_covariance[1, 1] = rate_process_variance
process_noise_covariance[2, 2] = acc_process_variance
self.logger.debug(
f"Process noise covariance matrix:\n{str(process_noise_covariance)}"
)
observation_noise_covariance = self.create_obs_noise_covariance()
self.logger.debug(
f"Observation noise covariance matrix:\n{str(observation_noise_covariance)}"
)
angles = [
angle
for (_, angle) in config["od_config.photodiode_channel"].items()
if angle in ["45", "90", "135", "180"]
]
self.logger.debug(f"{angles=}")
return CultureGrowthEKF(
initial_state,
initial_covariance,
process_noise_covariance,
observation_noise_covariance,
angles,
)
def __init__(self, target_temperature: float, **kwargs) -> None:
super(Stable, self).__init__(**kwargs)
assert target_temperature is not None, "target_temperature must be set"
self.target_temperature = float(target_temperature)
self.pid = PID(
Kp=config.getfloat("temperature_automation.stable", "Kp"),
Ki=config.getfloat("temperature_automation.stable", "Ki"),
Kd=config.getfloat("temperature_automation.stable", "Kd"),
setpoint=self.target_temperature,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="temperature",
)
def __init__(self, ignore_cache=False, unit=None, experiment=None):
super(GrowthRateCalculator, self).__init__(job_name=JOB_NAME,
unit=unit,
experiment=experiment)
self.ignore_cache = ignore_cache
self.initial_growth_rate = self.set_initial_growth_rate()
self.od_normalization_factors = self.set_od_normalization_factors()
self.od_variances = self.set_od_variances()
self.samples_per_minute = 60 * config.getfloat("od_config.od_sampling",
"samples_per_second")
self.dt = (1 / config.getfloat("od_config.od_sampling",
"samples_per_second") / 60 / 60)
self.ekf, self.angles = self.initialize_extended_kalman_filter()
self.start_passive_listeners()
def __init__(self,
target_growth_rate=None,
target_od=None,
volume=None,
**kwargs):
super(PIDMorbidostat, self).__init__(**kwargs)
assert target_od is not None, "`target_od` must be set"
assert target_growth_rate is not None, "`target_growth_rate` must be set"
with local_persistant_storage("pump_calibration") as cache:
if "media_ml_calibration" not in cache:
raise RuntimeError(
"Media pump calibration must be performed first.")
elif "waste_ml_calibration" not in cache:
raise RuntimeError(
"Waste pump calibration must be performed first.")
elif "alt_media_ml_calibration" not in cache:
raise RuntimeError(
"Alt-Media pump calibration must be performed first.")
self.set_target_growth_rate(target_growth_rate)
self.target_od = float(target_od)
Kp = config.getfloat("dosing_automation.pid_morbidostat", "Kp")
Ki = config.getfloat("dosing_automation.pid_morbidostat", "Ki")
Kd = config.getfloat("dosing_automation.pid_morbidostat", "Kd")
self.pid = PID(
-Kp,
-Ki,
-Kd,
setpoint=self.target_growth_rate,
output_limits=(0, 1),
sample_time=None,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="growth_rate",
)
if volume is not None:
self.logger.info(
"Ignoring volume parameter; volume set by target growth rate and duration."
)
assert isinstance(self.duration, float)
self.volume = round(
self.target_growth_rate * VIAL_VOLUME * (self.duration / 60), 4)
def create_OD_covariance(self, angles):
import numpy as np
d = len(angles)
variances = {
"135": (config.getfloat("growth_rate_kalman", "od_variance") *
self.dt)**2,
"90": (config.getfloat("growth_rate_kalman", "od_variance") *
self.dt)**2,
"45": (config.getfloat("growth_rate_kalman", "od_variance") *
self.dt)**2,
}
OD_covariance = 0 * np.ones((d, d))
for i, a in enumerate(angles):
for k in variances:
if a.startswith(k):
OD_covariance[i, i] = variances[k]
return OD_covariance
def __init__(self, target_od=None, volume=None, **kwargs):
super(PIDTurbidostat, self).__init__(**kwargs)
assert target_od is not None, "`target_od` must be set"
assert volume is not None, "`volume` must be set"
self.set_target_od(target_od)
self.volume = float(volume)
# PID%20controller%20turbidostat.ipynb
Kp = config.getfloat("pid_turbidostat", "Kp")
Ki = config.getfloat("pid_turbidostat", "Ki")
Kd = config.getfloat("pid_turbidostat", "Kd")
self.pid = PID(
-Kp,
-Ki,
-Kd,
setpoint=self.target_od,
output_limits=(0, 1),
sample_time=None,
unit=self.unit,
experiment=self.experiment,
)
def create_obs_noise_covariance(self): # typing: ignore
"""
Our sensor measurements have initial variance V, but in our KF, we scale them their
initial mean, M. Hence the observed variance of the _normalized_ measurements is
var(measurement / M) = V / M^2
(there's also a blank to consider)
However, we offer the variable ods_std to tweak this a bit.
"""
import numpy as np
try:
scaling_obs_variances = np.array(
[
self.od_variances[channel]
/ (self.od_normalization_factors[channel] - self.od_blank[channel])
** 2
for channel in self.od_normalization_factors
]
)
obs_variances = config.getfloat(
"growth_rate_kalman", "obs_std"
) ** 2 * np.diag(scaling_obs_variances)
return obs_variances
except ZeroDivisionError:
self.logger.debug(
"Is there an OD Reading that is 0? Maybe there's a loose photodiode connection?",
exc_info=True,
)
self.logger.error(
"Is there an OD Reading that is 0? Maybe there's a loose photodiode connection?"
)
# we should clear the cache here...
with local_persistant_storage("od_normalization_mean") as cache:
del cache[self.experiment]
with local_persistant_storage("od_normalization_variance") as cache:
del cache[self.experiment]
raise
def __init__(
self,
unit: str,
experiment: str,
ignore_cache: bool = False,
):
super(GrowthRateCalculator, self).__init__(
job_name="growth_rate_calculating", unit=unit, experiment=experiment
)
self.ignore_cache = ignore_cache
self.time_of_previous_observation = datetime.utcnow()
self.expected_dt = 1 / (
60 * 60 * config.getfloat("od_config", "samples_per_second")
)
def click_stirring_calibration(min_dc: int, max_dc: int) -> None:
"""
Generate a lookup between stirring and voltage
"""
if max_dc is None and min_dc is None:
# seed with initial_duty_cycle
config_initial_duty_cycle = round(
config.getfloat("stirring", "initial_duty_cycle"))
min_dc, max_dc = config_initial_duty_cycle - 10, config_initial_duty_cycle + 10
elif (max_dc is not None) and (min_dc is not None):
assert min_dc < max_dc, "min_dc >= max_dc"
else:
raise ValueError("min_dc and max_dc must both be set.")
stirring_calibration(min_dc, max_dc)
def test_positive_correlation_between_rpm_and_stirring(
logger: Logger, unit: str, experiment: str
) -> None:
with local_persistant_storage("stirring_calibration") as cache:
if "linear_v1" in cache:
parameters = loads(cache["linear_v1"])
coef = parameters["rpm_coef"]
intercept = parameters["intercept"]
initial_dc = coef * 700 + intercept
else:
initial_dc = config.getfloat("stirring", "initial_duty_cycle")
dcs = []
measured_rpms = []
n_samples = 8
start = initial_dc
end = initial_dc * 0.66
with stirring.Stirrer(
target_rpm=0, unit=unit, experiment=experiment, rpm_calculator=None
) as st, stirring.RpmFromFrequency() as rpm_calc:
st.duty_cycle = initial_dc
st.start_stirring()
sleep(1)
for i in range(n_samples):
dc = start * (1 - i / n_samples) + (i / n_samples) * end
st.set_duty_cycle(dc)
sleep(1)
measured_rpms.append(rpm_calc(4))
dcs.append(dc)
measured_correlation = round(correlation(dcs, measured_rpms), 2)
logger.debug(
f"Correlation between stirring RPM and duty cycle: {measured_correlation}"
)
logger.debug(f"{dcs=}, {measured_rpms=}")
assert measured_correlation > 0.9, (dcs, measured_rpms)
class AltMediaCalculator:
"""
Computes the fraction of the vial that is from the alt-media vs the regular media.
"""
vial_volume = config.getfloat("bioreactor", "volume_ml")
def update(self, payload: dict, current_alt_media_fraction) -> float:
volume, event = float(payload["volume_change"]), payload["event"]
if event == "add_media":
return self.update_alt_media_fraction(current_alt_media_fraction,
volume, 0)
elif event == "add_alt_media":
return self.update_alt_media_fraction(current_alt_media_fraction,
0, volume)
elif event == "remove_waste":
return current_alt_media_fraction
else:
raise ValueError("Unknown event type")
def update_alt_media_fraction(
self,
current_alt_media_fraction: float,
media_delta: float,
alt_media_delta: float,
) -> float:
total_delta = media_delta + alt_media_delta
# current mL
alt_media_ml = self.vial_volume * current_alt_media_fraction
media_ml = self.vial_volume * (1 - current_alt_media_fraction)
# remove
alt_media_ml = alt_media_ml * (1 - total_delta / self.vial_volume)
media_ml = media_ml * (1 - total_delta / self.vial_volume)
# add (alt) media
alt_media_ml = alt_media_ml + alt_media_delta
media_ml = media_ml + media_delta
return alt_media_ml / self.vial_volume
def initialize_extended_kalman_filter(self):
import numpy as np
latest_od = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/od_raw_batched")
angles_and_initial_points = self.scale_raw_observations(
self.json_to_sorted_dict(latest_od.payload))
initial_state = np.array(
[*angles_and_initial_points.values(), self.initial_growth_rate])
d = initial_state.shape[0]
# empirically selected
initial_covariance = 0.0001 * np.diag(initial_state.tolist()[:-1] +
[0.00001])
OD_process_covariance = self.create_OD_covariance(
angles_and_initial_points.keys())
rate_process_variance = (
config.getfloat("growth_rate_kalman", "rate_variance") *
self.dt)**2
process_noise_covariance = np.block([
[OD_process_covariance, 0 * np.ones((d - 1, 1))],
[0 * np.ones((1, d - 1)), rate_process_variance],
])
observation_noise_covariance = self.create_obs_noise_covariance(
angles_and_initial_points.keys())
return (
ExtendedKalmanFilter(
initial_state,
initial_covariance,
process_noise_covariance,
observation_noise_covariance,
dt=self.dt,
),
angles_and_initial_points.keys(),
)
def start_od_reading(
od_angle_channel1: Optional[pt.PdAngle] = None,
od_angle_channel2: Optional[pt.PdAngle] = None,
sampling_rate: float = 1 / config.getfloat("od_config", "samples_per_second"),
fake_data: bool = False,
unit: Optional[str] = None,
experiment: Optional[str] = None,
) -> ODReader:
unit = unit or get_unit_name()
experiment = experiment or get_latest_experiment_name()
ir_led_reference_channel = find_ir_led_reference(od_angle_channel1, od_angle_channel2)
channel_angle_map = create_channel_angle_map(od_angle_channel1, od_angle_channel2)
channels = list(channel_angle_map.keys())
ir_led_reference_tracker: IrLedReferenceTracker
if ir_led_reference_channel is not None:
ir_led_reference_tracker = PhotodiodeIrLedReferenceTracker(
ir_led_reference_channel, ignore_blank=fake_data
)
channels.append(ir_led_reference_channel)
else:
ir_led_reference_tracker = NullIrLedReferenceTracker()
return ODReader(
channel_angle_map,
interval=sampling_rate,
unit=unit,
experiment=experiment,
adc_reader=ADCReader(
channels=channels, fake_data=fake_data, interval=sampling_rate
),
ir_led_reference_tracker=ir_led_reference_tracker,
)
rpm_calculator = RpmFromFrequency()
stirrer = Stirrer(
target_rpm=target_rpm,
unit=unit,
experiment=experiment,
rpm_calculator=rpm_calculator,
)
stirrer.start_stirring()
return stirrer
@click.command(name="stirring")
@click.option(
"--target-rpm",
default=config.getfloat("stirring", "target_rpm", fallback=0),
help="set the target RPM",
show_default=True,
type=click.FloatRange(0, 1200, clamp=True),
)
@click.option(
"--ignore-rpm",
help="don't use feedback loop",
is_flag=True,
)
def click_stirring(target_rpm, ignore_rpm):
"""
Start the stirring of the Pioreactor.
"""
st = start_stirring(target_rpm=target_rpm, ignore_rpm=ignore_rpm)
st.block_until_disconnected()
def __init__(
self,
channel_angle_map: dict[pt.PdChannel, pt.PdAngle],
interval: float,
adc_reader: ADCReader,
ir_led_reference_tracker: IrLedReferenceTracker,
unit: str,
experiment: str,
) -> None:
super(ODReader, self).__init__(
job_name="od_reading", unit=unit, experiment=experiment
)
self.adc_reader = adc_reader
self.channel_angle_map = channel_angle_map
self.interval = interval
self.ir_led_reference_tracker = ir_led_reference_tracker
self.first_od_obs_time: Optional[float] = None
self.ir_channel: pt.LedChannel = self.get_ir_channel_from_configuration()
self.ir_led_intensity: float = config.getfloat("od_config", "ir_intensity")
self.non_ir_led_channels: list[pt.LedChannel] = [
ch for ch in ALL_LED_CHANNELS if ch != self.ir_channel
]
if not hardware.is_HAT_present():
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
self.logger.debug(
f"Starting od_reading with PD channels {channel_angle_map}, with IR LED intensity {self.ir_led_intensity}% from channel {self.ir_channel}."
)
# setup the ADC and IrLedReference by turning off all LEDs.
with change_leds_intensities_temporarily(
ALL_LED_CHANNELS,
[0.0, 0.0, 0.0, 0.0],
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
):
with lock_leds_temporarily(self.non_ir_led_channels):
# start IR led before ADC starts, as it needs it.
self.start_ir_led()
self.adc_reader.setup_adc() # determine best gain, max-signal, etc.
self.stop_ir_led()
# get blank values of reference PD.
# This slightly improves the accuracy of the IR LED output tracker,
# See that class's docs.
self.ir_led_reference_tracker.set_blank(self.adc_reader.take_reading())
self.record_from_adc_timer = RepeatedTimer(
self.interval,
self.record_and_publish_from_adc,
run_immediately=True,
).start()
def od_blank(
od_angle_channel1,
od_angle_channel2,
n_samples: int = 30,
):
"""
Compute the sample average of the photodiodes attached.
Note that because of the sensitivity of the growth rate (and normalized OD) to the starting values,
we need a very accurate estimate of these statistics.
"""
from statistics import mean, variance
action_name = "od_blank"
logger = create_logger(action_name)
unit = get_unit_name()
experiment = get_latest_experiment_name()
testing_experiment = get_latest_testing_experiment_name()
logger.info(
"Starting reading of blank OD. This will take about a few minutes.")
with publish_ready_to_disconnected_state(unit, experiment, action_name):
# running this will mess with OD Reading - best to just not let it happen.
if (is_pio_job_running("od_reading")
# but if test mode, ignore
and not is_testing_env()):
logger.error(
"od_reading should not be running. Stop od_reading first. Exiting."
)
return
# turn on stirring if not already on
if not is_pio_job_running("stirring"):
# start stirring
st = start_stirring(
target_rpm=config.getint("stirring", "target_rpm"),
unit=unit,
experiment=testing_experiment,
)
st.block_until_rpm_is_close_to_target()
else:
# TODO: it could be paused, we should make sure it's running
...
sampling_rate = 1 / config.getfloat("od_config", "samples_per_second")
# start od_reading
start_od_reading(
od_angle_channel1,
od_angle_channel2,
sampling_rate=sampling_rate,
unit=unit,
experiment=testing_experiment,
fake_data=is_testing_env(),
)
def yield_from_mqtt():
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{testing_experiment}/od_reading/od_raw_batched"
)
yield json.loads(msg.payload)
signal = yield_from_mqtt()
readings = defaultdict(list)
for count, batched_reading in enumerate(signal, start=1):
for (channel, reading) in batched_reading["od_raw"].items():
readings[channel].append(reading["voltage"])
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count // n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
if count == n_samples:
break
means = {}
variances = {}
autocorrelations = {} # lag 1
for channel, od_reading_series in readings.items():
# measure the mean and publish. The mean will be used to normalize the readings in downstream jobs
means[channel] = mean(od_reading_series)
variances[channel] = variance(od_reading_series)
autocorrelations[channel] = correlation(od_reading_series[:-1],
od_reading_series[1:])
# warn users that a blank is 0 - maybe this should be an error instead? TODO: link this to a docs page.
if means[channel] == 0.0:
logger.warning(
f"OD reading for PD Channel {channel} is 0.0 - that shouldn't be. Is there a loose connection, or an extra channel in the configuration's [od_config.photodiode_channel] section?"
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/od_blank/{channel}",
json.dumps({
"timestamp": current_utc_time(),
"od_reading_v": means[channel]
}),
)
# store locally as the source of truth.
with local_persistant_storage(action_name) as cache:
cache[experiment] = json.dumps(means)
# publish to UI and database
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/mean",
json.dumps(means),
qos=pubsub.QOS.AT_LEAST_ONCE,
retain=True,
)
if config.getboolean(
"data_sharing_with_pioreactor",
"send_od_statistics_to_Pioreactor",
fallback=False,
):
to_share = {"mean": means, "variance": variances}
to_share["ir_intensity"] = config["od_config"]["ir_intensity"]
to_share["od_angle_channel1"] = od_angle_channel1
to_share["od_angle_channel2"] = od_angle_channel2
pubsub.publish_to_pioreactor_cloud("od_blank_mean", json=to_share)
logger.debug(f"measured mean: {means}")
logger.debug(f"measured variances: {variances}")
logger.debug(f"measured autocorrelations: {autocorrelations}")
logger.debug("OD normalization finished.")
return means