def __init__(
self,
topic,
output_dir,
extract_label,
ignore_cache=False,
job_name=DEFAULT_JOB_NAME, # this is overwritten importantly in instantiation
record_every_n_seconds=None, # controls how often we should sample data. Ex: growth_rate is ~5min
write_every_n_seconds=None, # controls how often we write to disk. Ex: about 30seconds
time_window_seconds=None,
**kwargs,
):
super(TimeSeriesAggregation, self).__init__(job_name=job_name,
**kwargs)
self.topic = topic
self.output_dir = output_dir
self.aggregated_time_series = self.read(ignore_cache)
self.extract_label = extract_label
self.time_window_seconds = time_window_seconds
self.cache = {}
self.write_thread = RepeatedTimer(write_every_n_seconds,
self.write,
job_name=self.job_name).start()
self.append_cache_thread = RepeatedTimer(
record_every_n_seconds,
self.append_cache_and_clear,
job_name=self.job_name).start()
self.start_passive_listeners()
def __init__(
self,
target_rpm: float,
unit: str,
experiment: str,
rpm_calculator: Optional[RpmCalculator],
hertz: float = 150,
) -> None:
super(Stirrer, self).__init__(job_name="stirring",
unit=unit,
experiment=experiment)
self.logger.debug(f"Starting stirring with initial {target_rpm} RPM.")
self.rpm_calculator = rpm_calculator
if not hardware.is_HAT_present():
self.logger.error("Pioreactor HAT must be present.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
if (self.rpm_calculator
is not None) and not hardware.is_heating_pcb_present():
self.logger.error("Heating PCB must be present to measure RPM.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError(
"Heating PCB must be present to measure RPM.")
pin = hardware.PWM_TO_PIN[config.get("PWM_reverse", "stirring")]
self.pwm = PWM(pin, hertz)
self.pwm.lock()
self.rpm_to_dc_lookup = self.initialize_rpm_to_dc_lookup()
self.target_rpm = target_rpm
self.duty_cycle = self.rpm_to_dc_lookup(self.target_rpm)
# set up PID
self.pid = PID(
Kp=config.getfloat("stirring.pid", "Kp"),
Ki=config.getfloat("stirring.pid", "Ki"),
Kd=config.getfloat("stirring.pid", "Kd"),
setpoint=self.target_rpm,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="rpm",
output_limits=(-20, 20), # avoid whiplashing
)
# set up thread to periodically check the rpm
self.rpm_check_repeated_thread = RepeatedTimer(
17, # 17 and 5 are coprime
self.poll_and_update_dc,
job_name=self.job_name,
run_immediately=True,
run_after=5,
poll_for_seconds=
4, # technically should be a function of the RPM: lower RPM, longer to get sufficient data.
)
def set_duration(self, value):
self.duration = float(value)
try:
self.timer_thread.cancel()
except AttributeError:
pass
finally:
if self.duration is not None:
self.timer_thread = RepeatedTimer(
self.duration * 60,
self.run,
job_name=self.job_name,
run_immediately=(not self.skip_first_run),
).start()
def __init__(self, unit=None, experiment=None, **kwargs) -> None:
super(AltMediaCalculator, self).__init__(job_name=JOB_NAME,
unit=unit,
experiment=experiment)
self.latest_alt_media_fraction = self.get_initial_alt_media_fraction()
# publish often to fill in gaps in UI chart.
self.publish_periodically_thead = RepeatedTimer(
5 * 60,
self.publish_latest_alt_media_fraction,
job_name=self.job_name)
self.publish_periodically_thead.start()
self.start_passive_listeners()
def __init__(self):
self.thread = RepeatedTimer(
3,
self.run,
run_immediately=True,
).start()
def set_duration(self, duration: Optional[float]) -> None:
if duration:
self.duration = float(duration)
with suppress(AttributeError):
self.run_thread.cancel() # type: ignore
if self._latest_run_at:
# what's the correct logic when changing from duration N and duration M?
# - N=20, and it's been 5m since the last run (or initialization). I change to M=30, I should wait M-5 minutes.
# - N=60, and it's been 50m since last run. I change to M=30, I should run immediately.
run_after = max(0, (self.duration * 60) -
(time.time() - self._latest_run_at))
else:
# there is a race condition here: self.run() will run immediately (see run_immediately), but the state of the job is not READY, since
# set_duration is run in the __init__ (hence the job is INIT). So we wait 2 seconds for the __init__ to finish, and then run.
run_after = 2
self.run_thread = RepeatedTimer(
self.duration * 60,
self.run,
job_name=self.job_name,
run_immediately=(not self.skip_first_run)
or (self._latest_run_at is not None),
run_after=run_after,
).start()
else:
self.duration = None
self.run_thread = Thread(target=self.run, daemon=True)
self.run_thread.start()
def __init__(self, unit, experiment) -> None:
super().__init__(job_name="monitor", unit=unit, experiment=experiment)
def to_version(info: tuple[int, ...]) -> str:
return ".".join((str(x) for x in info))
self.logger.info(
f"Pioreactor software version: {to_version(software_version_info)}"
)
self.logger.info(
f"Pioreactor HAT version: {to_version(hardware_version_info)}")
# set up GPIO for accessing the button and changing the LED
self.setup_GPIO()
# set up a self check function to periodically check vitals and log them
# we manually run a self_check outside of a thread first, as if there are
# problems detected, we may want to block and not let the job continue.
self.self_checks()
self.self_check_thread = RepeatedTimer(
8 * 60 * 60,
self.self_checks,
job_name=self.job_name,
run_immediately=False,
).start()
self.start_passive_listeners()
def __init__(self, run_immediately):
self.thread = RepeatedTimer(
5,
self.run,
run_immediately=run_immediately,
).start()
def test_90_angle(self) -> None:
import json
import numpy as np
from pioreactor.utils.timing import RepeatedTimer
unit = get_unit_name()
experiment = get_latest_experiment_name()
samples_per_second = 0.2
config["od_config"]["samples_per_second"] = str(samples_per_second)
config["od_config.photodiode_channel"]["1"] = "90"
config["od_config.photodiode_channel"]["2"] = None
with local_persistant_storage("od_normalization_mean") as cache:
cache[experiment] = json.dumps({"1": 0.1})
with local_persistant_storage("od_normalization_variance") as cache:
cache[experiment] = json.dumps({"1": 8.2e-02})
class Mock180ODReadings:
growth_rate = 0.1
od_reading = 1.0
def __call__(self):
self.od_reading *= np.exp(self.growth_rate / 60 / 60 / samples_per_second)
voltage = 0.1 * self.od_reading
payload = {
"od_raw": {"1": {"voltage": voltage, "angle": "90"}},
"timestamp": "2021-06-06T15:08:12.081153",
}
publish(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
json.dumps(payload),
)
thread = RepeatedTimer(0.025, Mock180ODReadings()).start()
with GrowthRateCalculator(unit=unit, experiment=experiment) as calc:
time.sleep(35)
assert calc.ekf.state_[1] > 0
thread.cancel()
def __init__(self,
sampling_rate=1,
fake_data=False,
unit=None,
experiment=None):
super(ADCReader, self).__init__(job_name="adc_reader",
unit=unit,
experiment=experiment)
self.fake_data = fake_data
self.ma = MovingStats(lookback=10)
self.timer = RepeatedTimer(sampling_rate, self.take_reading)
self.counter = 0
self.ads = None
self.analog_in = []
def __init__(self, unit, experiment):
super(Monitor, self).__init__(job_name=JOB_NAME, unit=unit, experiment=experiment)
self.disk_usage_timer = RepeatedTimer(
12 * 60 * 60,
self.publish_disk_space,
job_name=self.job_name,
run_immediately=True,
)
GPIO.setup(BUTTON_PIN, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
GPIO.setup(LED_PIN, GPIO.OUT)
GPIO.add_event_detect(BUTTON_PIN, GPIO.RISING, callback=self.button_down_and_up)
self.start_passive_listeners()
self.flicker_led()
class DosingAutomation(BackgroundSubJob):
"""
This is the super class that automations inherit from. The `run` function will
execute every `duration` minutes (selected at the start of the program). If `duration` is left
as None, manually call `run`. This calls the `execute` function, which is what subclasses will define.
To change setting over MQTT:
`pioreactor/<unit>/<experiment>/dosing_automation/<setting>/set` value
"""
latest_growth_rate = None
latest_od = None
latest_od_timestamp = None
latest_growth_rate_timestamp = None
latest_settings_started_at = current_time()
latest_settings_ended_at = None
editable_settings = [
"volume", "target_od", "target_growth_rate", "duration"
]
sub_jobs = []
def __init__(
self,
unit=None,
experiment=None,
duration=60,
sensor="135/0",
skip_first_run=False,
**kwargs,
):
super(DosingAutomation, self).__init__(job_name="dosing_automation",
unit=unit,
experiment=experiment)
self.latest_event = None
self.sensor = sensor
self.skip_first_run = skip_first_run
# the below subjobs should run in the "init()"?
self.alt_media_calculator = AltMediaCalculator(
unit=self.unit, experiment=self.experiment)
self.throughput_calculator = ThroughputCalculator(
unit=self.unit, experiment=self.experiment)
self.sub_jobs.extend(
[self.alt_media_calculator, self.throughput_calculator])
self.set_duration(duration)
self.start_passive_listeners()
self.logger.info(
f"starting {self.__class__.__name__} with {duration}min intervals, metadata: {kwargs}"
)
def set_duration(self, value):
self.duration = float(value)
try:
self.timer_thread.cancel()
except AttributeError:
pass
finally:
if self.duration is not None:
self.timer_thread = RepeatedTimer(
self.duration * 60,
self.run,
job_name=self.job_name,
run_immediately=(not self.skip_first_run),
).start()
def run(self, counter=None):
time.sleep(8) # wait some time for data to arrive
if (self.latest_growth_rate is None) or (self.latest_od is None):
self.logger.debug("Waiting for OD and growth rate data to arrive")
if not ("od_reading" in pio_jobs_running()) and (
"growth_rate_calculating" in pio_jobs_running()):
self.logger.warn(
"`od_reading` and `growth_rate_calculating` should be running."
)
event = events.NoEvent(
"waiting for OD and growth rate data to arrive")
elif self.state != self.READY:
event = events.NoEvent(f"currently in state {self.state}")
elif (time.time() - self.most_stale_time) > 5 * 60:
event = events.NoEvent(
"readings are too stale (over 5 minutes old) - are `od_reading` and `growth_rate_calculating` running?"
)
else:
try:
event = self.execute(counter)
except Exception as e:
self.logger.debug(e, exc_info=True)
self.logger.error(e)
event = events.NoEvent("")
self.logger.info(f"triggered {event}.")
self.latest_event = event
return event
def execute(self, counter) -> events.Event:
raise NotImplementedError
def execute_io_action(self, alt_media_ml=0, media_ml=0, waste_ml=0):
assert (
abs(alt_media_ml + media_ml - waste_ml) < 1e-5
), f"in order to keep same volume, IO should be equal. {alt_media_ml}, {media_ml}, {waste_ml}"
max_ = 0.3
if alt_media_ml > max_:
self.execute_io_action(
alt_media_ml=alt_media_ml / 2,
media_ml=media_ml,
waste_ml=media_ml + alt_media_ml / 2,
)
self.execute_io_action(alt_media_ml=alt_media_ml / 2,
media_ml=0,
waste_ml=alt_media_ml / 2)
elif media_ml > max_:
self.execute_io_action(alt_media_ml=0,
media_ml=media_ml / 2,
waste_ml=media_ml / 2)
self.execute_io_action(
alt_media_ml=alt_media_ml,
media_ml=media_ml / 2,
waste_ml=alt_media_ml + media_ml / 2,
)
else:
if alt_media_ml > 0:
add_alt_media(
ml=alt_media_ml,
source_of_event=self.job_name,
unit=self.unit,
experiment=self.experiment,
)
brief_pause(
) # allow time for the addition to mix, and reduce the step response that can cause ringing in the output V.
if media_ml > 0:
add_media(
ml=media_ml,
source_of_event=self.job_name,
unit=self.unit,
experiment=self.experiment,
)
brief_pause()
if waste_ml > 0:
remove_waste(
ml=waste_ml,
source_of_event=self.job_name,
unit=self.unit,
experiment=self.experiment,
)
# run remove_waste for an additional few seconds to keep volume constant (determined by the length of the waste tube)
remove_waste(
duration=2,
source_of_event=self.job_name,
unit=self.unit,
experiment=self.experiment,
)
brief_pause()
@property
def most_stale_time(self):
return min(self.latest_od_timestamp, self.latest_growth_rate_timestamp)
########## Private & internal methods
def on_disconnect(self):
self.latest_settings_ended_at = current_time()
self._send_details_to_mqtt()
try:
self.timer_thread.cancel()
except AttributeError:
self.logger.debug("no timer_thread", exc_info=True)
for job in self.sub_jobs:
job.set_state("disconnected")
self._clear_mqtt_cache()
def __setattr__(self, name, value) -> None:
super(DosingAutomation, self).__setattr__(name, value)
if name in self.editable_settings and name != "state":
self.latest_settings_ended_at = current_time()
self._send_details_to_mqtt()
self.latest_settings_started_at = current_time()
self.latest_settings_ended_at = None
def _set_growth_rate(self, message):
self.previous_growth_rate = self.latest_growth_rate
self.latest_growth_rate = float(message.payload)
self.latest_growth_rate_timestamp = time.time()
def _set_OD(self, message):
self.previous_od = self.latest_od
self.latest_od = float(message.payload)
self.latest_od_timestamp = time.time()
def _clear_mqtt_cache(self):
# From homie: Devices can remove old properties and nodes by publishing a zero-length payload on the respective topics.
for attr in self.editable_settings:
if attr == "state":
continue
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/{attr}",
None,
retain=True,
qos=QOS.EXACTLY_ONCE,
)
def _send_details_to_mqtt(self):
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/dosing_automation_settings",
json.dumps({
"pioreactor_unit":
self.unit,
"experiment":
self.experiment,
"started_at":
self.latest_settings_started_at,
"ended_at":
self.latest_settings_ended_at,
"automation":
self.__class__.__name__,
"settings":
json.dumps({
attr: getattr(self, attr, None)
for attr in self.editable_settings if attr != "state"
}),
}),
qos=QOS.EXACTLY_ONCE,
retain=True,
)
def start_passive_listeners(self):
self.subscribe_and_callback(
self._set_OD,
f"pioreactor/{self.unit}/{self.experiment}/od_filtered/{self.sensor}",
)
self.subscribe_and_callback(
self._set_growth_rate,
f"pioreactor/{self.unit}/{self.experiment}/growth_rate")
def __init__(
self,
automation_name: str,
unit: str,
experiment: str,
eval_and_publish_immediately: bool = True,
**kwargs,
) -> None:
super().__init__(job_name="temperature_control",
unit=unit,
experiment=experiment)
if not is_HAT_present():
self.logger.error("Pioreactor HAT must be present.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
if not is_heating_pcb_present():
self.logger.error("Heating PCB must be attached to Pioreactor HAT")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError(
"Heating PCB must be attached to Pioreactor HAT")
if is_testing_env():
self.logger.debug("TMP1075 not available; using MockTMP1075")
from pioreactor.utils.mock import MockTMP1075 as TMP1075
else:
from TMP1075 import TMP1075 # type: ignore
self.pwm = self.setup_pwm()
self.update_heater(0)
self.tmp_driver = TMP1075()
self.read_external_temperature_timer = RepeatedTimer(
45, self.read_external_temperature, run_immediately=False)
self.read_external_temperature_timer.start()
self.publish_temperature_timer = RepeatedTimer(
4 * 60,
self.evaluate_and_publish_temperature,
run_immediately=eval_and_publish_immediately,
run_after=60,
)
self.publish_temperature_timer.start()
self.automation = AutomationDict(automation_name=automation_name,
**kwargs)
try:
automation_class = self.automations[
self.automation["automation_name"]]
except KeyError:
raise KeyError(
f"Unable to find automation {self.automation['automation_name']}. Available automations are {list(self.automations.keys())}"
)
self.logger.info(f"Starting {self.automation}.")
try:
self.automation_job = automation_class(unit=self.unit,
experiment=self.experiment,
parent=self,
**kwargs)
except Exception as e:
self.logger.error(e)
self.logger.debug(e, exc_info=True)
self.set_state(self.DISCONNECTED)
raise e
self.automation_name = self.automation["automation_name"]
self.temperature = {
"temperature": self.read_external_temperature(),
"timestamp": current_utc_time(),
}
def __init__(self, run_after):
self.thread = RepeatedTimer(5,
self.run,
run_immediately=True,
run_after=run_after).start()
class TimeSeriesAggregation(BackgroundJob):
"""
This aggregates data _regardless_ of the experiment - users can choose to clear it (using the button), but better would
be for the UI to clear it on new experiment creation.
"""
def __init__(
self,
topic,
output_dir,
extract_label,
ignore_cache=False,
job_name=DEFAULT_JOB_NAME, # this is overwritten importantly in instantiation
record_every_n_seconds=None, # controls how often we should sample data. Ex: growth_rate is ~5min
write_every_n_seconds=None, # controls how often we write to disk. Ex: about 30seconds
time_window_seconds=None,
**kwargs,
):
super(TimeSeriesAggregation, self).__init__(job_name=job_name,
**kwargs)
self.topic = topic
self.output_dir = output_dir
self.aggregated_time_series = self.read(ignore_cache)
self.extract_label = extract_label
self.time_window_seconds = time_window_seconds
self.cache = {}
self.write_thread = RepeatedTimer(write_every_n_seconds,
self.write,
job_name=self.job_name).start()
self.append_cache_thread = RepeatedTimer(
record_every_n_seconds,
self.append_cache_and_clear,
job_name=self.job_name).start()
self.start_passive_listeners()
def on_disconnect(self):
self.write_thread.cancel()
self.append_cache_thread.cancel()
@property
def output(self):
return self.output_dir + self.job_name + ".json"
def read(self, ignore_cache):
if ignore_cache:
return {"series": [], "data": []}
try:
# try except hell
with open(self.output, "r") as f:
return json.loads(f.read())
except (OSError, FileNotFoundError) as e:
self.logger.debug(f"Loading failed or not found. {str(e)}")
return {"series": [], "data": []}
except Exception as e:
self.logger.debug(f"Loading failed or not found. {str(e)}")
return {"series": [], "data": []}
def write(self):
self.latest_write = current_time()
with open(self.output, mode="wt") as f:
json.dump(self.aggregated_time_series, f)
def append_cache_and_clear(self):
self.update_data_series()
self.cache = {}
def update_data_series(self):
time = current_time()
# .copy because a thread may try to update this while iterating.
for (label, latest_value) in self.cache.copy().items():
if label not in self.aggregated_time_series["series"]:
self.aggregated_time_series["series"].append(label)
self.aggregated_time_series["data"].append([])
ix = self.aggregated_time_series["series"].index(label)
self.aggregated_time_series["data"][ix].append({
"x": time,
"y": latest_value
})
if self.time_window_seconds:
for ix, _ in enumerate(self.aggregated_time_series["data"]):
# this is pretty inefficient, but okay for now.
self.aggregated_time_series["data"][ix] = [
point for point in self.aggregated_time_series["data"][ix]
if point["x"] > (current_time() -
self.time_window_seconds * 1000)
]
def on_message(self, message):
label = self.extract_label(message.topic)
try:
self.cache[label] = float(message.payload)
except ValueError:
# sometimes a empty string is sent to clear the MQTT cache - that's okay - just pass.
pass
def on_clear(self, message):
payload = message.payload
if not payload:
self.cache = {}
self.aggregated_time_series = {"series": [], "data": []}
self.write()
else:
self.logger.warning(
"Only empty messages allowed to empty the cache.")
def start_passive_listeners(self):
self.subscribe_and_callback(self.on_message,
self.topic,
qos=QOS.EXACTLY_ONCE,
allow_retained=False)
self.subscribe_and_callback(
self.on_clear,
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/aggregated_time_series/set",
qos=QOS.AT_LEAST_ONCE,
)
class LEDAutomation(BackgroundSubJob):
"""
This is the super class that LED automations inherit from. The `run` function will
execute every `duration` minutes (selected at the start of the program). If `duration` is left
as None, manually call `run`. This calls the `execute` function, which is what subclasses will define.
To change setting over MQTT:
`pioreactor/<unit>/<experiment>/led_automation/<setting>/set` value
"""
latest_growth_rate = None
latest_od = None
latest_od_timestamp = None
latest_growth_rate_timestamp = None
latest_settings_started_at = current_time()
latest_settings_ended_at = None
editable_settings = ["duration"]
edited_channels = []
sub_jobs = []
def __init__(
self,
unit=None,
experiment=None,
duration=60,
sensor="135/0",
skip_first_run=False,
**kwargs,
):
super(LEDAutomation, self).__init__(job_name="led_automation",
unit=unit,
experiment=experiment)
self.latest_event = None
self.sensor = sensor
self.skip_first_run = skip_first_run
self.set_duration(duration)
self.start_passive_listeners()
self.logger.info(
f"starting {self.__class__.__name__} with {duration}min intervals, metadata: {kwargs}"
)
def set_duration(self, value):
self.duration = float(value)
try:
self.timer_thread.cancel()
except AttributeError:
pass
finally:
if self.duration is not None:
self.timer_thread = RepeatedTimer(
self.duration * 60,
self.run,
job_name=self.job_name,
run_immediately=(not self.skip_first_run),
).start()
def run(self, counter=None):
time.sleep(8) # wait some time for data to arrive
if (self.latest_growth_rate is None) or (self.latest_od is None):
self.logger.debug("Waiting for OD and growth rate data to arrive")
if not ("od_reading" in pio_jobs_running()) and (
"growth_rate_calculating" in pio_jobs_running()):
self.logger.warn(
"`od_reading` and `growth_rate_calculating` should be running."
)
event = events.NoEvent(
"waiting for OD and growth rate data to arrive")
elif self.state != self.READY:
event = events.NoEvent(f"currently in state {self.state}")
elif (time.time() - self.most_stale_time) > 5 * 60:
event = events.NoEvent(
"readings are too stale (over 5 minutes old) - are `od_reading` and `growth_rate_calculating` running?"
)
else:
try:
event = self.execute(counter)
except Exception as e:
self.logger.debug(e, exc_info=True)
self.logger.error(e)
event = events.NoEvent("")
self.logger.info(f"triggered {event}.")
self.latest_event = event
return event
def execute(self, counter) -> events.Event:
raise NotImplementedError
@property
def most_stale_time(self):
return min(self.latest_od_timestamp, self.latest_growth_rate_timestamp)
def set_led_intensity(self, channel, intensity):
self.edited_channels.append(channel)
led_intensity(channel,
intensity,
unit=self.unit,
experiment=self.experiment)
########## Private & internal methods
def on_disconnect(self):
self.latest_settings_ended_at = current_time()
self._send_details_to_mqtt()
try:
self.timer_thread.cancel()
except AttributeError:
pass
for job in self.sub_jobs:
job.set_state("disconnected")
for channel in self.edited_channels:
led_intensity(channel,
0,
unit=self.unit,
experiment=self.experiment)
self._clear_mqtt_cache()
def __setattr__(self, name, value) -> None:
super(LEDAutomation, self).__setattr__(name, value)
if name in self.editable_settings and name != "state":
self.latest_settings_ended_at = current_time()
self._send_details_to_mqtt()
self.latest_settings_started_at = current_time()
self.latest_settings_ended_at = None
def _set_growth_rate(self, message):
self.previous_growth_rate = self.latest_growth_rate
self.latest_growth_rate = float(message.payload)
self.latest_growth_rate_timestamp = time.time()
def _set_OD(self, message):
self.previous_od = self.latest_od
self.latest_od = float(message.payload)
self.latest_od_timestamp = time.time()
def _clear_mqtt_cache(self):
# From homie: Devices can remove old properties and nodes by publishing a zero-length payload on the respective topics.
for attr in self.editable_settings:
if attr == "state":
continue
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/{attr}",
None,
retain=True,
qos=QOS.EXACTLY_ONCE,
)
def _send_details_to_mqtt(self):
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/led_automation_settings",
json.dumps({
"pioreactor_unit":
self.unit,
"experiment":
self.experiment,
"started_at":
self.latest_settings_started_at,
"ended_at":
self.latest_settings_ended_at,
"automation":
self.__class__.__name__,
"settings":
json.dumps({
attr: getattr(self, attr, None)
for attr in self.editable_settings if attr != "state"
}),
}),
qos=QOS.EXACTLY_ONCE,
retain=True,
)
def start_passive_listeners(self):
self.subscribe_and_callback(
self._set_OD,
f"pioreactor/{self.unit}/{self.experiment}/od_filtered/{self.sensor}",
)
self.subscribe_and_callback(
self._set_growth_rate,
f"pioreactor/{self.unit}/{self.experiment}/growth_rate")
class AltMediaCalculator(BackgroundSubJob):
"""
Computes the fraction of the vial that is from the alt-media vs the regular media.
We periodically publish this, too, so the UI
graph looks better.
"""
def __init__(self, unit=None, experiment=None, **kwargs) -> None:
super(AltMediaCalculator, self).__init__(job_name=JOB_NAME,
unit=unit,
experiment=experiment)
self.latest_alt_media_fraction = self.get_initial_alt_media_fraction()
# publish often to fill in gaps in UI chart.
self.publish_periodically_thead = RepeatedTimer(
5 * 60,
self.publish_latest_alt_media_fraction,
job_name=self.job_name)
self.publish_periodically_thead.start()
self.start_passive_listeners()
def on_disconnect(self):
self.publish_periodically_thead.cancel()
def on_dosing_event(self, message):
payload = json.loads(message.payload)
volume, event = float(payload["volume_change"]), payload["event"]
if event == "add_media":
self.update_alt_media_fraction(volume, 0)
elif event == "add_alt_media":
self.update_alt_media_fraction(0, volume)
elif event == "remove_waste":
pass
else:
raise ValueError("Unknown event type")
def publish_latest_alt_media_fraction(self):
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{JOB_NAME}/alt_media_fraction",
self.latest_alt_media_fraction,
retain=True,
qos=QOS.EXACTLY_ONCE,
)
def update_alt_media_fraction(self, media_delta, alt_media_delta):
total_delta = media_delta + alt_media_delta
# current mL
alt_media_ml = VIAL_VOLUME * self.latest_alt_media_fraction
media_ml = VIAL_VOLUME * (1 - self.latest_alt_media_fraction)
# remove
alt_media_ml = alt_media_ml * (1 - total_delta / VIAL_VOLUME)
media_ml = media_ml * (1 - total_delta / VIAL_VOLUME)
# add (alt) media
alt_media_ml = alt_media_ml + alt_media_delta
media_ml = media_ml + media_delta
self.latest_alt_media_fraction = alt_media_ml / VIAL_VOLUME
self.publish_latest_alt_media_fraction()
return self.latest_alt_media_fraction
def get_initial_alt_media_fraction(self):
message = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/alt_media_fraction",
timeout=2,
)
if message:
return float(message.payload)
else:
return 0
def start_passive_listeners(self) -> None:
self.subscribe_and_callback(
self.on_dosing_event,
f"pioreactor/{self.unit}/{self.experiment}/dosing_events",
qos=QOS.EXACTLY_ONCE,
)
def __init__(
self,
channel_angle_map: dict[pt.PdChannel, pt.PdAngle],
interval: float,
adc_reader: ADCReader,
ir_led_reference_tracker: IrLedReferenceTracker,
unit: str,
experiment: str,
) -> None:
super(ODReader, self).__init__(
job_name="od_reading", unit=unit, experiment=experiment
)
self.adc_reader = adc_reader
self.channel_angle_map = channel_angle_map
self.interval = interval
self.ir_led_reference_tracker = ir_led_reference_tracker
self.first_od_obs_time: Optional[float] = None
self.ir_channel: pt.LedChannel = self.get_ir_channel_from_configuration()
self.ir_led_intensity: float = config.getfloat("od_config", "ir_intensity")
self.non_ir_led_channels: list[pt.LedChannel] = [
ch for ch in ALL_LED_CHANNELS if ch != self.ir_channel
]
if not hardware.is_HAT_present():
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
self.logger.debug(
f"Starting od_reading with PD channels {channel_angle_map}, with IR LED intensity {self.ir_led_intensity}% from channel {self.ir_channel}."
)
# setup the ADC and IrLedReference by turning off all LEDs.
with change_leds_intensities_temporarily(
ALL_LED_CHANNELS,
[0.0, 0.0, 0.0, 0.0],
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
):
with lock_leds_temporarily(self.non_ir_led_channels):
# start IR led before ADC starts, as it needs it.
self.start_ir_led()
self.adc_reader.setup_adc() # determine best gain, max-signal, etc.
self.stop_ir_led()
# get blank values of reference PD.
# This slightly improves the accuracy of the IR LED output tracker,
# See that class's docs.
self.ir_led_reference_tracker.set_blank(self.adc_reader.take_reading())
self.record_from_adc_timer = RepeatedTimer(
self.interval,
self.record_and_publish_from_adc,
run_immediately=True,
).start()
class ODReader(BackgroundJob):
"""
Produce a stream of OD readings from the sensors.
Parameters
-----------
channel_angle_map: dict
dict of (channel: angle) pairs, ex: {1: "135", 2: "90"}
interval: float
seconds between readings
adc_reader: ADCReader
ir_led_reference_tracker: IrLedReferenceTracker
Attributes
------------
adc_reader: ADCReader
latest_reading: dict
represents the most recent dict from the adc_reader
"""
published_settings = {
"first_od_obs_time": {"datatype": "float", "settable": False},
"led_intensity": {"datatype": "float", "settable": True, "unit": "%"},
"interval": {"datatype": "float", "settable": False, "unit": "s"},
}
latest_reading: dict[pt.PdChannel, float]
def __init__(
self,
channel_angle_map: dict[pt.PdChannel, pt.PdAngle],
interval: float,
adc_reader: ADCReader,
ir_led_reference_tracker: IrLedReferenceTracker,
unit: str,
experiment: str,
) -> None:
super(ODReader, self).__init__(
job_name="od_reading", unit=unit, experiment=experiment
)
self.adc_reader = adc_reader
self.channel_angle_map = channel_angle_map
self.interval = interval
self.ir_led_reference_tracker = ir_led_reference_tracker
self.first_od_obs_time: Optional[float] = None
self.ir_channel: pt.LedChannel = self.get_ir_channel_from_configuration()
self.ir_led_intensity: float = config.getfloat("od_config", "ir_intensity")
self.non_ir_led_channels: list[pt.LedChannel] = [
ch for ch in ALL_LED_CHANNELS if ch != self.ir_channel
]
if not hardware.is_HAT_present():
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
self.logger.debug(
f"Starting od_reading with PD channels {channel_angle_map}, with IR LED intensity {self.ir_led_intensity}% from channel {self.ir_channel}."
)
# setup the ADC and IrLedReference by turning off all LEDs.
with change_leds_intensities_temporarily(
ALL_LED_CHANNELS,
[0.0, 0.0, 0.0, 0.0],
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
):
with lock_leds_temporarily(self.non_ir_led_channels):
# start IR led before ADC starts, as it needs it.
self.start_ir_led()
self.adc_reader.setup_adc() # determine best gain, max-signal, etc.
self.stop_ir_led()
# get blank values of reference PD.
# This slightly improves the accuracy of the IR LED output tracker,
# See that class's docs.
self.ir_led_reference_tracker.set_blank(self.adc_reader.take_reading())
self.record_from_adc_timer = RepeatedTimer(
self.interval,
self.record_and_publish_from_adc,
run_immediately=True,
).start()
def get_ir_channel_from_configuration(self) -> pt.LedChannel:
try:
return cast(pt.LedChannel, config.get("leds_reverse", IR_keyword))
except Exception:
self.logger.error(
"""`leds` section must contain `IR` value. Ex:
[leds]
A=IR
"""
)
raise KeyError("`IR` value not found in section.")
def record_and_publish_from_adc(self) -> None:
if self.first_od_obs_time is None:
self.first_od_obs_time = time()
pre_duration = 0.01 # turn on LED prior to taking snapshot and wait
# we put a soft lock on the LED channels - it's up to the
# other jobs to make sure they check the locks.
with change_leds_intensities_temporarily(
ALL_LED_CHANNELS,
[0.0, 0.0, 0.0, 0.0],
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
):
with lock_leds_temporarily(self.non_ir_led_channels):
self.start_ir_led()
sleep(pre_duration)
timestamp_of_readings = current_utc_time()
batched_readings = self.adc_reader.take_reading()
self.latest_reading = batched_readings
self.ir_led_reference_tracker.update(batched_readings)
self.publish_single(batched_readings, timestamp_of_readings)
self.publish_batch(batched_readings, timestamp_of_readings)
def start_ir_led(self) -> None:
r = change_led_intensity(
channels=self.ir_channel,
intensities=self.ir_led_intensity,
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
)
if not r:
raise OSError("IR LED could not be started. Stopping OD reading.")
return
def stop_ir_led(self) -> None:
change_led_intensity(
channels=self.ir_channel,
intensities=0.0,
unit=self.unit,
experiment=self.experiment,
source_of_event=self.job_name,
pubsub_client=self.pub_client,
verbose=False,
)
def on_sleeping(self) -> None:
self.record_from_adc_timer.pause()
def on_sleeping_to_ready(self) -> None:
self.record_from_adc_timer.unpause()
def on_disconnected(self) -> None:
# turn off the LED after we have take our last ADC reading..
try:
self.record_from_adc_timer.cancel()
except Exception:
pass
def publish_batch(
self, batched_ads_readings: dict[pt.PdChannel, float], timestamp: str
) -> None:
if self.state != self.READY:
return
output = {
"od_raw": dict(),
"timestamp": timestamp,
}
for channel, angle in self.channel_angle_map.items():
output["od_raw"][channel] = { # type: ignore
"voltage": self.normalize_by_led_output(batched_ads_readings[channel]),
"angle": angle,
}
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/od_raw_batched",
output,
qos=QOS.EXACTLY_ONCE,
)
def publish_single(
self, batched_ads_readings: dict[pt.PdChannel, float], timestamp: str
) -> None:
if self.state != self.READY:
return
for channel, angle in self.channel_angle_map.items():
payload = {
"voltage": self.normalize_by_led_output(batched_ads_readings[channel]),
"angle": angle,
"timestamp": timestamp,
}
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/od_raw/{channel}",
payload,
qos=QOS.EXACTLY_ONCE,
)
def normalize_by_led_output(self, od_signal: float) -> float:
return self.ir_led_reference_tracker(od_signal)
class Stirrer(BackgroundJob):
"""
Parameters
------------
target_rpm: float
Send message to "pioreactor/{unit}/{experiment}/stirring/target_rpm/set" to change the stirring speed.
rpm_calculator: RpmCalculator
See RpmCalculator and examples below.
Notes
-------
The create a feedback loop between the duty-cycle level and the RPM, we set up a polling algorithm. We set up
an edge detector on the hall sensor pin, and count the number of pulses in N seconds. We convert this count to RPM, and
then use a PID system to update the amount of duty cycle to apply.
We perform the above every N seconds. That is, there is PID controller that checks every N seconds and nudges the duty cycle
to match the requested RPM.
Examples
---------
> st = Stirrer(500, unit, experiment)
> st.start_stirring()
"""
published_settings = {
"target_rpm": {
"datatype": "json",
"settable": True,
"unit": "RPM"
},
"measured_rpm": {
"datatype": "json",
"settable": False,
"unit": "RPM"
},
"duty_cycle": {
"datatype": "float",
"settable": True,
"unit": "%"
},
}
_previous_duty_cycle: float = 0
duty_cycle: float = config.getint(
"stirring", "initial_duty_cycle",
fallback=60.0) # only used if calibration isn't defined.
_measured_rpm: Optional[float] = None
def __init__(
self,
target_rpm: float,
unit: str,
experiment: str,
rpm_calculator: Optional[RpmCalculator],
hertz: float = 150,
) -> None:
super(Stirrer, self).__init__(job_name="stirring",
unit=unit,
experiment=experiment)
self.logger.debug(f"Starting stirring with initial {target_rpm} RPM.")
self.rpm_calculator = rpm_calculator
if not hardware.is_HAT_present():
self.logger.error("Pioreactor HAT must be present.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
if (self.rpm_calculator
is not None) and not hardware.is_heating_pcb_present():
self.logger.error("Heating PCB must be present to measure RPM.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError(
"Heating PCB must be present to measure RPM.")
pin = hardware.PWM_TO_PIN[config.get("PWM_reverse", "stirring")]
self.pwm = PWM(pin, hertz)
self.pwm.lock()
self.rpm_to_dc_lookup = self.initialize_rpm_to_dc_lookup()
self.target_rpm = target_rpm
self.duty_cycle = self.rpm_to_dc_lookup(self.target_rpm)
# set up PID
self.pid = PID(
Kp=config.getfloat("stirring.pid", "Kp"),
Ki=config.getfloat("stirring.pid", "Ki"),
Kd=config.getfloat("stirring.pid", "Kd"),
setpoint=self.target_rpm,
unit=self.unit,
experiment=self.experiment,
job_name=self.job_name,
target_name="rpm",
output_limits=(-20, 20), # avoid whiplashing
)
# set up thread to periodically check the rpm
self.rpm_check_repeated_thread = RepeatedTimer(
17, # 17 and 5 are coprime
self.poll_and_update_dc,
job_name=self.job_name,
run_immediately=True,
run_after=5,
poll_for_seconds=
4, # technically should be a function of the RPM: lower RPM, longer to get sufficient data.
)
def initialize_rpm_to_dc_lookup(self) -> Callable:
if self.rpm_calculator is None:
# if we can't track RPM, no point in adjusting DC
return lambda rpm: self.duty_cycle
with local_persistant_storage("stirring_calibration") as cache:
if "linear_v1" in cache:
parameters = json.loads(cache["linear_v1"])
coef = parameters["rpm_coef"]
intercept = parameters["intercept"]
# we scale this by 90% to make sure the PID + prediction doesn't overshoot,
# better to be conservative here.
# equivalent to a weighted average: 0.1 * current + 0.9 * predicted
return lambda rpm: self.duty_cycle - 0.90 * (
self.duty_cycle - (coef * rpm + intercept))
else:
return lambda rpm: self.duty_cycle
def on_disconnected(self) -> None:
with suppress(AttributeError):
self.rpm_check_repeated_thread.cancel()
with suppress(AttributeError):
self.stop_stirring()
self.pwm.cleanup()
with suppress(AttributeError):
if self.rpm_calculator:
self.rpm_calculator.cleanup()
def start_stirring(self) -> None:
self.pwm.start(100) # get momentum to start
sleep(0.25)
self.set_duty_cycle(self.duty_cycle)
sleep(0.75)
self.rpm_check_repeated_thread.start() # .start is idempotent
def poll(self, poll_for_seconds: float) -> Optional[float]:
"""
Returns an RPM, or None if not measuring RPM.
"""
if self.rpm_calculator is None:
return None
recent_rpm = self.rpm_calculator(poll_for_seconds)
if recent_rpm == 0:
# TODO: attempt to restart stirring
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/monitor/flicker_led_with_error_code",
error_codes.STIRRING_FAILED_ERROR_CODE,
)
self.logger.warning("Stirring RPM is 0 - has it failed?")
if self._measured_rpm is not None:
# use a simple EMA, alpha chosen arbitrarily, but should be a function of delta time.
self._measured_rpm = 0.025 * self._measured_rpm + 0.975 * recent_rpm
else:
self._measured_rpm = recent_rpm
self.measured_rpm = {
"timestamp": current_utc_time(),
"rpm": self._measured_rpm
}
return self._measured_rpm
def poll_and_update_dc(self, poll_for_seconds: float) -> None:
self.poll(poll_for_seconds)
if self._measured_rpm is None:
return
result = self.pid.update(self._measured_rpm, dt=1)
self.set_duty_cycle(self.duty_cycle + result)
def stop_stirring(self) -> None:
# if the user unpauses, we want to go back to their previous value, and not the default.
self.set_duty_cycle(0)
def on_ready_to_sleeping(self) -> None:
self.rpm_check_repeated_thread.pause()
self.stop_stirring()
def on_sleeping_to_ready(self) -> None:
self.duty_cycle = self._previous_duty_cycle
self.rpm_check_repeated_thread.unpause()
self.start_stirring()
def set_duty_cycle(self, value: float) -> None:
self._previous_duty_cycle = self.duty_cycle
self.duty_cycle = clamp(0, round(float(value), 5), 100)
self.pwm.change_duty_cycle(self.duty_cycle)
def set_target_rpm(self, value: float) -> None:
self.target_rpm = float(value)
self.set_duty_cycle(self.rpm_to_dc_lookup(self.target_rpm))
self.pid.set_setpoint(self.target_rpm)
def block_until_rpm_is_close_to_target(self,
abs_tolerance: float = 15) -> None:
"""
This function blocks until the stirring is "close enough" to the target RPM.
"""
if self.rpm_calculator is None:
# can't block if we aren't recording the RPM
return
while (self._measured_rpm is not None
) and abs(self._measured_rpm - self.target_rpm) > abs_tolerance:
sleep(0.25)
class LEDAutomation(BackgroundSubJob):
"""
This is the super class that LED automations inherit from. The `run` function will
execute every `duration` minutes (selected at the start of the program), and call the `execute` function
which is what subclasses define.
To change setting over MQTT:
`pioreactor/<unit>/<experiment>/led_automation/<setting>/set` value
"""
automation_name = "led_automation_base" # is overwritten in subclasses
published_settings = {"duration": {"datatype": "float", "settable": True}}
_latest_growth_rate: Optional[float] = None
_latest_od: Optional[float] = None
previous_od: Optional[float] = None
previous_growth_rate: Optional[float] = None
_latest_settings_started_at: str = current_utc_time()
_latest_settings_ended_at: Optional[str] = None
_latest_run_at: Optional[float] = None
latest_event: Optional[events.Event] = None
run_thread: RepeatedTimer | Thread
# next two are seconds-since-unix-epoch
latest_od_at: float = 0
latest_growth_rate_at: float = 0
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# this registers all subclasses of LEDAutomation back to LEDController, so the subclass
# can be invoked in LEDController.
if hasattr(cls, "automation_name"):
LEDController.automations[cls.automation_name] = cls
def __init__(
self,
duration: float,
skip_first_run: bool = False,
unit: str = None,
experiment: str = None,
**kwargs,
) -> None:
super(LEDAutomation, self).__init__(
job_name="led_automation", unit=unit, experiment=experiment
)
self.skip_first_run = skip_first_run
self.edited_channels: set[LedChannel] = set()
self.set_duration(duration)
self.start_passive_listeners()
self.logger.info(f"Starting {self.automation_name} LED automation.")
def set_duration(self, duration: float) -> None:
self.duration = float(duration)
if self._latest_run_at:
# what's the correct logic when changing from duration N and duration M?
# - N=20, and it's been 5m since the last run (or initialization). I change to M=30, I should wait M-5 minutes.
# - N=60, and it's been 50m since last run. I change to M=30, I should run immediately.
run_after = max(0, (self.duration * 60) - (time.time() - self._latest_run_at))
else:
# there is a race condition here: self.run() will run immediately (see run_immediately), but the state of the job is not READY, since
# set_duration is run in the __init__ (hence the job is INIT). So we wait 2 seconds for the __init__ to finish, and then run.
run_after = 2
self.run_thread = RepeatedTimer(
self.duration * 60, # RepeatedTimer uses seconds
self.run,
job_name=self.job_name,
run_immediately=(not self.skip_first_run)
or (self._latest_run_at is not None),
run_after=run_after,
).start()
def run(self) -> Optional[events.Event]:
# TODO: this should be close to or equal to the function in DosingAutomation
event: Optional[events.Event]
if self.state == self.DISCONNECTED:
# NOOP
# we ended early.
return None
elif self.state != self.READY:
# wait a minute, and if not unpaused, just move on.
time_waited = 0
sleep_for = 5
while self.state != self.READY:
time.sleep(sleep_for)
time_waited += sleep_for
if time_waited > 60:
return None
else:
return self.run()
else:
try:
event = self.execute()
except Exception as e:
self.logger.debug(e, exc_info=True)
self.logger.error(e)
event = events.ErrorOccurred()
if event:
self.logger.info(str(event))
self.latest_event = event
self._latest_run_at = time.time()
return event
def execute(self) -> Optional[events.Event]:
pass
@property
def most_stale_time(self) -> float:
return min(self.latest_od_at, self.latest_growth_rate_at)
def set_led_intensity(self, channel: LedChannel, intensity: float) -> bool:
"""
This first checks the lock on the LED channel, and will wait a few seconds for it to clear,
and error out if it waits too long.
Parameters
------------
Channel:
The LED channel to modify.
Intensity: float
A float between 0-100, inclusive.
"""
for _ in range(12):
success = led_intensity(
channel,
intensity,
unit=self.unit,
experiment=self.experiment,
pubsub_client=self.pub_client,
source_of_event=self.job_name,
)
if success:
self.edited_channels.add(channel)
return True
time.sleep(0.1)
self.logger.warning(
f"Unable to update channel {channel} due to a long lock being on the channel."
)
return False
########## Private & internal methods
def on_disconnected(self) -> None:
self._latest_settings_ended_at = current_utc_time()
self._send_details_to_mqtt()
with suppress(AttributeError):
self.run_thread.join()
for channel in self.edited_channels:
led_intensity(channel, 0, unit=self.unit, experiment=self.experiment)
@property
def latest_growth_rate(self) -> float:
# check if None
if self._latest_growth_rate is None:
# this should really only happen on the initialization.
self.logger.debug("Waiting for OD and growth rate data to arrive")
if not is_pio_job_running("od_reading", "growth_rate_calculating"):
raise exc.JobRequiredError(
"`od_reading` and `growth_rate_calculating` should be running."
)
# check most stale time
if (time.time() - self.most_stale_time) > 5 * 60:
raise exc.JobRequiredError(
"readings are too stale (over 5 minutes old) - are `od_reading` and `growth_rate_calculating` running?"
)
return cast(float, self._latest_growth_rate)
@property
def latest_od(self) -> float:
# check if None
if self._latest_od is None:
# this should really only happen on the initialization.
self.logger.debug("Waiting for OD and growth rate data to arrive")
if not is_pio_job_running("od_reading", "growth_rate_calculating"):
raise exc.JobRequiredError(
"`od_reading` and `growth_rate_calculating` should be running."
)
# check most stale time
if (time.time() - self.most_stale_time) > 5 * 60:
raise exc.JobRequiredError(
"readings are too stale (over 5 minutes old) - are `od_reading` and `growth_rate_calculating` running?"
)
return cast(float, self._latest_od)
def __setattr__(self, name, value) -> None:
super(LEDAutomation, self).__setattr__(name, value)
if name in self.published_settings and name != "state":
self._latest_settings_ended_at = current_utc_time()
self._send_details_to_mqtt()
self._latest_settings_started_at = current_utc_time()
self._latest_settings_ended_at = None
def _set_growth_rate(self, message) -> None:
self.previous_growth_rate = self._latest_growth_rate
self._latest_growth_rate = float(json.loads(message.payload)["growth_rate"])
self.latest_growth_rate_at = time.time()
def _set_OD(self, message) -> None:
self.previous_od = self._latest_od
self._latest_od = float(json.loads(message.payload)["od_filtered"])
self.latest_od_at = time.time()
def _send_details_to_mqtt(self) -> None:
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/led_automation_settings",
json.dumps(
{
"pioreactor_unit": self.unit,
"experiment": self.experiment,
"started_at": self._latest_settings_started_at,
"ended_at": self._latest_settings_ended_at,
"automation": self.automation_name,
"settings": json.dumps(
{
attr: getattr(self, attr, None)
for attr in self.published_settings
if attr != "state"
}
),
}
),
qos=QOS.EXACTLY_ONCE,
)
def start_passive_listeners(self) -> None:
self.subscribe_and_callback(
self._set_OD,
f"pioreactor/{self.unit}/{self.experiment}/growth_rate_calculating/od_filtered",
)
self.subscribe_and_callback(
self._set_growth_rate,
f"pioreactor/{self.unit}/{self.experiment}/growth_rate_calculating/growth_rate",
)
class TemperatureController(BackgroundJob):
"""
This job publishes to
pioreactor/<unit>/<experiment>/temperature_control/temperature
the following:
{
"temperature": <float>,
"timestamp": <ISO 8601 timestamp>
}
If you have your own thermo-couple, you can publish to this topic, with the same schema
and all should just work™️. You'll need to provide your own feedback loops however.
Parameters
------------
eval_and_publish_immediately: bool, default True
evaluate and publish the temperature once the class is created (in the background)
TODO: do I need this still?
"""
MAX_TEMP_TO_REDUCE_HEATING = 60.0 # ~PLA glass transition temp
MAX_TEMP_TO_DISABLE_HEATING = 62.0
MAX_TEMP_TO_SHUTDOWN = 64.0
automations = {} # type: ignore
published_settings = {
"automation": {
"datatype": "json",
"settable": True
},
"automation_name": {
"datatype": "string",
"settable": False
},
"temperature": {
"datatype": "json",
"settable": False,
"unit": "℃"
},
"heater_duty_cycle": {
"datatype": "float",
"settable": False,
"unit": "%"
},
}
temperature: Optional[dict[str, Any]] = None
def __init__(
self,
automation_name: str,
unit: str,
experiment: str,
eval_and_publish_immediately: bool = True,
**kwargs,
) -> None:
super().__init__(job_name="temperature_control",
unit=unit,
experiment=experiment)
if not is_HAT_present():
self.logger.error("Pioreactor HAT must be present.")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError("Pioreactor HAT must be present.")
if not is_heating_pcb_present():
self.logger.error("Heating PCB must be attached to Pioreactor HAT")
self.set_state(self.DISCONNECTED)
raise exc.HardwareNotFoundError(
"Heating PCB must be attached to Pioreactor HAT")
if is_testing_env():
self.logger.debug("TMP1075 not available; using MockTMP1075")
from pioreactor.utils.mock import MockTMP1075 as TMP1075
else:
from TMP1075 import TMP1075 # type: ignore
self.pwm = self.setup_pwm()
self.update_heater(0)
self.tmp_driver = TMP1075()
self.read_external_temperature_timer = RepeatedTimer(
45, self.read_external_temperature, run_immediately=False)
self.read_external_temperature_timer.start()
self.publish_temperature_timer = RepeatedTimer(
4 * 60,
self.evaluate_and_publish_temperature,
run_immediately=eval_and_publish_immediately,
run_after=60,
)
self.publish_temperature_timer.start()
self.automation = AutomationDict(automation_name=automation_name,
**kwargs)
try:
automation_class = self.automations[
self.automation["automation_name"]]
except KeyError:
raise KeyError(
f"Unable to find automation {self.automation['automation_name']}. Available automations are {list(self.automations.keys())}"
)
self.logger.info(f"Starting {self.automation}.")
try:
self.automation_job = automation_class(unit=self.unit,
experiment=self.experiment,
parent=self,
**kwargs)
except Exception as e:
self.logger.error(e)
self.logger.debug(e, exc_info=True)
self.set_state(self.DISCONNECTED)
raise e
self.automation_name = self.automation["automation_name"]
self.temperature = {
"temperature": self.read_external_temperature(),
"timestamp": current_utc_time(),
}
def turn_off_heater(self) -> None:
self._update_heater(0)
self.pwm.stop()
self.pwm.cleanup()
# we re-instantiate it as some other process may have messed with the channel.
self.pwm = self.setup_pwm()
self._update_heater(0)
self.pwm.stop()
def update_heater(self, new_duty_cycle: float) -> bool:
"""
Update heater's duty cycle. This function checks for the PWM lock, and will not
update if the PWM is locked.
Returns true if the update was made (eg: no lock), else returns false
"""
if not self.pwm.is_locked():
self._update_heater(new_duty_cycle)
return True
else:
return False
def update_heater_with_delta(self, delta_duty_cycle: float) -> bool:
"""
Update heater's duty cycle by `delta_duty_cycle` amount. This function checks for the PWM lock, and will not
update if the PWM is locked.
Returns true if the update was made (eg: no lock), else returns false
"""
return self.update_heater(self.heater_duty_cycle + delta_duty_cycle)
def read_external_temperature(self) -> float:
"""
Read the current temperature from our sensor, in Celsius
"""
try:
# check temp is fast, let's do it twice to reduce variance.
pcb_temp = 0.5 * (self.tmp_driver.get_temperature() +
self.tmp_driver.get_temperature())
except OSError:
# could not find temp driver on i2c
self.logger.error(
"Is the Heating PCB attached to the Pioreactor HAT? Unable to find I²C for temperature driver."
)
raise exc.HardwareNotFoundError(
"Is the Heating PCB attached to the Pioreactor HAT? Unable to find I²C for temperature driver."
)
self._check_if_exceeds_max_temp(pcb_temp)
return pcb_temp
##### internal and private methods ########
def set_automation(self, new_temperature_automation_json) -> None:
# TODO: this needs a better rollback. Ex: in except, something like
# self.automation_job.set_state("init")
# self.automation_job.set_state("ready")
# OR should just bail...
algo_metadata = AutomationDict(
**loads(new_temperature_automation_json))
try:
self.automation_job.set_state("disconnected")
except AttributeError:
# sometimes the user will change the job too fast before the dosing job is created, let's protect against that.
sleep(1)
self.set_automation(new_temperature_automation_json)
# reset heater back to 0.
self._update_heater(0)
try:
self.logger.info(f"Starting {algo_metadata}.")
self.automation_job = self.automations[
algo_metadata["automation_name"]](unit=self.unit,
experiment=self.experiment,
parent=self,
**algo_metadata)
self.automation = algo_metadata
self.automation_name = algo_metadata["automation_name"]
except KeyError:
self.logger.debug(
f"Unable to find automation {algo_metadata['automation_name']}. Available automations are {list(self.automations.keys())}",
exc_info=True,
)
self.logger.warning(
f"Unable to find automation {algo_metadata['automation_name']}. Available automations are {list(self.automations.keys())}"
)
except Exception as e:
self.logger.debug(f"Change failed because of {str(e)}",
exc_info=True)
self.logger.warning(f"Change failed because of {str(e)}")
def _update_heater(self, new_duty_cycle: float) -> None:
self.heater_duty_cycle = round(float(new_duty_cycle), 5)
self.pwm.change_duty_cycle(self.heater_duty_cycle)
def _check_if_exceeds_max_temp(self, temp: float) -> None:
if temp > self.MAX_TEMP_TO_SHUTDOWN:
self.logger.error(
f"Temperature of heating surface has exceeded {self.MAX_TEMP_TO_SHUTDOWN}℃ - currently {temp} ℃. This is beyond our recommendations. Shutting down Raspberry Pi to prevent further problems. Take caution when touching the heating surface and wetware."
)
from subprocess import call
call("sudo shutdown --poweroff", shell=True)
elif temp > self.MAX_TEMP_TO_DISABLE_HEATING:
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/monitor/flicker_led_with_error_code",
error_codes.PCB_TEMPERATURE_TOO_HIGH,
)
self.logger.warning(
f"Temperature of heating surface has exceeded {self.MAX_TEMP_TO_DISABLE_HEATING}℃ - currently {temp} ℃. This is beyond our recommendations. The heating PWM channel will be forced to 0. Take caution when touching the heating surface and wetware."
)
self._update_heater(0)
elif temp > self.MAX_TEMP_TO_REDUCE_HEATING:
self.publish(
f"pioreactor/{self.unit}/{self.experiment}/monitor/flicker_led_with_error_code",
error_codes.PCB_TEMPERATURE_TOO_HIGH,
)
self.logger.debug(
f"Temperature of heating surface has exceeded {self.MAX_TEMP_TO_REDUCE_HEATING}℃ - currently {temp} ℃. This is close to our maximum recommended value. The heating PWM channel will be reduced to 90% its current value. Take caution when touching the heating surface and wetware."
)
self._update_heater(self.heater_duty_cycle * 0.9)
def on_sleeping(self) -> None:
self.automation_job.set_state(self.SLEEPING)
def on_sleeping_to_ready(self) -> None:
self.automation_job.set_state(self.READY)
def on_disconnected(self) -> None:
try:
self.automation_job.set_state(self.DISCONNECTED)
except AttributeError:
# if disconnect is called right after starting, temperature_automation_job isn't instantiated
pass
try:
self.read_external_temperature_timer.cancel()
self.publish_temperature_timer.cancel()
except AttributeError:
pass
try:
self._update_heater(0)
self.pwm.stop()
self.pwm.cleanup()
except AttributeError:
pass
def setup_pwm(self) -> PWM:
hertz = 1
pin = PWM_TO_PIN[HEATER_PWM_TO_PIN]
pin = PWM_TO_PIN[config.get("PWM_reverse", "heating")]
pwm = PWM(pin, hertz)
pwm.start(0)
return pwm
def evaluate_and_publish_temperature(self) -> None:
"""
1. lock PWM and turn off heater
2. start recording temperatures from the sensor
3. After collected M samples, pass to a model to approx temp
4. assign temp to publish to ../temperature
5. return heater to previous DC value and unlock heater
"""
assert not self.pwm.is_locked(
), "PWM is locked - it shouldn't be though!"
with self.pwm.lock_temporarily():
previous_heater_dc = self.heater_duty_cycle
self._update_heater(0)
# we pause heating for (N_sample_points * time_between_samples) seconds
N_sample_points = 30
time_between_samples = 5
features = {}
features["prev_temp"] = (self.temperature["temperature"]
if self.temperature else None)
features["previous_heater_dc"] = previous_heater_dc
time_series_of_temp = []
for i in range(N_sample_points):
time_series_of_temp.append(self.read_external_temperature())
sleep(time_between_samples)
if self.state != self.READY:
# if our state changes in this loop, exit.
return
features["time_series_of_temp"] = time_series_of_temp
self.logger.debug(features)
# update heater first, before publishing the temperature. Why? A downstream process
# might listen for the updating temperature, and update the heater (pid_stable),
# and if we update here too late, we may overwrite their changes.
# We also want to remove the lock first, so close this context early.
self._update_heater(previous_heater_dc)
try:
approximated_temperature = self.approximate_temperature(features)
except Exception as e:
self.logger.debug(e, exc_info=True)
self.logger.error(e)
self.temperature = {
"temperature": approximated_temperature,
"timestamp": current_utc_time(),
}
def approximate_temperature(self, features: dict[str, Any]) -> float:
"""
models
temp = b * exp(p * t) + c * exp(q * t) + ROOM_TEMP
Reference
-------------
https://www.scribd.com/doc/14674814/Regressions-et-equations-integrales
page 71 - 72
It's possible that we can determine if the vial is in using the heat loss coefficient. Quick look:
when the vial is in, heat coefficient is ~ -0.008, when not in, coefficient is ~ -0.028.
"""
if features["previous_heater_dc"] == 0:
return features["time_series_of_temp"][-1]
import numpy as np
from numpy import exp
ROOM_TEMP = 10.0 # ??
times_series = features["time_series_of_temp"]
n = len(times_series)
y = np.array(times_series) - ROOM_TEMP
x = np.arange(n) # scaled by factor of 1/10 seconds
S = np.zeros(n)
SS = np.zeros(n)
for i in range(1, n):
S[i] = S[i - 1] + 0.5 * (y[i - 1] + y[i]) * (x[i] - x[i - 1])
SS[i] = SS[i - 1] + 0.5 * (S[i - 1] + S[i]) * (x[i] - x[i - 1])
# first regression
M1 = np.array([
[(SS**2).sum(), (SS * S).sum(), (SS * x).sum(), (SS).sum()],
[(SS * S).sum(), (S**2).sum(), (S * x).sum(), (S).sum()],
[(SS * x).sum(), (S * x).sum(), (x**2).sum(), (x).sum()],
[(SS).sum(), (S).sum(), (x).sum(), n],
])
Y1 = np.array([(y * SS).sum(), (y * S).sum(), (y * x).sum(), y.sum()])
try:
A, B, _, _ = np.linalg.solve(M1, Y1)
except np.linalg.LinAlgError:
self.logger.error("Error in first regression.")
self.logger.debug(f"x={x}")
self.logger.debug(f"y={y}")
return features["prev_temp"]
if (B**2 + 4 * A) < 0:
# something when wrong in the data collection - the data doesn't look enough like a sum of two expos
self.logger.error(f"Error in regression: {(B ** 2 + 4 * A)=} < 0")
self.logger.debug(f"x={x}")
self.logger.debug(f"y={y}")
return features["prev_temp"]
p = 0.5 * (B + np.sqrt(B**2 + 4 * A))
q = 0.5 * (B - np.sqrt(B**2 + 4 * A))
# second regression
M2 = np.array([
[exp(2 * p * x).sum(),
exp((p + q) * x).sum()],
[exp((q + p) * x).sum(),
exp(2 * q * x).sum()],
])
Y2 = np.array([(y * exp(p * x)).sum(), (y * exp(q * x)).sum()])
try:
b, c = np.linalg.solve(M2, Y2)
except np.linalg.LinAlgError:
self.logger.error("Error in second regression")
self.logger.debug(f"x={x}")
self.logger.debug(f"y={y}")
return features["prev_temp"]
if abs(p) < abs(q):
# since the regression can have identifiable problems, we use
# our domain knowledge to choose the pair that has the lower heat transfer coefficient.
alpha, beta = b, p
else:
alpha, beta = c, q
self.logger.debug(f"{b=}, {c=}, {p=} , {q=}")
temp_at_start_of_obs = ROOM_TEMP + alpha * exp(beta * 0)
temp_at_end_of_obs = ROOM_TEMP + alpha * exp(beta * n)
# the recent estimate weighted because I trust the predicted temperature at the start of observation more
# than the predicted temperature at the end.
return 2 / 3 * temp_at_start_of_obs + 1 / 3 * temp_at_end_of_obs