def test_od_blank_being_non_zero(self) -> None:
unit = get_unit_name()
experiment = get_latest_experiment_name()
with local_persistant_storage("od_blank") as cache:
cache[experiment] = json.dumps({"1": 0.25, "2": 0.4})
with local_persistant_storage("od_normalization_mean") as cache:
cache[experiment] = json.dumps({"1": 0.5, "2": 0.8})
with local_persistant_storage("od_normalization_variance") as cache:
cache[experiment] = json.dumps({"1": 1e-6, "2": 1e-4})
calc = GrowthRateCalculator(unit=unit, experiment=experiment)
pause()
pause()
publish(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
create_od_raw_batched_json(
["1", "2"], [0.50, 0.80], ["90", "135"], timestamp="2010-01-01 12:02:00"
),
retain=True,
)
pause()
pause()
assert calc.od_normalization_factors == {"2": 0.8, "1": 0.5}
assert calc.od_blank == {"2": 0.4, "1": 0.25}
results = calc.scale_raw_observations({"2": 1.0, "1": 0.6})
print(results)
assert abs(results["2"] - 1.5) < 0.00001
assert abs(results["1"] - 1.4) < 0.00001
calc.set_state(calc.DISCONNECTED)
def test_single_observation(self) -> None:
unit = get_unit_name()
experiment = get_latest_experiment_name()
with local_persistant_storage("od_normalization_mean") as cache:
cache[experiment] = json.dumps({1: 1})
with local_persistant_storage("od_normalization_variance") as cache:
cache[experiment] = json.dumps({1: 1})
publish(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
create_od_raw_batched_json(
["1"], [1.153], ["90"], timestamp="2010-01-01 12:00:30"
),
retain=True,
)
calc = GrowthRateCalculator(unit=unit, experiment=experiment)
publish(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
create_od_raw_batched_json(
["1"], [1.155], ["90"], timestamp="2010-01-01 12:00:35"
),
)
pause()
assert True
calc.set_state(calc.DISCONNECTED)
def test_changing_algo_over_mqtt_with_wrong_type_is_okay() -> None:
with local_persistant_storage("media_throughput") as c:
c[experiment] = "0.0"
algo = DosingController(
"pid_turbidostat",
volume=1.0,
target_od=0.4,
duration=2 / 60,
unit=unit,
experiment=experiment,
)
assert algo.automation["automation_name"] == "pid_turbidostat"
assert algo.automation_name == "pid_turbidostat"
pause()
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_control/automation/set",
'{"automation_name": "pid_turbidostat", "duration": "60", "target_od": "1.0", "volume": "1.0"}',
)
time.sleep(
7
) # need to wait for all jobs to disconnect correctly and threads to join.
assert isinstance(algo.automation_job, PIDTurbidostat)
assert algo.automation_job.target_od == 1.0
algo.set_state(algo.DISCONNECTED)
def test_single_observation():
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/median",
'{"135/0": 1}',
retain=True,
)
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/variance",
'{"135/0": 1}',
retain=True,
)
publish(f"pioreactor/{unit}/{experiment}/growth_rate", None, retain=True)
publish(
f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"135/0": 0.20944389172032837}',
retain=True,
)
GrowthRateCalculator(unit=unit, experiment=experiment)
publish(f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"135/0": 0.20944389172032837}')
pause()
assert True
def test_pump_will_disconnect_via_mqtt() -> None:
class ThreadWithReturnValue(threading.Thread):
def __init__(self, *init_args, **init_kwargs):
threading.Thread.__init__(self, *init_args, **init_kwargs)
self._return = None
def run(self):
self._return = self._target(*self._args, **self._kwargs)
def join(self):
threading.Thread.join(self)
return self._return
expected_ml = 20
t = ThreadWithReturnValue(target=add_media,
args=(unit, exp, expected_ml),
daemon=True)
t.start()
pause()
pause()
publish(f"pioreactor/{unit}/{exp}/add_media/$state/set", "disconnected")
pause()
pause()
pause()
resulting_ml = t.join()
assert resulting_ml < expected_ml
def test_skip_180():
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/median",
'{"135/0": 1, "180/2": 1, "90/1": 1}',
retain=True,
)
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/variance",
'{"135/0": 1, "180/2": 1, "90/1": 1}',
retain=True,
)
publish(f"pioreactor/{unit}/{experiment}/growth_rate", None, retain=True)
publish(
f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"180/2": 0.778586260567034, "135/0": 0.20944389172032837, "90/1": 0.1}',
retain=True,
)
calc = GrowthRateCalculator(unit=unit, experiment=experiment)
publish(
f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"180/2": 0.778586260567034, "135/0": 0.20944389172032837, "90/1": 0.1}',
)
pause()
assert "180/2" not in calc.angles
def test_mis_shapen_data(monkeypatch):
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/median",
'{"135/0": 1, "90/0": 1}',
retain=True,
)
publish(
f"pioreactor/{unit}/{experiment}/od_normalization/variance",
'{"135/0": 1, "90/0": 1}',
retain=True,
)
publish(
f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"135/0": 0.778586260567034, "90/0": 0.1}',
retain=True,
)
GrowthRateCalculator(unit=unit, experiment=experiment)
publish(
f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"135/0": 0.778586260567034, "90/0": 0.1}',
)
pause()
publish(f"pioreactor/{unit}/{experiment}/od_raw_batched",
'{"135/0": 0.808586260567034}')
pause()
def test_changing_algo_over_mqtt_with_wrong_type_is_okay():
pubsub.publish(
f"pioreactor/{unit}/{experiment}/throughput_calculating/media_throughput",
None,
retain=True,
)
algo = DosingController(
"pid_turbidostat",
volume=1.0,
target_od=0.4,
duration=2 / 60,
unit=unit,
experiment=experiment,
)
assert algo.dosing_automation == "pid_turbidostat"
pause()
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_control/dosing_automation/set",
'{"dosing_automation": "pid_turbidostat", "duration": "60", "target_od": "1.0", "volume": "1.0"}',
)
time.sleep(
7
) # need to wait for all jobs to disconnect correctly and threads to join.
assert isinstance(algo.dosing_automation_job, PIDTurbidostat)
assert algo.dosing_automation_job.target_od == 1.0
def test_drops_retained_messages():
publish(f"pioreactor/{unit}/exp1/growth_rate", None, retain=True)
publish(f"pioreactor/{unit}/exp2/growth_rate", None, retain=True)
def unit_from_topic(topic):
return topic.split("/")[1]
publish(f"pioreactor/{unit}/exp2/growth_rate", 0.9, retain=True)
# NOTE: see the +, this is what is used in production
ts = TimeSeriesAggregation(
"pioreactor/+/+/growth_rate",
output_dir="./",
experiment=experiment,
unit=leader,
ignore_cache=True,
extract_label=unit_from_topic,
record_every_n_seconds=0.1,
)
pause()
pause()
pause()
publish(f"pioreactor/{unit}/exp2/growth_rate", 1.0, retain=True)
pause()
publish(f"pioreactor/{unit}/exp2/growth_rate", 1.1, retain=True)
pause()
assert [_["y"] for _ in ts.aggregated_time_series["data"][0]] == [1.0, 1.1]
def test_subscribe_and_listen_to_clear_different_formatter():
def single_sensor_label_from_topic(topic):
split_topic = topic.split("/")
return f"{split_topic[1]}-{split_topic[-1]}"
ts = TimeSeriesAggregation(
f"pioreactor/+/{experiment}/od_raw/135/+",
output_dir="./",
experiment=experiment,
unit=leader,
ignore_cache=True,
record_every_n_seconds=0.1,
extract_label=single_sensor_label_from_topic,
)
publish(f"pioreactor/{unit}1/{experiment}/od_raw/135/0", 1.0)
publish(f"pioreactor/{unit}1/{experiment}/od_raw/135/0", 1.1)
publish(f"pioreactor/{unit}1/{experiment}/od_raw/135/1", 1.0)
publish(f"pioreactor/{unit}2/{experiment}/od_raw/135/0", 1.0)
pause()
assert ts.aggregated_time_series["series"] == [
"testing_unit1-A",
"testing_unit1-B",
"testing_unit2-A",
]
publish(
f"pioreactor/{leader}/{experiment}/time_series_aggregating/aggregated_time_series/set",
None,
)
pause()
assert ts.aggregated_time_series["series"] == []
def test_duty_cycle_is_published_and_not_settable() -> None:
dc_msgs = []
def collect(msg) -> None:
dc_msgs.append(msg.payload)
pubsub.subscribe_and_callback(
collect,
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/temperature_control/heater_duty_cycle",
)
with temperature_control.TemperatureController("silent",
unit=unit,
experiment=experiment):
# change to PID stable
pubsub.publish(
f"pioreactor/{unit}/{experiment}/temperature_control/automation/set",
'{"automation_name": "stable", "target_temperature": 35}',
)
pause(3)
# should produce an "Unable to set heater_duty_cycle"
pubsub.publish(
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/temperature_control/heater_duty_cycle/set",
10,
)
pause(1)
assert len(dc_msgs) > 0
def test_stirring_with_lookup_linear_v1() -> None:
class FakeRpmCalculator:
def __call__(self, *args):
return 475
def cleanup(self):
pass
with local_persistant_storage("stirring_calibration") as cache:
cache["linear_v1"] = json.dumps({"rpm_coef": 0.1, "intercept": 20})
target_rpm = 500
current_dc = Stirrer.duty_cycle
with Stirrer(target_rpm, unit, exp,
rpm_calculator=FakeRpmCalculator()) as st: # type: ignore
st.start_stirring()
assert st.duty_cycle == current_dc - 0.9 * (current_dc -
(0.1 * target_rpm + 20))
pause()
pause()
current_dc = st.duty_cycle
target_rpm = 600
publish(f"pioreactor/{unit}/{exp}/stirring/target_rpm/set", target_rpm)
pause()
pause()
assert st.duty_cycle == current_dc - 0.9 * (current_dc -
(0.1 * target_rpm + 20))
def test_editing_readonly_attr_via_mqtt() -> None:
class TestJob(BackgroundJob):
published_settings = {
"readonly_attr": {
"datatype": "float",
"settable": False,
},
}
warning_logs = []
def collect_logs(msg: MQTTMessage) -> None:
if "readonly" in msg.payload.decode():
warning_logs.append(msg)
subscribe_and_callback(
collect_logs,
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/logs/app",
)
with TestJob(job_name="job",
unit=get_unit_name(),
experiment=get_latest_experiment_name()):
publish(
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/job/readonly_attr/set",
1.0,
)
pause()
assert len(warning_logs) > 0
def test_changing_parameters_over_mqtt_with_unknown_parameter() -> None:
with DosingAutomation(
target_growth_rate=0.05,
target_od=1.0,
duration=60,
unit=unit,
experiment=experiment,
):
logs = []
def append_logs(msg):
if "garbage" in msg.payload.decode():
logs.append(msg.payload)
pubsub.subscribe_and_callback(
append_logs, f"pioreactor/{unit}/{experiment}/logs/app")
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_automation/garbage/set",
0.07)
# there should be a log published with "Unable to set garbage in dosing_automation"
pause()
assert len(logs) > 0
def test_watchdog_will_try_to_fix_lost_job() -> None:
wd = WatchDog(leader_hostname, UNIVERSAL_EXPERIMENT)
pause()
# start a monitor job
monitor = Monitor(leader_hostname, UNIVERSAL_EXPERIMENT)
pause()
pause()
# suppose it disconnects from broker for long enough that the last will is sent
publish(
f"pioreactor/{leader_hostname}/{UNIVERSAL_EXPERIMENT}/monitor/$state",
"lost")
pause()
pause()
pause()
pause()
pause()
pause()
pause()
assert monitor.sub_client._will
wd.set_state(wd.DISCONNECTED)
monitor.set_state(monitor.DISCONNECTED)
def test_execute_io_action():
pubsub.publish(
f"pioreactor/{unit}/{experiment}/throughput_calculating/media_throughput",
None,
retain=True,
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/throughput_calculating/alt_media_throughput",
None,
retain=True,
)
ca = DosingAutomation(unit=unit, experiment=experiment)
ca.execute_io_action(media_ml=0.65,
alt_media_ml=0.35,
waste_ml=0.65 + 0.35)
pause()
assert ca.throughput_calculator.media_throughput == 0.65
assert ca.throughput_calculator.alt_media_throughput == 0.35
ca.execute_io_action(media_ml=0.15, alt_media_ml=0.15, waste_ml=0.3)
pause()
assert ca.throughput_calculator.media_throughput == 0.80
assert ca.throughput_calculator.alt_media_throughput == 0.50
ca.execute_io_action(media_ml=1.0, alt_media_ml=0, waste_ml=1)
pause()
assert ca.throughput_calculator.media_throughput == 1.80
assert ca.throughput_calculator.alt_media_throughput == 0.50
ca.execute_io_action(media_ml=0.0, alt_media_ml=1.0, waste_ml=1)
pause()
assert ca.throughput_calculator.media_throughput == 1.80
assert ca.throughput_calculator.alt_media_throughput == 1.50
ca.set_state("disconnected")
def test_throughput_calculator_manual_set() -> None:
with local_persistant_storage("media_throughput") as c:
c[experiment] = str(1.0)
with local_persistant_storage("alt_media_throughput") as c:
c[experiment] = str(1.5)
with DosingController(
"turbidostat",
target_od=1.0,
duration=5 / 60,
volume=1.0,
unit=unit,
experiment=experiment,
) as algo:
pause()
assert algo.automation_job.media_throughput == 1.0
assert algo.automation_job.alt_media_throughput == 1.5
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_automation/alt_media_throughput/set",
0,
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_automation/media_throughput/set",
0)
pause()
pause()
assert algo.automation_job.media_throughput == 0
assert algo.automation_job.alt_media_throughput == 0
def test_what_happens_when_an_error_occurs_in_init_but_we_catch_and_disconnect(
) -> None:
class TestJob(BackgroundJob):
def __init__(self, unit: str, experiment: str) -> None:
super(TestJob, self).__init__(job_name="testjob",
unit=unit,
experiment=experiment)
try:
1 / 0
except Exception as e:
self.logger.error("Error!")
self.set_state("disconnected")
raise e
publish("pioreactor/unit/exp/testjob/$state", None, retain=True)
state = []
def update_state(msg: MQTTMessage) -> None:
state.append(msg.payload.decode())
subscribe_and_callback(update_state, "pioreactor/unit/exp/testjob/$state")
with pytest.raises(ZeroDivisionError):
with TestJob(unit="unit", experiment="exp"):
pass
pause()
assert state[-1] == "disconnected"
with local_intermittent_storage("pio_jobs_running") as cache:
assert "testjob" not in cache
def test_error_in_subscribe_and_callback_is_logged() -> None:
class TestJob(BackgroundJob):
def __init__(self, *args, **kwargs) -> None:
super(TestJob, self).__init__(*args, **kwargs)
self.start_passive_listeners()
def start_passive_listeners(self) -> None:
self.subscribe_and_callback(self.callback,
"pioreactor/testing/subscription")
def callback(self, msg: MQTTMessage) -> None:
print(1 / 0)
error_logs = []
def collect_error_logs(msg: MQTTMessage) -> None:
if "ERROR" in msg.payload.decode():
error_logs.append(msg)
subscribe_and_callback(
collect_error_logs,
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/logs/app",
)
with TestJob(job_name="job",
unit=get_unit_name(),
experiment=get_latest_experiment_name()):
publish("pioreactor/testing/subscription", "test")
pause()
pause()
assert len(error_logs) > 0
assert "division by zero" in error_logs[0].payload.decode()
def test_changing_duty_cycle_over_mqtt() -> None:
with ContinuousCycle(unit=unit, experiment=experiment) as algo:
assert algo.duty_cycle == 100
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_automation/duty_cycle/set",
50)
pause()
assert algo.duty_cycle == 50
def __exit__(self, *args) -> None:
self.client.loop_stop()
self.client.disconnect()
publish(
f"pioreactor/{self.unit}/{self.experiment}/{self.name}/$state",
"disconnected",
qos=QOS.AT_LEAST_ONCE,
retain=True,
)
return
def test_publish_duty_cycle():
publish(f"pioreactor/{unit}/{exp}/stirring/duty_cycle", None, retain=True)
pause()
original_dc = 50
st = Stirrer(original_dc, unit, exp)
assert st.duty_cycle == original_dc
pause()
message = subscribe(f"pioreactor/{unit}/{exp}/stirring/duty_cycle")
assert float(message.payload) == 50
def test_silent():
LEDController("silent", unit=unit, experiment=experiment)
pause()
pause()
pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01")
pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "1.0")
pause()
r = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/led_control/led_automation",
timeout=1)
assert r.payload.decode() == "silent"
def emit(self, record):
msg = self.format(record)
self.client.publish(self.topic, msg, qos=self.qos, retain=False)
if config.getboolean("error_reporting",
"send_to_Pioreactor_com",
fallback=False):
# turned off, by default
if record.levelno == logging.ERROR:
# TODO: build this service!
publish(self.topic, msg, hostname="mqtt.pioreactor.com")
def test_publish_target_rpm() -> None:
publish(f"pioreactor/{unit}/{exp}/stirring/target_rpm", None, retain=True)
pause()
target_rpm = 500
with Stirrer(target_rpm, unit, exp, rpm_calculator=RpmCalculator()) as st:
assert st.target_rpm == target_rpm
pause()
message = subscribe(f"pioreactor/{unit}/{exp}/stirring/target_rpm")
assert message is not None
assert float(message.payload) == 500
def test_state_transition_callbacks() -> None:
class TestJob(BackgroundJob):
called_on_init = False
called_on_ready = False
called_on_sleeping = False
called_on_ready_to_sleeping = False
called_on_sleeping_to_ready = False
called_on_init_to_ready = False
def __init__(self, unit: str, experiment: str) -> None:
super(TestJob, self).__init__(job_name="testjob",
unit=unit,
experiment=experiment)
def on_init(self) -> None:
self.called_on_init = True
def on_ready(self) -> None:
self.called_on_ready = True
def on_sleeping(self) -> None:
self.called_on_sleeping = True
def on_ready_to_sleeping(self) -> None:
self.called_on_ready_to_sleeping = True
def on_sleeping_to_ready(self) -> None:
self.called_on_sleeping_to_ready = True
def on_init_to_ready(self) -> None:
self.called_on_init_to_ready = True
unit, exp = get_unit_name(), get_latest_experiment_name()
with TestJob(unit, exp) as tj:
assert tj.called_on_init
assert tj.called_on_init_to_ready
assert tj.called_on_ready
publish(f"pioreactor/{unit}/{exp}/{tj.job_name}/$state/set",
tj.SLEEPING)
pause()
pause()
pause()
pause()
assert tj.called_on_ready_to_sleeping
assert tj.called_on_sleeping
publish(f"pioreactor/{unit}/{exp}/{tj.job_name}/$state/set", tj.READY)
pause()
pause()
pause()
pause()
assert tj.called_on_sleeping_to_ready
def test_changing_parameters_over_mqtt_with_unknown_parameter():
algo = DosingAutomation(
target_growth_rate=0.05,
target_od=1.0,
duration=60,
unit=unit,
experiment=experiment,
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/dosing_automation/garbage/set", 0.07)
pause()
algo.set_state("disconnected")
def __call__(self):
self.od_reading *= np.exp(self.growth_rate / 60 / 60 / samples_per_second)
voltage = 0.1 * self.od_reading
payload = {
"od_raw": {"1": {"voltage": voltage, "angle": "90"}},
"timestamp": "2021-06-06T15:08:12.081153",
}
publish(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
json.dumps(payload),
)
def test_duration_and_timer() -> None:
algo = PIDMorbidostat(
target_od=1.0,
target_growth_rate=0.01,
duration=5 / 60,
unit=unit,
experiment=experiment,
)
assert algo.latest_event is None
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate",
'{"growth_rate": 0.08}',
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered",
'{"od_filtered": 0.5}',
)
time.sleep(10)
pause()
assert isinstance(algo.latest_event, events.NoEvent)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate",
'{"growth_rate": 0.08}',
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered",
'{"od_filtered": 0.95}',
)
time.sleep(10)
pause()
assert isinstance(algo.latest_event, events.AddAltMediaEvent)
algo.set_state(algo.DISCONNECTED)
def test_silent_automation() -> None:
with Silent(volume=None, duration=60, unit=unit,
experiment=experiment) as algo:
pause()
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate",
'{"growth_rate": 0.01}',
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered",
'{"od_filtered": 1.0}',
)
pause()
assert algo.run() is None
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate",
'{"growth_rate": 0.02}',
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered",
'{"od_filtered": 1.1}',
)
pause()
assert algo.run() is None