def watch_for_lost_state(self, state_message: MQTTMessage) -> None:
if state_message.payload.decode() == self.LOST:
# TODO: this song-and-dance works for monitor, why not extend it to other jobs...
# let's try pinging the unit a few times first:
unit = state_message.topic.split("/")[1]
self.logger.warning(
f"{unit} seems to be lost. Trying to re-establish connection..."
)
time.sleep(5)
self.pub_client.publish(
f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state/set",
self.READY)
time.sleep(25)
msg = subscribe(
f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state",
timeout=15)
if msg is None:
return
current_state = msg.payload.decode()
if current_state == self.LOST:
# failed, let's confirm to user
self.logger.error(
f"{unit} was lost. We will continue checking for re-connection."
)
else:
self.logger.info(f"Update: {unit} is connected. All is well.")
# continue to pull the latest state to see if anything has changed.
while True:
time.sleep(2.5 * 60)
msg = subscribe(
f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state",
timeout=15)
assert msg is not None
current_state = msg.payload.decode()
if current_state != self.LOST:
self.logger.info(
f"Update: {unit} is connected. All is well.")
return
def get_initial_alt_media_throughput(self):
message = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/alt_media_throughput",
timeout=2,
)
if message:
return float(message.payload)
else:
return 0
def yield_from_mqtt() -> Generator[dict, None, None]:
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
allow_retained=False,
)
if msg is None:
continue
yield json.loads(msg.payload)
def get_current_state_from_broker(unit, experiment):
# TODO: It's possible to also get this information from the DAC device. Not
# sure what is better
# this also ignores the status of "power on"
msg = subscribe(f"pioreactor/{unit}/{experiment}/leds/intensity",
timeout=0.5)
if msg:
return json.loads(msg.payload)
else:
return {c: 0 for c in CHANNELS}
def test_silent():
LEDController("silent", unit=unit, experiment=experiment)
pause()
pause()
pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01")
pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "1.0")
pause()
r = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/led_control/led_automation",
timeout=1)
assert r.payload.decode() == "silent"
def test_publish_duty_cycle():
publish(f"pioreactor/{unit}/{exp}/stirring/duty_cycle", None, retain=True)
pause()
original_dc = 50
st = Stirrer(original_dc, unit, exp)
assert st.duty_cycle == original_dc
pause()
message = subscribe(f"pioreactor/{unit}/{exp}/stirring/duty_cycle")
assert float(message.payload) == 50
def test_track_od():
con = LEDController("track_od", unit=unit, experiment=experiment)
pause()
pause()
pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01")
pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "1.0")
pause()
pause()
r = pubsub.subscribe(f"pioreactor/{unit}/{experiment}/leds/B/intensity",
timeout=1)
assert float(r.payload.decode()) == 0.1
pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01")
pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "2.0")
pause()
con.led_automation_job.run()
pause()
r = pubsub.subscribe(f"pioreactor/{unit}/{experiment}/leds/B/intensity",
timeout=1)
assert float(r.payload.decode()) == 0.2
def test_publish_target_rpm() -> None:
publish(f"pioreactor/{unit}/{exp}/stirring/target_rpm", None, retain=True)
pause()
target_rpm = 500
with Stirrer(target_rpm, unit, exp, rpm_calculator=RpmCalculator()) as st:
assert st.target_rpm == target_rpm
pause()
message = subscribe(f"pioreactor/{unit}/{exp}/stirring/target_rpm")
assert message is not None
assert float(message.payload) == 500
def set_od_variances(self):
# we check if the broker has variance/median stats, and if not, run it ourselves.
message = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/od_normalization/variance",
timeout=2,
qos=QOS.EXACTLY_ONCE,
)
if message and not self.ignore_cache:
return self.json_to_sorted_dict(message.payload)
else:
od_normalization(unit=self.unit, experiment=self.experiment)
return self.set_od_normalization_factors()
def set_initial_growth_rate(self):
if self.ignore_cache:
return 0
message = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/growth_rate",
timeout=2,
qos=QOS.EXACTLY_ONCE,
)
if message:
return float(message.payload)
else:
return 0
def test_persist_in_published_settings() -> None:
class TestJob(BackgroundJob):
published_settings = {
"persist_this": {
"datatype": "float",
"settable": True,
"persist": True
},
"dont_persist_this": {
"datatype": "float",
"settable": True,
},
}
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.persist_this = "persist_this"
self.dont_persist_this = "dont_persist_this"
with TestJob(job_name="test_job",
unit=get_unit_name(),
experiment=get_latest_experiment_name()):
pause()
pause()
pause()
msg = subscribe(
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/test_job/persist_this",
timeout=2,
)
assert msg is not None
assert msg.payload.decode() == "persist_this"
msg = subscribe(
f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/test_job/dont_persist_this",
timeout=2,
)
assert msg is None
def set_od_normalization_factors(self):
# we check if the broker has variance/median stats, and if not, run it ourselves.
message = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/od_normalization/median",
timeout=2,
qos=QOS.EXACTLY_ONCE,
)
if message and not self.ignore_cache:
return self.json_to_sorted_dict(message.payload)
else:
assert ("od_reading" in pio_jobs_running()
), "OD reading should be running. Stopping."
od_normalization(unit=self.unit, experiment=self.experiment)
return self.set_od_normalization_factors()
def update_ekf_variance_after_event(self, minutes: float, factor: float) -> None:
if is_testing_env():
msg = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/adc_reader/interval",
timeout=1.0,
)
if msg:
interval = float(msg.payload)
else:
interval = 1
self.ekf.scale_OD_variance_for_next_n_seconds(
factor, minutes * (12 * interval)
)
else:
self.ekf.scale_OD_variance_for_next_n_seconds(factor, minutes * 60)
def test_publish_measured_rpm() -> None:
publish(f"pioreactor/{unit}/{exp}/stirring/measured_rpm",
None,
retain=True)
pause()
target_rpm = 500
with Stirrer(target_rpm, unit, exp,
rpm_calculator=RpmFromFrequency()) as st:
st.start_stirring()
assert st.target_rpm == target_rpm
pause()
message = subscribe(f"pioreactor/{unit}/{exp}/stirring/measured_rpm")
assert message is not None
assert json.loads(message.payload)["rpm"] == 0
def get_latest_experiment_name():
if "pytest" in sys.modules or os.environ.get("TESTING"):
return "testing_experiment"
from pioreactor.pubsub import subscribe
mqtt_msg = subscribe("pioreactor/latest_experiment", timeout=1)
if mqtt_msg:
return mqtt_msg.payload.decode()
else:
# if there is no experiment (i.e. on first boot of device), don't run.
import logging
logger = logging.getLogger("pioreactor")
logger.info(
"No experiment running, exiting. Try creating a new experiment first."
)
sys.exit()
def test_check_job_states_in_monitor() -> None:
unit = get_unit_name()
exp = UNIVERSAL_EXPERIMENT
# suppose od_reading is READY when monitor starts, but isn't actually running, ex after a reboot on a worker.
publish(
f"pioreactor/{unit}/{get_latest_experiment_name()}/od_reading/$state",
"ready",
retain=True,
)
with Monitor(unit=unit, experiment=exp):
time.sleep(10)
message = subscribe(
f"pioreactor/{unit}/{get_latest_experiment_name()}/od_reading/$state"
)
assert message is not None
assert message.payload.decode() == "lost"
def get_latest_experiment_name() -> str:
if os.environ.get("EXPERIMENT") is not None:
return os.environ["EXPERIMENT"]
elif is_testing_env():
return "_testing_experiment"
from pioreactor.pubsub import subscribe
mqtt_msg = subscribe("pioreactor/latest_experiment", timeout=2)
if mqtt_msg:
return mqtt_msg.payload.decode()
else:
from pioreactor.logging import create_logger
logger = create_logger("pioreactor", experiment=UNIVERSAL_EXPERIMENT)
logger.info(
"No experiment running. Try creating a new experiment first.")
return NO_EXPERIMENT
def initialize_extended_kalman_filter(self):
import numpy as np
latest_od = subscribe(
f"pioreactor/{self.unit}/{self.experiment}/od_raw_batched")
angles_and_initial_points = self.scale_raw_observations(
self.json_to_sorted_dict(latest_od.payload))
initial_state = np.array(
[*angles_and_initial_points.values(), self.initial_growth_rate])
d = initial_state.shape[0]
# empirically selected
initial_covariance = 0.0001 * np.diag(initial_state.tolist()[:-1] +
[0.00001])
OD_process_covariance = self.create_OD_covariance(
angles_and_initial_points.keys())
rate_process_variance = (
config.getfloat("growth_rate_kalman", "rate_variance") *
self.dt)**2
process_noise_covariance = np.block([
[OD_process_covariance, 0 * np.ones((d - 1, 1))],
[0 * np.ones((1, d - 1)), rate_process_variance],
])
observation_noise_covariance = self.create_obs_noise_covariance(
angles_and_initial_points.keys())
return (
ExtendedKalmanFilter(
initial_state,
initial_covariance,
process_noise_covariance,
observation_noise_covariance,
dt=self.dt,
),
angles_and_initial_points.keys(),
)
def cluster_status() -> None:
click.secho(
f"{'Unit / hostname':20s} {'Is leader?':15s} {'IP address':20s} {'State':15s} {'Reachable?':10s}",
bold=True,
)
for hostname, inventory_status in config["network.inventory"].items():
if inventory_status == "0":
continue
# get ip
if get_unit_name() == hostname:
ip = networking.get_ip()
else:
try:
ip = socket.gethostbyname(hostname)
except OSError:
ip = "Unknown"
# get state
result = subscribe(
f"pioreactor/{hostname}/{UNIVERSAL_EXPERIMENT}/monitor/$state", timeout=1
)
if result:
state = result.payload.decode()
else:
state = "Unknown"
state = click.style(f"{state:15s}", fg="green" if state == "ready" else "red")
# is reachable?
reachable = networking.is_reachable(hostname)
click.echo(
f"{hostname:20s} {('Y' if hostname==get_leader_hostname() else 'N'):15s} {ip:20s} {state} {( click.style('Y', fg='green') if reachable else click.style('N', fg='red') ):10s}"
)
def test_changing_automation_over_mqtt() -> None:
with LEDController("silent", duration=60, unit=unit,
experiment=experiment) as ld:
pause()
pause()
r = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/led_control/automation_name",
timeout=1)
assert r is not None
assert r.payload.decode() == "silent"
pause()
pause()
pubsub.publish(
f"pioreactor/{unit}/{experiment}/led_control/automation/set",
'{"automation_name": "silent", "duration": "20"}',
)
pause()
pause()
pause()
pause()
pause()
assert ld.automation_name == "silent"
assert ld.automation["duration"] == "20"
def test_silent() -> None:
with LEDController("silent", duration=60, unit=unit,
experiment=experiment):
pause()
pause()
pause()
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate",
json.dumps({
"growth_rate": 0.01,
"timestamp": "2010-01-01 12:00:00"
}),
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered",
'{"od_filtered": 1.0}',
)
pause()
pause()
r = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/led_control/automation_name",
timeout=1)
assert r is not None
assert r.payload.decode() == "silent"
def yield_from_mqtt():
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{testing_experiment}/od_reading/od_raw_batched"
)
yield json.loads(msg.payload)
