def watch_for_lost_state(self, state_message: MQTTMessage) -> None: if state_message.payload.decode() == self.LOST: # TODO: this song-and-dance works for monitor, why not extend it to other jobs... # let's try pinging the unit a few times first: unit = state_message.topic.split("/")[1] self.logger.warning( f"{unit} seems to be lost. Trying to re-establish connection..." ) time.sleep(5) self.pub_client.publish( f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state/set", self.READY) time.sleep(25) msg = subscribe( f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state", timeout=15) if msg is None: return current_state = msg.payload.decode() if current_state == self.LOST: # failed, let's confirm to user self.logger.error( f"{unit} was lost. We will continue checking for re-connection." ) else: self.logger.info(f"Update: {unit} is connected. All is well.") # continue to pull the latest state to see if anything has changed. while True: time.sleep(2.5 * 60) msg = subscribe( f"pioreactor/{unit}/{UNIVERSAL_EXPERIMENT}/monitor/$state", timeout=15) assert msg is not None current_state = msg.payload.decode() if current_state != self.LOST: self.logger.info( f"Update: {unit} is connected. All is well.") return
def get_initial_alt_media_throughput(self): message = subscribe( f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/alt_media_throughput", timeout=2, ) if message: return float(message.payload) else: return 0
def yield_from_mqtt() -> Generator[dict, None, None]: while True: msg = pubsub.subscribe( f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched", allow_retained=False, ) if msg is None: continue yield json.loads(msg.payload)
def get_current_state_from_broker(unit, experiment): # TODO: It's possible to also get this information from the DAC device. Not # sure what is better # this also ignores the status of "power on" msg = subscribe(f"pioreactor/{unit}/{experiment}/leds/intensity", timeout=0.5) if msg: return json.loads(msg.payload) else: return {c: 0 for c in CHANNELS}
def test_silent(): LEDController("silent", unit=unit, experiment=experiment) pause() pause() pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01") pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "1.0") pause() r = pubsub.subscribe( f"pioreactor/{unit}/{experiment}/led_control/led_automation", timeout=1) assert r.payload.decode() == "silent"
def test_publish_duty_cycle(): publish(f"pioreactor/{unit}/{exp}/stirring/duty_cycle", None, retain=True) pause() original_dc = 50 st = Stirrer(original_dc, unit, exp) assert st.duty_cycle == original_dc pause() message = subscribe(f"pioreactor/{unit}/{exp}/stirring/duty_cycle") assert float(message.payload) == 50
def test_track_od(): con = LEDController("track_od", unit=unit, experiment=experiment) pause() pause() pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01") pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "1.0") pause() pause() r = pubsub.subscribe(f"pioreactor/{unit}/{experiment}/leds/B/intensity", timeout=1) assert float(r.payload.decode()) == 0.1 pubsub.publish(f"pioreactor/{unit}/{experiment}/growth_rate", "0.01") pubsub.publish(f"pioreactor/{unit}/{experiment}/od_filtered/135/0", "2.0") pause() con.led_automation_job.run() pause() r = pubsub.subscribe(f"pioreactor/{unit}/{experiment}/leds/B/intensity", timeout=1) assert float(r.payload.decode()) == 0.2
def test_publish_target_rpm() -> None: publish(f"pioreactor/{unit}/{exp}/stirring/target_rpm", None, retain=True) pause() target_rpm = 500 with Stirrer(target_rpm, unit, exp, rpm_calculator=RpmCalculator()) as st: assert st.target_rpm == target_rpm pause() message = subscribe(f"pioreactor/{unit}/{exp}/stirring/target_rpm") assert message is not None assert float(message.payload) == 500
def set_od_variances(self): # we check if the broker has variance/median stats, and if not, run it ourselves. message = subscribe( f"pioreactor/{self.unit}/{self.experiment}/od_normalization/variance", timeout=2, qos=QOS.EXACTLY_ONCE, ) if message and not self.ignore_cache: return self.json_to_sorted_dict(message.payload) else: od_normalization(unit=self.unit, experiment=self.experiment) return self.set_od_normalization_factors()
def set_initial_growth_rate(self): if self.ignore_cache: return 0 message = subscribe( f"pioreactor/{self.unit}/{self.experiment}/growth_rate", timeout=2, qos=QOS.EXACTLY_ONCE, ) if message: return float(message.payload) else: return 0
def test_persist_in_published_settings() -> None: class TestJob(BackgroundJob): published_settings = { "persist_this": { "datatype": "float", "settable": True, "persist": True }, "dont_persist_this": { "datatype": "float", "settable": True, }, } def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.persist_this = "persist_this" self.dont_persist_this = "dont_persist_this" with TestJob(job_name="test_job", unit=get_unit_name(), experiment=get_latest_experiment_name()): pause() pause() pause() msg = subscribe( f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/test_job/persist_this", timeout=2, ) assert msg is not None assert msg.payload.decode() == "persist_this" msg = subscribe( f"pioreactor/{get_unit_name()}/{get_latest_experiment_name()}/test_job/dont_persist_this", timeout=2, ) assert msg is None
def set_od_normalization_factors(self): # we check if the broker has variance/median stats, and if not, run it ourselves. message = subscribe( f"pioreactor/{self.unit}/{self.experiment}/od_normalization/median", timeout=2, qos=QOS.EXACTLY_ONCE, ) if message and not self.ignore_cache: return self.json_to_sorted_dict(message.payload) else: assert ("od_reading" in pio_jobs_running() ), "OD reading should be running. Stopping." od_normalization(unit=self.unit, experiment=self.experiment) return self.set_od_normalization_factors()
def update_ekf_variance_after_event(self, minutes: float, factor: float) -> None: if is_testing_env(): msg = subscribe( f"pioreactor/{self.unit}/{self.experiment}/adc_reader/interval", timeout=1.0, ) if msg: interval = float(msg.payload) else: interval = 1 self.ekf.scale_OD_variance_for_next_n_seconds( factor, minutes * (12 * interval) ) else: self.ekf.scale_OD_variance_for_next_n_seconds(factor, minutes * 60)
def test_publish_measured_rpm() -> None: publish(f"pioreactor/{unit}/{exp}/stirring/measured_rpm", None, retain=True) pause() target_rpm = 500 with Stirrer(target_rpm, unit, exp, rpm_calculator=RpmFromFrequency()) as st: st.start_stirring() assert st.target_rpm == target_rpm pause() message = subscribe(f"pioreactor/{unit}/{exp}/stirring/measured_rpm") assert message is not None assert json.loads(message.payload)["rpm"] == 0
def get_latest_experiment_name(): if "pytest" in sys.modules or os.environ.get("TESTING"): return "testing_experiment" from pioreactor.pubsub import subscribe mqtt_msg = subscribe("pioreactor/latest_experiment", timeout=1) if mqtt_msg: return mqtt_msg.payload.decode() else: # if there is no experiment (i.e. on first boot of device), don't run. import logging logger = logging.getLogger("pioreactor") logger.info( "No experiment running, exiting. Try creating a new experiment first." ) sys.exit()
def test_check_job_states_in_monitor() -> None: unit = get_unit_name() exp = UNIVERSAL_EXPERIMENT # suppose od_reading is READY when monitor starts, but isn't actually running, ex after a reboot on a worker. publish( f"pioreactor/{unit}/{get_latest_experiment_name()}/od_reading/$state", "ready", retain=True, ) with Monitor(unit=unit, experiment=exp): time.sleep(10) message = subscribe( f"pioreactor/{unit}/{get_latest_experiment_name()}/od_reading/$state" ) assert message is not None assert message.payload.decode() == "lost"
def get_latest_experiment_name() -> str: if os.environ.get("EXPERIMENT") is not None: return os.environ["EXPERIMENT"] elif is_testing_env(): return "_testing_experiment" from pioreactor.pubsub import subscribe mqtt_msg = subscribe("pioreactor/latest_experiment", timeout=2) if mqtt_msg: return mqtt_msg.payload.decode() else: from pioreactor.logging import create_logger logger = create_logger("pioreactor", experiment=UNIVERSAL_EXPERIMENT) logger.info( "No experiment running. Try creating a new experiment first.") return NO_EXPERIMENT
def initialize_extended_kalman_filter(self): import numpy as np latest_od = subscribe( f"pioreactor/{self.unit}/{self.experiment}/od_raw_batched") angles_and_initial_points = self.scale_raw_observations( self.json_to_sorted_dict(latest_od.payload)) initial_state = np.array( [*angles_and_initial_points.values(), self.initial_growth_rate]) d = initial_state.shape[0] # empirically selected initial_covariance = 0.0001 * np.diag(initial_state.tolist()[:-1] + [0.00001]) OD_process_covariance = self.create_OD_covariance( angles_and_initial_points.keys()) rate_process_variance = ( config.getfloat("growth_rate_kalman", "rate_variance") * self.dt)**2 process_noise_covariance = np.block([ [OD_process_covariance, 0 * np.ones((d - 1, 1))], [0 * np.ones((1, d - 1)), rate_process_variance], ]) observation_noise_covariance = self.create_obs_noise_covariance( angles_and_initial_points.keys()) return ( ExtendedKalmanFilter( initial_state, initial_covariance, process_noise_covariance, observation_noise_covariance, dt=self.dt, ), angles_and_initial_points.keys(), )
def cluster_status() -> None: click.secho( f"{'Unit / hostname':20s} {'Is leader?':15s} {'IP address':20s} {'State':15s} {'Reachable?':10s}", bold=True, ) for hostname, inventory_status in config["network.inventory"].items(): if inventory_status == "0": continue # get ip if get_unit_name() == hostname: ip = networking.get_ip() else: try: ip = socket.gethostbyname(hostname) except OSError: ip = "Unknown" # get state result = subscribe( f"pioreactor/{hostname}/{UNIVERSAL_EXPERIMENT}/monitor/$state", timeout=1 ) if result: state = result.payload.decode() else: state = "Unknown" state = click.style(f"{state:15s}", fg="green" if state == "ready" else "red") # is reachable? reachable = networking.is_reachable(hostname) click.echo( f"{hostname:20s} {('Y' if hostname==get_leader_hostname() else 'N'):15s} {ip:20s} {state} {( click.style('Y', fg='green') if reachable else click.style('N', fg='red') ):10s}" )
def test_changing_automation_over_mqtt() -> None: with LEDController("silent", duration=60, unit=unit, experiment=experiment) as ld: pause() pause() r = pubsub.subscribe( f"pioreactor/{unit}/{experiment}/led_control/automation_name", timeout=1) assert r is not None assert r.payload.decode() == "silent" pause() pause() pubsub.publish( f"pioreactor/{unit}/{experiment}/led_control/automation/set", '{"automation_name": "silent", "duration": "20"}', ) pause() pause() pause() pause() pause() assert ld.automation_name == "silent" assert ld.automation["duration"] == "20"
def test_silent() -> None: with LEDController("silent", duration=60, unit=unit, experiment=experiment): pause() pause() pause() pubsub.publish( f"pioreactor/{unit}/{experiment}/growth_rate_calculating/growth_rate", json.dumps({ "growth_rate": 0.01, "timestamp": "2010-01-01 12:00:00" }), ) pubsub.publish( f"pioreactor/{unit}/{experiment}/growth_rate_calculating/od_filtered", '{"od_filtered": 1.0}', ) pause() pause() r = pubsub.subscribe( f"pioreactor/{unit}/{experiment}/led_control/automation_name", timeout=1) assert r is not None assert r.payload.decode() == "silent"
def yield_from_mqtt(): while True: msg = pubsub.subscribe( f"pioreactor/{unit}/{testing_experiment}/od_reading/od_raw_batched" ) yield json.loads(msg.payload)