def publish_multiple(
list_of_topic_message_tuples, hostname=leader_hostname, retries=10, **mqtt_kwargs
):
"""
list_of_topic_message_tuples is of the form ("<topic>", "<payload>", qos, retain)
"""
for retry_count in range(retries):
try:
mqtt_publish.multiple(
list_of_topic_message_tuples, hostname=hostname, **mqtt_kwargs
)
return
except (ConnectionRefusedError, socket.gaierror, OSError, socket.timeout):
# possible that leader is down/restarting, keep trying, but log to local machine.
from pioreactor.logging import create_logger
logger = create_logger("pubsub.publish_multiple", to_mqtt=False)
logger.debug(
f"Attempt {retry_count}: Unable to connect to host: {hostname}",
exc_info=True,
)
time.sleep(5 * retry_count) # linear backoff
else:
logger = create_logger("pubsub.publish_multiple", to_mqtt=False)
logger.error(f"Unable to connect to host: {hostname}. Exiting.")
raise ConnectionRefusedError(f"Unable to connect to host: {hostname}.")
def get(self, section: str, option: str, *args, **kwargs): # type: ignore
try:
return super().get(section, option, *args, **kwargs)
except (configparser.NoSectionError, configparser.NoOptionError):
from pioreactor.logging import create_logger
create_logger("read config").error(
f"""No found in configuration: '{section}.{option}'. Are you missing the following in your config?
[{section}]
{option}=some value
""")
raise
def update(ui: bool, app: bool, dev: bool) -> None:
import subprocess
from json import loads
from pioreactor.mureq import get
logger = create_logger(
"update", unit=get_unit_name(), experiment=UNIVERSAL_EXPERIMENT
)
if (not app) and (not ui):
click.echo("Nothing to do. Specify either --app or --ui.")
if app:
if not dev:
latest_release_metadata = loads(
get(
"https://api.github.com/repos/pioreactor/pioreactor/releases/latest"
).body
)
latest_release_version = latest_release_metadata["name"]
url_to_get_whl = f"https://github.com/Pioreactor/pioreactor/releases/download/{latest_release_version}/pioreactor-{latest_release_version}-py3-none-any.whl"
command = f'sudo pip3 install "pioreactor @ {url_to_get_whl}"'
else:
latest_release_version = "master"
command = "sudo pip3 install -U --force-reinstall https://github.com/pioreactor/pioreactor/archive/master.zip"
p = subprocess.run(
command,
shell=True,
universal_newlines=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
if p.returncode == 0:
logger.info(f"Updated Pioreactor to version {latest_release_version}.")
else:
logger.error(p.stderr)
if ui and am_I_leader():
cd = "cd ~/pioreactorui/backend"
gitp = "git pull origin master"
npm_install = "npm install"
setup = "pm2 restart ui"
unedit_edited_files = "git checkout ." # TODO: why do I do this. Can I be more specific than `.`? This blocks edits to the contrib folder from sticking around.
command = " && ".join([cd, gitp, setup, npm_install, unedit_edited_files])
p = subprocess.run(
command,
shell=True,
universal_newlines=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
if p.returncode == 0:
logger.info("Updated PioreactorUI to latest version.")
else:
logger.error(p.stderr)
def export_experiment_data(experiment: str, output: str, tables: list) -> None:
"""
Set an experiment, else it defaults to the entire table.
"""
import sqlite3
import zipfile
import csv
logger = create_logger("export_experiment_data")
logger.info(
f"Starting export of table{'s' if len(tables) > 1 else ''}: {', '.join(tables)}."
)
time = datetime.now().strftime("%Y%m%d%H%m%S")
zf = zipfile.ZipFile(output, mode="w", compression=zipfile.ZIP_DEFLATED)
con = sqlite3.connect(config["storage"]["database"])
for table in tables:
cursor = con.cursor()
# so apparently, you can't parameterize the table name in python's sqlite3, so I
# have to use string formatting (SQL-injection vector), but first check that the table exists (else fail)
if not exists_table(cursor, table):
raise ValueError(f"Table {table} does not exist.")
timestamp_to_localtimestamp_clause = generate_timestamp_to_localtimestamp_clause(
cursor, table)
order_by = filter_to_timestamp_columns(get_column_names(
cursor, table)).pop() # just take first...
if experiment is None:
query = f"SELECT {timestamp_to_localtimestamp_clause} * from {table} ORDER BY :order_by"
cursor.execute(query, {"order_by": order_by})
_filename = f"{table}-{time}.dump.csv".replace(" ", "_")
else:
query = f"SELECT {timestamp_to_localtimestamp_clause} * from {table} WHERE experiment=:experiment ORDER BY :order_by"
cursor.execute(query, {
"experiment": experiment,
"order_by": order_by
})
_filename = f"{experiment}-{table}-{time}.dump.csv".replace(
" ", "_")
path_to_file = os.path.join(os.path.dirname(output), _filename)
with open(path_to_file, "w") as csv_file:
csv_writer = csv.writer(csv_file, delimiter=",")
csv_writer.writerow([i[0] for i in cursor.description])
csv_writer.writerows(cursor)
zf.write(path_to_file, arcname=_filename)
con.close()
zf.close()
logger.info("Finished export.")
return
def kill(job: str, units: tuple[str, ...], all_jobs: bool, y: bool) -> None:
"""
Send a SIGTERM signal to JOB. JOB can be any Pioreactor job name, like "stirring".
Example:
> pios kill stirring
multiple jobs accepted:
> pios kill stirring dosing_control
Kill all worker jobs (i.e. this excludes leader jobs like watchdog). Ignores `job` argument.
> pios kill --all
"""
from sh import ssh # type: ignore
if not y:
confirm = input(
f"Confirm killing {str(job) if (not all_jobs) else 'all jobs'} on {units}? Y/n: "
).strip()
if confirm != "Y":
return
command = f"pio kill {' '.join(job)}"
command += "--all-jobs" if all_jobs else ""
logger = create_logger("CLI",
unit=get_unit_name(),
experiment=get_latest_experiment_name())
def _thread_function(unit: str):
logger.debug(f"Executing `{command}` on {unit}.")
try:
ssh(unit, command)
if all_jobs: # tech debt
ssh(
unit,
"pio run led_intensity --intensity 0 --channel A --channel B --channel C --channel D --no-log",
)
return True
except Exception as e:
logger.debug(e, exc_info=True)
logger.error(f"Unable to connect to unit {unit}.")
return False
units = universal_identifier_to_all_active_workers(units)
with ThreadPoolExecutor(max_workers=len(units)) as executor:
results = executor.map(_thread_function, units)
if not all(results):
sys.exit(1)
def _thread_function(unit: str) -> bool:
click.echo(f"Executing `{core_command}` on {unit}.")
try:
ssh(unit, command)
return True
except Exception as e:
logger = create_logger("CLI",
unit=get_unit_name(),
experiment=get_latest_experiment_name())
logger.debug(e, exc_info=True)
logger.error(f"Unable to connect to unit {unit}.")
return False
def _callback(client: Client, userdata: dict, message):
try:
if not allow_retained and message.retain:
return
return actual_callback(message)
except Exception as e:
from pioreactor.logging import create_logger
logger = create_logger(userdata.get("job_name", "pioreactor"))
logger.error(e, exc_info=True)
raise e
def publish(
topic: str, message, hostname: str = leader_hostname, retries: int = 10, **mqtt_kwargs
):
for retry_count in range(retries):
try:
mqtt_publish.single(topic, payload=message, hostname=hostname, **mqtt_kwargs)
return
except (ConnectionRefusedError, socket.gaierror, OSError, socket.timeout):
# possible that leader is down/restarting, keep trying, but log to local machine.
from pioreactor.logging import create_logger
logger = create_logger("pubsub.publish", to_mqtt=False)
logger.debug(
f"Attempt {retry_count}: Unable to connect to host: {hostname}",
exc_info=True,
)
time.sleep(5 * retry_count) # linear backoff
else:
logger = create_logger("pubsub.publish", to_mqtt=False)
logger.error(f"Unable to connect to host: {hostname}.")
raise ConnectionRefusedError(f"Unable to connect to host: {hostname}.")
def uninstall_plugin(name_of_plugin):
logger = create_logger("install_plugin", experiment=UNIVERSAL_EXPERIMENT)
result = subprocess.run(
[
"bash",
"/usr/local/bin/uninstall_pioreactor_plugin.sh",
quote(name_of_plugin),
]
)
if result.returncode == 0:
logger.info(f"Successfully uninstalled plugin {name_of_plugin}.")
else:
logger.error(f"Failed to uninstall plugin {name_of_plugin}. See logs.")
logger.debug(result.stdout)
logger.debug(result.stderr)
def get_latest_experiment_name() -> str:
if os.environ.get("EXPERIMENT") is not None:
return os.environ["EXPERIMENT"]
elif is_testing_env():
return "_testing_experiment"
from pioreactor.pubsub import subscribe
mqtt_msg = subscribe("pioreactor/latest_experiment", timeout=2)
if mqtt_msg:
return mqtt_msg.payload.decode()
else:
from pioreactor.logging import create_logger
logger = create_logger("pioreactor", experiment=UNIVERSAL_EXPERIMENT)
logger.info(
"No experiment running. Try creating a new experiment first.")
return NO_EXPERIMENT
def __init__(self,
pin: GpioPin,
hz: float,
always_use_software: bool = False) -> None:
self.logger = create_logger("PWM")
self.pin = pin
self.hz = hz
if self.is_locked():
self.logger.debug(
f"GPIO-{self.pin} is currently locked but a task is overwriting it. Either too many jobs are trying to access this pin, or a job didn't clean up properly."
)
gpio_helpers.set_gpio_availability(
self.pin, gpio_helpers.GPIO_states.GPIO_UNAVAILABLE)
if (not always_use_software) and (pin in self.HARDWARE_PWM_CHANNELS):
self.pwm = HardwarePWM(self.HARDWARE_PWM_CHANNELS[self.pin],
self.hz)
else:
import RPi.GPIO as GPIO # type: ignore
GPIO.setmode(GPIO.BCM)
GPIO.setup(self.pin, GPIO.OUT, initial=GPIO.LOW)
if self.hz > 2000:
self.logger.warning(
"Setting a PWM to a very high frequency with software. Did you mean to use a hardware PWM?"
)
self.pwm = GPIO.PWM(self.pin, self.hz)
with local_intermittent_storage("pwm_hz") as cache:
cache[str(self.pin)] = str(self.hz)
self.logger.debug(
f"Initialized PWM-{self.pin} with {'hardware' if self.using_hardware else 'software'}, initial frequency is {self.hz}hz."
)
def add_pioreactor(new_name: str) -> None:
"""
Add a new pioreactor worker to the cluster. The pioreactor should already have the worker image installed and is turned on.
"""
# TODO: move this to its own file
import socket
import subprocess
logger = create_logger(
"add_pioreactor", unit=get_unit_name(), experiment=UNIVERSAL_EXPERIMENT
)
logger.info(f"Adding new pioreactor {new_name} to cluster.")
# check to make sure new_name isn't already on the network
# check to make sure raspberrypi.local is on network
checks, max_checks = 0, 20
while not networking.is_hostname_on_network(new_name):
checks += 1
try:
socket.gethostbyname(new_name)
except socket.gaierror:
sleep(3)
click.echo(f"`{new_name}` not found on network - checking again.")
if checks >= max_checks:
logger.error(
f"`{new_name}` not found on network after {max_checks} seconds. Check that you provided the right WiFi credentials to the network, and that the Raspberry Pi is turned on."
)
sys.exit(1)
res = subprocess.call(
[
"bash /usr/local/bin//add_new_pioreactor_worker_from_leader.sh %s"
% (new_name)
],
shell=True,
)
if res == 0:
logger.info(f"New pioreactor {new_name} successfully added to cluster.")
def update(units: tuple[str, ...], dev: bool) -> None:
"""
Pulls and installs the latest code
"""
import paramiko # type: ignore
logger = create_logger("update",
unit=get_unit_name(),
experiment=get_latest_experiment_name())
if dev:
command = "pio update --app --dev"
else:
command = "pio update --app"
def _thread_function(unit: str):
logger.debug(f"Executing `{command}` on {unit}...")
try:
with paramiko.SSHClient() as client:
client.load_system_host_keys()
client.connect(unit, username="******", compress=True)
(stdin, stdout, stderr) = client.exec_command(command)
for line in stderr.readlines():
pass
return True
except Exception as e:
logger.error(f"Unable to connect to unit {unit}.")
logger.debug(e, exc_info=True)
return False
units = universal_identifier_to_all_active_workers(units)
with ThreadPoolExecutor(max_workers=len(units)) as executor:
results = executor.map(_thread_function, units)
if not all(results):
sys.exit(1)
def sync_configs(units: tuple[str, ...], shared: bool, specific: bool) -> None:
"""
Deploys the shared config.ini and worker specific config.inis to the workers.
If neither `--shared` not `--specific` are specified, both are set to true.
"""
import paramiko
logger = create_logger("sync_configs",
unit=get_unit_name(),
experiment=get_latest_experiment_name())
units = universal_identifier_to_all_active_workers(units)
if not shared and not specific:
shared = specific = True
def _thread_function(unit: str) -> bool:
logger.debug(f"Syncing configs on {unit}...")
try:
with paramiko.SSHClient() as client:
client.load_system_host_keys()
client.connect(unit, username="******", compress=True)
with client.open_sftp() as ftp_client:
sync_config_files(ftp_client, unit, shared, specific)
return True
except Exception as e:
logger.error(f"Unable to connect to unit {unit}.")
logger.debug(e, exc_info=True)
return False
# save config.inis to database
save_config_files_to_db(units, shared, specific)
with ThreadPoolExecutor(max_workers=len(units)) as executor:
results = executor.map(_thread_function, units)
if not all(results):
sys.exit(1)
def start_led_control(
automation_name: str,
duration: Optional[float] = None,
skip_first_run=False,
unit: Optional[str] = None,
experiment: Optional[str] = None,
**kwargs,
) -> LEDController:
try:
return LEDController(
automation_name=automation_name,
unit=unit or get_unit_name(),
experiment=experiment or get_latest_experiment_name(),
skip_first_run=skip_first_run,
duration=duration,
**kwargs,
)
except Exception as e:
logger = create_logger("led_automation")
logger.error(e)
logger.debug(e, exc_info=True)
raise e
def uninstall_plugin(plugin: str, units: tuple[str, ...]) -> None:
"""
Uninstalls a plugin from worker and leader
"""
import paramiko
logger = create_logger("uninstall_plugin",
unit=get_unit_name(),
experiment=get_latest_experiment_name())
command = f"pio uninstall-plugin {plugin}"
def _thread_function(unit: str):
logger.debug(f"Executing `{command}` on {unit}...")
try:
with paramiko.SSHClient() as client:
client.load_system_host_keys()
client.connect(unit, username="******", compress=True)
(stdin, stdout, stderr) = client.exec_command(command)
for line in stderr.readlines():
pass
return True
except Exception as e:
logger.error(f"Unable to connect to unit {unit}.")
logger.debug(e, exc_info=True)
return False
units = add_leader(universal_identifier_to_all_active_workers(units))
with ThreadPoolExecutor(max_workers=len(units)) as executor:
results = executor.map(_thread_function, units)
if not all(results):
sys.exit(1)
def start_dosing_control(
automation_name: str,
duration: Optional[float] = None,
skip_first_run: bool = False,
unit: Optional[str] = None,
experiment: Optional[str] = None,
**kwargs,
) -> DosingController:
unit = unit or get_unit_name()
experiment = experiment or get_latest_experiment_name()
try:
kwargs["duration"] = duration
kwargs["unit"] = unit
kwargs["experiment"] = experiment
kwargs["skip_first_run"] = skip_first_run
return DosingController(automation_name, **kwargs) # noqa: F841
except Exception as e:
logger = create_logger("dosing_automation")
logger.error(e)
logger.debug(e, exc_info=True)
raise e
def click_self_test(k: str) -> int:
"""
Test the input/output in the Pioreactor
"""
import sys
unit = get_unit_name()
testing_experiment = get_latest_testing_experiment_name()
experiment = get_latest_experiment_name()
logger = create_logger("self_test", unit=unit, experiment=experiment)
with publish_ready_to_disconnected_state(unit, testing_experiment, "self_test"):
if is_pio_job_running("od_reading", "temperature_automation", "stirring"):
logger.error(
"Make sure Optical Density, Temperature Automation, and Stirring are off before running a self test. Exiting."
)
return 1
functions_to_test = [
(name, f)
for (name, f) in vars(sys.modules[__name__]).items()
if name.startswith("test_")
] # automagically finds the test_ functions.
if k:
functions_to_test = [
(name, f) for (name, f) in functions_to_test if (k in name)
]
# clear the mqtt cache
for name, _ in functions_to_test:
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/{name}",
None,
retain=True,
)
count_tested: int = 0
count_passed: int = 0
for name, test in functions_to_test:
try:
test(logger, unit, testing_experiment)
except Exception:
import traceback
traceback.print_exc()
res = False
else:
res = True
logger.debug(f"{name}: {'✅' if res else '❌'}")
count_tested += 1
count_passed += res
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/{name}",
int(res),
retain=True,
)
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/all_tests_passed",
int(count_passed == count_tested),
retain=True,
)
if count_passed == count_tested:
logger.info("All tests passed ✅")
else:
logger.info(
f"{count_tested-count_passed} failed test{'s' if (count_tested-count_passed) > 1 else ''}."
)
return int(count_passed != count_tested)
def led_intensity(
channels: LedChannel | list[LedChannel],
intensities: float | list[float],
unit: str,
experiment: str,
verbose: bool = True,
source_of_event: Optional[str] = None,
pubsub_client: Optional[Client] = None,
) -> bool:
"""
Parameters
------------
channel: an LED channel or list
intensity: float or list
a value between 0 and 100 to set the LED channel to.
verbose: bool
if True, log the change, and send event to led_event table & mqtt. This is FALSE
in od_reading job, so as to not create spam.
pubsub_client:
provide a MQTT paho client to use for publishing.
Returns
--------
bool representing if the all LED channels intensity were successfully changed
State is also updated in
pioreactor/<unit>/<experiment>/led/<channel>/intensity <intensity>
and
pioreactor/<unit>/<experiment>/leds/intensity {'A': intensityA, 'B': intensityB, ...}
"""
logger = create_logger("led_intensity", experiment=experiment, unit=unit)
updated_successfully = True
if not is_testing_env():
from DAC43608 import DAC43608
else:
logger.debug("DAC43608 not available; using MockDAC43608")
from pioreactor.utils.mock import MockDAC43608 as DAC43608 # type: ignore
if pubsub_client is None:
pubsub_client = create_client()
channels, intensities = _list(channels), _list(intensities)
if len(channels) != len(intensities):
raise ValueError("channels must be the same length as intensities")
# any locked channels?
for channel in channels:
if is_led_channel_locked(channel):
updated_successfully = False
logger.warning(
f"Unable to update channel {channel} due to a lock on it. Please try again."
)
# remove locked channels:
try:
channels, intensities = zip( # type: ignore
*[
(c, i)
for c, i in zip(channels, intensities)
if not is_led_channel_locked(c)
]
)
except ValueError:
# if the only channel being updated is locked, the resulting error is a ValueError: not enough values to unpack (expected 2, got 0)
return updated_successfully
for channel, intensity in zip(channels, intensities):
try:
assert (
0.0 <= intensity <= 100.0
), "intensity should be between 0 and 100, inclusive"
assert (
channel in ALL_LED_CHANNELS
), f"saw incorrect channel {channel}, not in {ALL_LED_CHANNELS}"
intensity = float(intensity)
dac = DAC43608()
dac.power_up(getattr(dac, channel))
dac.set_intensity_to(getattr(dac, channel), intensity / 100.0)
if intensity == 0:
# setting to 0 doesn't fully remove the current, there is some residual current. We turn off
# the channel to guarantee no output.
dac.power_down(getattr(dac, channel))
pubsub_client.publish(
f"pioreactor/{unit}/{experiment}/led/{channel}/intensity",
intensity,
qos=QOS.AT_MOST_ONCE,
retain=True,
)
except ValueError as e:
logger.debug(e, exc_info=True)
logger.error(
"Unable to find I²C for LED driver. Is the Pioreactor HAT attached to the Raspberry Pi? Is I²C enabled on the Raspberry Pi?"
)
updated_successfully = False
return updated_successfully
new_state, old_state = _update_current_state(channels, intensities)
pubsub_client.publish(
f"pioreactor/{unit}/{experiment}/leds/intensity",
dumps(new_state),
qos=QOS.AT_MOST_ONCE,
retain=True,
)
if verbose:
for channel, intensity in zip(channels, intensities):
event = {
"channel": channel,
"intensity": intensity,
"source_of_event": source_of_event,
"timestamp": current_utc_time(),
}
pubsub_client.publish(
f"pioreactor/{unit}/{experiment}/led_events",
dumps(event),
qos=QOS.AT_MOST_ONCE,
retain=False,
)
logger.info(
f"Updated LED {channel} from {old_state[channel]:0.3g}% to {new_state[channel]:0.3g}%."
)
return updated_successfully
def on_connect(client: Client, userdata, flags, rc: int, properties=None):
if rc > 1:
from pioreactor.logging import create_logger
logger = create_logger("pubsub.create_client", to_mqtt=False)
logger.error(f"Connection failed with error code {rc=}: {connack_string(rc)}")
def pump(
unit: str,
experiment: str,
pump_name: str,
ml: Optional[float] = None,
duration: Optional[float] = None,
source_of_event: Optional[str] = None,
calibration: Optional[dict] = None,
continuously: bool = False,
):
"""
Parameters
------------
pump_name: one of "media", "alt_media", "waste"
calibration:
specify a calibration for the dosing. Should be a dict
with fields "duration_", "hz", "dc", and "bias_"
Returns
-----------
Amount of volume passed (approximate in some cases)
"""
action_name = {
"media": "add_media",
"alt_media": "add_alt_media",
"waste": "remove_waste",
}[pump_name]
logger = create_logger(action_name)
with utils.publish_ready_to_disconnected_state(
unit, experiment, action_name
) as exit_event:
assert (
(ml is not None) or (duration is not None) or continuously
), "either ml or duration must be set"
assert not (
(ml is not None) and (duration is not None)
), "Only select ml or duration"
if calibration is None:
with utils.local_persistant_storage("pump_calibration") as cache:
try:
calibration = loads(cache[f"{pump_name}_ml_calibration"])
except KeyError:
logger.error("Calibration not defined. Run pump calibration first.")
return 0.0
try:
GPIO_PIN = PWM_TO_PIN[config.get("PWM_reverse", pump_name)]
except NoOptionError:
logger.error(f"Add `{pump_name}` to `PWM` section to config_{unit}.ini.")
return 0.0
if ml is not None:
assert ml >= 0, "ml should be greater than 0"
duration = utils.pump_ml_to_duration(
ml, calibration["duration_"], calibration["bias_"]
)
logger.info(f"{round(ml, 2)}mL")
elif duration is not None:
ml = utils.pump_duration_to_ml(
duration, calibration["duration_"], calibration["bias_"]
)
logger.info(f"{round(duration, 2)}s")
elif continuously:
duration = 600
ml = utils.pump_duration_to_ml(
duration, calibration["duration_"], calibration["bias_"]
)
logger.info("Running pump continuously.")
assert isinstance(ml, (float, int))
assert isinstance(duration, (float, int))
assert duration >= 0, "duration should be greater than 0"
if duration == 0:
return 0.0
# publish this first, as downstream jobs need to know about it.
json_output = dumps(
{
"volume_change": ml,
"event": action_name,
"source_of_event": source_of_event,
"timestamp": current_utc_time(),
}
)
publish(
f"pioreactor/{unit}/{experiment}/dosing_events",
json_output,
qos=QOS.EXACTLY_ONCE,
)
try:
pwm = PWM(GPIO_PIN, calibration["hz"])
pwm.lock()
with catchtime() as delta_time:
pwm.start(calibration["dc"])
pump_start_time = time.time()
exit_event.wait(max(0, duration - delta_time()))
if continuously:
while not exit_event.wait(duration):
publish(
f"pioreactor/{unit}/{experiment}/dosing_events",
json_output,
qos=QOS.EXACTLY_ONCE,
)
except SystemExit:
# a SigInt, SigKill occurred
pass
except Exception as e:
# some other unexpected error
logger.debug(e, exc_info=True)
logger.error(e)
finally:
pwm.stop()
pwm.cleanup()
if continuously:
logger.info(f"Stopping {pump_name} pump.")
if exit_event.is_set():
# ended early for some reason
shortened_duration = time.time() - pump_start_time
ml = utils.pump_duration_to_ml(
shortened_duration, calibration["duration_"], calibration["bias_"]
)
return ml
def subscribe(
topics: str | list[str],
hostname=leader_hostname,
retries: int = 10,
timeout: Optional[float] = None,
allow_retained: bool = True,
**mqtt_kwargs,
) -> Optional[MQTTMessage]:
"""
Modeled closely after the paho version, this also includes some try/excepts and
a timeout. Note that this _does_ disconnect after receiving a single message.
A failure case occurs if this is called in a thread (eg: a callback) and is waiting
indefinitely for a message. The parent job may not exit properly.
"""
retry_count = 1
for retry_count in range(retries):
try:
lock: Optional[threading.Lock]
def on_connect(client, userdata, flags, rc):
client.subscribe(userdata["topics"])
return
def on_message(client, userdata, message: MQTTMessage):
if not allow_retained and message.retain:
return
userdata["messages"] = message
client.disconnect()
if userdata["lock"]:
userdata["lock"].release()
return
if timeout:
lock = threading.Lock()
else:
lock = None
topics = [topics] if isinstance(topics, str) else topics
userdata: dict[str, Any] = {
"topics": [(topic, mqtt_kwargs.pop("qos", 0)) for topic in topics],
"messages": None,
"lock": lock,
}
client = Client(userdata=userdata)
client.on_connect = on_connect
client.on_message = on_message
client.connect(leader_hostname)
if timeout is None:
client.loop_forever()
else:
assert lock is not None
lock.acquire()
client.loop_start()
lock.acquire(timeout=timeout)
client.loop_stop()
client.disconnect()
return userdata["messages"]
except (ConnectionRefusedError, socket.gaierror, OSError, socket.timeout):
from pioreactor.logging import create_logger
logger = create_logger("pubsub.subscribe", to_mqtt=False)
logger.debug(
f"Attempt {retry_count}: Unable to connect to host: {hostname}",
exc_info=True,
)
time.sleep(5 * retry_count) # linear backoff
else:
logger = create_logger("pubsub.subscribe", to_mqtt=False)
logger.error(f"Unable to connect to host: {hostname}. Exiting.")
raise ConnectionRefusedError(f"Unable to connect to host: {hostname}.")
def backup_database(output_file: str) -> None:
"""
This action will create a backup of the SQLite3 database into specified output. It then
will try to copy the backup to any available worker Pioreactors as a further backup.
This job actually consumes _a lot_ of resources, and I've seen the LED output
drop due to this running. See issue #81. For now, we will skip the backup if `od_reading` is running
Elsewhere, a cronjob is set up as well to run this action every N days.
"""
import sqlite3
from sh import ErrorReturnCode, rsync # type: ignore
unit = get_unit_name()
experiment = UNIVERSAL_EXPERIMENT
with publish_ready_to_disconnected_state(unit, experiment,
"backup_database"):
logger = create_logger("backup_database",
experiment=experiment,
unit=unit)
if is_pio_job_running("od_reading"):
logger.warning("Won't run if OD Reading is running. Exiting")
return
def progress(status: int, remaining: int, total: int) -> None:
logger.debug(f"Copied {total-remaining} of {total} SQLite3 pages.")
logger.debug(f"Writing to local backup {output_file}.")
logger.debug(f"Starting backup of database to {output_file}")
sleep(
1
) # pause a second so the log entry above gets recorded into the DB.
con = sqlite3.connect(config.get("storage", "database"))
bck = sqlite3.connect(output_file)
with bck:
con.backup(bck, pages=-1, progress=progress)
bck.close()
con.close()
with local_persistant_storage("database_backups") as cache:
cache["latest_backup_timestamp"] = current_utc_time()
logger.info("Completed backup of database.")
n_backups = config.getint("number_of_backup_replicates_to_workers",
fallback=2)
backups_complete = 0
available_workers = list(get_active_workers_in_inventory())
while (backups_complete < n_backups) and (len(available_workers) > 0):
backup_unit = available_workers.pop()
if backup_unit == get_unit_name():
continue
try:
rsync(
"-hz",
"--partial",
"--inplace",
output_file,
f"{backup_unit}:{output_file}",
)
except ErrorReturnCode:
logger.debug(
f"Unable to backup database to {backup_unit}. Is it online?",
exc_info=True,
)
logger.warning(
f"Unable to backup database to {backup_unit}. Is it online?"
)
else:
logger.debug(
f"Backed up database to {backup_unit}:{output_file}.")
backups_complete += 1
return
def pump_calibration(min_duration: float, max_duration: float) -> None:
unit = get_unit_name()
experiment = get_latest_experiment_name()
logger = create_logger("pump_calibration",
unit=unit,
experiment=experiment)
logger.info("Starting pump calibration.")
with publish_ready_to_disconnected_state(unit, experiment,
"pump_calibration"):
click.clear()
click.echo()
pump_name, execute_pump = which_pump_are_you_calibrating()
hz, dc = choose_settings()
setup(pump_name, execute_pump, hz, dc)
durations, volumes = run_tests(execute_pump, hz, dc, min_duration,
max_duration)
(slope, std_slope), (
bias,
std_bias,
) = simple_linear_regression_with_forced_nil_intercept(
durations, volumes)
# check parameters for problems
if slope < 0:
logger.warning(
"Slope is negative - you probably want to rerun this calibration..."
)
if slope / std_slope < 5.0:
logger.warning(
"Too much uncertainty in slope - you probably want to rerun this calibration..."
)
# save to cache
with local_persistant_storage("pump_calibration") as cache:
cache[f"{pump_name}_ml_calibration"] = json.dumps({
"duration_":
slope,
"hz":
hz,
"dc":
dc,
"bias_":
bias,
"timestamp":
current_utc_time(),
})
cache[f"{pump_name}_calibration_data"] = json.dumps({
"timestamp":
current_utc_time(),
"data": {
"durations": durations,
"volumes": volumes
},
})
logger.debug(
f"slope={slope:0.2f} ± {std_slope:0.2f}, bias={bias:0.2f} ± {std_bias:0.2f}"
)
logger.debug(
f"Calibration is best for volumes between {(slope * min_duration + bias):0.1f}mL to {(slope * max_duration + bias):0.1f}mL, but will be okay for slightly outside this range too."
)
logger.info("Finished pump calibration.")
else:
try:
ip = socket.gethostbyname(hostname)
except OSError:
ip = "Unknown"
# get state
result = subscribe(
f"pioreactor/{hostname}/{UNIVERSAL_EXPERIMENT}/monitor/$state", timeout=1
)
if result:
state = result.payload.decode()
else:
state = "Unknown"
state = click.style(f"{state:15s}", fg="green" if state == "ready" else "red")
# is reachable?
reachable = networking.is_reachable(hostname)
click.echo(
f"{hostname:20s} {('Y' if hostname==get_leader_hostname() else 'N'):15s} {ip:20s} {state} {( click.style('Y', fg='green') if reachable else click.style('N', fg='red') ):10s}"
)
if not am_I_leader() and not am_I_active_worker():
logger = create_logger("CLI", unit=get_unit_name(), experiment=UNIVERSAL_EXPERIMENT)
logger.info(
f"Running `pio` on a non-active Pioreactor. Do you need to change `{get_unit_name()}` in `network.inventory` section in `config.ini`?"
)
def od_blank(
od_angle_channel1,
od_angle_channel2,
n_samples: int = 30,
):
"""
Compute the sample average of the photodiodes attached.
Note that because of the sensitivity of the growth rate (and normalized OD) to the starting values,
we need a very accurate estimate of these statistics.
"""
from statistics import mean, variance
action_name = "od_blank"
logger = create_logger(action_name)
unit = get_unit_name()
experiment = get_latest_experiment_name()
testing_experiment = get_latest_testing_experiment_name()
logger.info(
"Starting reading of blank OD. This will take about a few minutes.")
with publish_ready_to_disconnected_state(unit, experiment, action_name):
# running this will mess with OD Reading - best to just not let it happen.
if (is_pio_job_running("od_reading")
# but if test mode, ignore
and not is_testing_env()):
logger.error(
"od_reading should not be running. Stop od_reading first. Exiting."
)
return
# turn on stirring if not already on
if not is_pio_job_running("stirring"):
# start stirring
st = start_stirring(
target_rpm=config.getint("stirring", "target_rpm"),
unit=unit,
experiment=testing_experiment,
)
st.block_until_rpm_is_close_to_target()
else:
# TODO: it could be paused, we should make sure it's running
...
sampling_rate = 1 / config.getfloat("od_config", "samples_per_second")
# start od_reading
start_od_reading(
od_angle_channel1,
od_angle_channel2,
sampling_rate=sampling_rate,
unit=unit,
experiment=testing_experiment,
fake_data=is_testing_env(),
)
def yield_from_mqtt():
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{testing_experiment}/od_reading/od_raw_batched"
)
yield json.loads(msg.payload)
signal = yield_from_mqtt()
readings = defaultdict(list)
for count, batched_reading in enumerate(signal, start=1):
for (channel, reading) in batched_reading["od_raw"].items():
readings[channel].append(reading["voltage"])
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count // n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
if count == n_samples:
break
means = {}
variances = {}
autocorrelations = {} # lag 1
for channel, od_reading_series in readings.items():
# measure the mean and publish. The mean will be used to normalize the readings in downstream jobs
means[channel] = mean(od_reading_series)
variances[channel] = variance(od_reading_series)
autocorrelations[channel] = correlation(od_reading_series[:-1],
od_reading_series[1:])
# warn users that a blank is 0 - maybe this should be an error instead? TODO: link this to a docs page.
if means[channel] == 0.0:
logger.warning(
f"OD reading for PD Channel {channel} is 0.0 - that shouldn't be. Is there a loose connection, or an extra channel in the configuration's [od_config.photodiode_channel] section?"
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/od_blank/{channel}",
json.dumps({
"timestamp": current_utc_time(),
"od_reading_v": means[channel]
}),
)
# store locally as the source of truth.
with local_persistant_storage(action_name) as cache:
cache[experiment] = json.dumps(means)
# publish to UI and database
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/mean",
json.dumps(means),
qos=pubsub.QOS.AT_LEAST_ONCE,
retain=True,
)
if config.getboolean(
"data_sharing_with_pioreactor",
"send_od_statistics_to_Pioreactor",
fallback=False,
):
to_share = {"mean": means, "variance": variances}
to_share["ir_intensity"] = config["od_config"]["ir_intensity"]
to_share["od_angle_channel1"] = od_angle_channel1
to_share["od_angle_channel2"] = od_angle_channel2
pubsub.publish_to_pioreactor_cloud("od_blank_mean", json=to_share)
logger.debug(f"measured mean: {means}")
logger.debug(f"measured variances: {variances}")
logger.debug(f"measured autocorrelations: {autocorrelations}")
logger.debug("OD normalization finished.")
return means
def stirring_calibration(min_dc: int, max_dc: int) -> None:
unit = get_unit_name()
experiment = get_latest_testing_experiment_name()
action_name = "stirring_calibration"
logger = create_logger(action_name)
with publish_ready_to_disconnected_state(unit, experiment, action_name):
logger.info("Starting stirring calibration.")
if is_pio_job_running("stirring"):
logger.error(
"Make sure Stirring job is off before running stirring calibration. Exiting."
)
return
measured_rpms = []
# go up and down to observe any hystersis.
dcs = (list(range(max_dc, min_dc, -3)) +
list(range(min_dc, max_dc, 4)) +
list(range(max_dc, min_dc, -5)))
with stirring.RpmFromFrequency() as rpm_calc, stirring.Stirrer(
target_rpm=0,
unit=unit,
experiment=experiment,
rpm_calculator=None,
) as st:
st.duty_cycle = dcs[0]
st.start_stirring()
time.sleep(8)
n_samples = len(dcs)
for count, dc in enumerate(dcs, start=1):
st.set_duty_cycle(dc)
time.sleep(8)
rpm = rpm_calc(4)
measured_rpms.append(rpm)
logger.debug(f"Detected {rpm=} RPM @ {dc=}%")
# log progress
publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count / n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
publish_to_pioreactor_cloud(action_name,
json=dict(zip(dcs, measured_rpms)))
logger.debug(list(zip(dcs, measured_rpms)))
# drop any 0 in RPM, too little DC
try:
filtered_dcs, filtered_measured_rpms = zip(
*filter(lambda d: d[1] > 0, zip(dcs, measured_rpms)))
except ValueError:
# the above can fail if all measured rpms are 0
logger.error("No RPMs were measured. Is the stirring spinning?")
return
# since in practice, we want a look up from RPM -> required DC, we
# set x=measure_rpms, y=dcs
(rpm_coef, rpm_coef_std), (intercept,
intercept_std) = simple_linear_regression(
filtered_measured_rpms, filtered_dcs)
logger.debug(
f"{rpm_coef=}, {rpm_coef_std=}, {intercept=}, {intercept_std=}")
if rpm_coef <= 0:
logger.warning(
"Something went wrong - detected negative correlation between RPM and stirring."
)
return
if intercept <= 0:
logger.warning(
"Something went wrong - the intercept should be greater than 0."
)
return
with local_persistant_storage(action_name) as cache:
cache["linear_v1"] = json.dumps({
"rpm_coef": rpm_coef,
"intercept": intercept,
"timestamp": current_utc_time(),
})
cache["stirring_calibration_data"] = json.dumps({
"timestamp":
current_utc_time(),
"data": {
"dcs": dcs,
"measured_rpms": measured_rpms
},
})
def od_normalization(
unit: str,
experiment: str,
n_samples: int = 35
) -> tuple[dict[PdChannel, float], dict[PdChannel, float]]:
from statistics import mean, variance
action_name = "od_normalization"
logger = create_logger(action_name)
logger.debug("Starting OD normalization.")
with publish_ready_to_disconnected_state(unit, experiment, action_name):
if (not (is_pio_job_running("od_reading"))
# but if test mode, ignore
and not is_testing_env()):
logger.error(
" OD Reading should be running. Run OD Reading first. Exiting."
)
raise exc.JobRequiredError(
"OD Reading should be running. Run OD Reading first. Exiting.")
# TODO: write tests for this
def yield_from_mqtt() -> Generator[dict, None, None]:
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
allow_retained=False,
)
if msg is None:
continue
yield json.loads(msg.payload)
signal = yield_from_mqtt()
readings = defaultdict(list)
for count, batched_reading in enumerate(signal, start=1):
for (sensor, reading) in batched_reading["od_raw"].items():
readings[sensor].append(reading["voltage"])
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count // n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
if count == n_samples:
break
variances = {}
means = {}
autocorrelations = {} # lag 1
for sensor, od_reading_series in readings.items():
variances[sensor] = variance(
residuals_of_simple_linear_regression(list(
range(n_samples)), od_reading_series)) # see issue #206
means[sensor] = mean(od_reading_series)
autocorrelations[sensor] = correlation(od_reading_series[:-1],
od_reading_series[1:])
with local_persistant_storage("od_normalization_mean") as cache:
cache[experiment] = json.dumps(means)
with local_persistant_storage("od_normalization_variance") as cache:
cache[experiment] = json.dumps(variances)
logger.debug(f"measured mean: {means}")
logger.debug(f"measured variances: {variances}")
logger.debug(f"measured autocorrelations: {autocorrelations}")
logger.debug("OD normalization finished.")
if config.getboolean(
"data_sharing_with_pioreactor",
"send_od_statistics_to_Pioreactor",
fallback=False,
):
add_on = {
"ir_intensity": config["od_config"]["ir_intensity"],
}
pubsub.publish_to_pioreactor_cloud(
"od_normalization_variance",
json={
**variances,
**add_on,
}, # TODO: this syntax changed in a recent python version...
)
pubsub.publish_to_pioreactor_cloud(
"od_normalization_mean",
json={
**means,
**add_on
},
)
return means, variances
