def check_against_processes_running(msg: MQTTMessage) -> None:
job = msg.topic.split("/")[3]
if (msg.payload.decode() in [self.READY, self.INIT, self.SLEEPING
]) and (not is_pio_job_running(job)):
self.publish(
f"pioreactor/{self.unit}/{latest_exp}/{job}/$state",
self.LOST,
retain=True,
)
self.logger.debug(f"Manually changing {job} state in MQTT.")
def latest_growth_rate(self) -> float:
# check if None
if self._latest_growth_rate is None:
# this should really only happen on the initialization.
self.logger.debug("Waiting for OD and growth rate data to arrive")
if not is_pio_job_running("od_reading", "growth_rate_calculating"):
raise exc.JobRequiredError(
"`od_reading` and `growth_rate_calculating` should be running."
)
# check most stale time
if (time.time() - self.most_stale_time) > 5 * 60:
raise exc.JobRequiredError(
"readings are too stale (over 5 minutes old) - are `od_reading` and `growth_rate_calculating` running?"
)
return cast(float, self._latest_growth_rate)
def get_precomputed_values(self) -> tuple:
if self.ignore_cache:
if not is_pio_job_running("od_reading"):
self.logger.error("OD reading should be running. Stopping.")
raise exc.JobRequiredError("OD reading should be running. Stopping.")
self.logger.info(
"Computing OD normalization metrics. This may take a few minutes"
)
od_normalization_factors, od_variances = od_normalization(
unit=self.unit, experiment=self.experiment
)
self.logger.info("Completed OD normalization metrics.")
initial_growth_rate = 0.0
initial_od = 1.0
else:
od_normalization_factors = self.get_od_normalization_from_cache()
od_variances = self.get_od_variances_from_cache()
initial_growth_rate = self.get_growth_rate_from_cache()
initial_od = self.get_previous_od_from_cache()
initial_acc = 0
od_blank = self.get_od_blank_from_cache()
# what happens if od_blank is near / less than od_normalization_factors?
# this means that the inoculant had near 0 impact on the turbidity => very dilute.
# I think we should not use od_blank if so
for channel in od_normalization_factors.keys():
if od_normalization_factors[channel] * 0.95 < od_blank[channel]:
self.logger.debug(
"Resetting od_blank because it is too close to current observations."
)
od_blank[channel] = od_normalization_factors[channel] * 0.95
return (
initial_growth_rate,
initial_od,
od_normalization_factors,
od_variances,
od_blank,
initial_acc,
)
def click_self_test(k: str) -> int:
"""
Test the input/output in the Pioreactor
"""
import sys
unit = get_unit_name()
testing_experiment = get_latest_testing_experiment_name()
experiment = get_latest_experiment_name()
logger = create_logger("self_test", unit=unit, experiment=experiment)
with publish_ready_to_disconnected_state(unit, testing_experiment, "self_test"):
if is_pio_job_running("od_reading", "temperature_automation", "stirring"):
logger.error(
"Make sure Optical Density, Temperature Automation, and Stirring are off before running a self test. Exiting."
)
return 1
functions_to_test = [
(name, f)
for (name, f) in vars(sys.modules[__name__]).items()
if name.startswith("test_")
] # automagically finds the test_ functions.
if k:
functions_to_test = [
(name, f) for (name, f) in functions_to_test if (k in name)
]
# clear the mqtt cache
for name, _ in functions_to_test:
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/{name}",
None,
retain=True,
)
count_tested: int = 0
count_passed: int = 0
for name, test in functions_to_test:
try:
test(logger, unit, testing_experiment)
except Exception:
import traceback
traceback.print_exc()
res = False
else:
res = True
logger.debug(f"{name}: {'✅' if res else '❌'}")
count_tested += 1
count_passed += res
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/{name}",
int(res),
retain=True,
)
publish(
f"pioreactor/{unit}/{testing_experiment}/self_test/all_tests_passed",
int(count_passed == count_tested),
retain=True,
)
if count_passed == count_tested:
logger.info("All tests passed ✅")
else:
logger.info(
f"{count_tested-count_passed} failed test{'s' if (count_tested-count_passed) > 1 else ''}."
)
return int(count_passed != count_tested)
def backup_database(output_file: str) -> None:
"""
This action will create a backup of the SQLite3 database into specified output. It then
will try to copy the backup to any available worker Pioreactors as a further backup.
This job actually consumes _a lot_ of resources, and I've seen the LED output
drop due to this running. See issue #81. For now, we will skip the backup if `od_reading` is running
Elsewhere, a cronjob is set up as well to run this action every N days.
"""
import sqlite3
from sh import ErrorReturnCode, rsync # type: ignore
unit = get_unit_name()
experiment = UNIVERSAL_EXPERIMENT
with publish_ready_to_disconnected_state(unit, experiment,
"backup_database"):
logger = create_logger("backup_database",
experiment=experiment,
unit=unit)
if is_pio_job_running("od_reading"):
logger.warning("Won't run if OD Reading is running. Exiting")
return
def progress(status: int, remaining: int, total: int) -> None:
logger.debug(f"Copied {total-remaining} of {total} SQLite3 pages.")
logger.debug(f"Writing to local backup {output_file}.")
logger.debug(f"Starting backup of database to {output_file}")
sleep(
1
) # pause a second so the log entry above gets recorded into the DB.
con = sqlite3.connect(config.get("storage", "database"))
bck = sqlite3.connect(output_file)
with bck:
con.backup(bck, pages=-1, progress=progress)
bck.close()
con.close()
with local_persistant_storage("database_backups") as cache:
cache["latest_backup_timestamp"] = current_utc_time()
logger.info("Completed backup of database.")
n_backups = config.getint("number_of_backup_replicates_to_workers",
fallback=2)
backups_complete = 0
available_workers = list(get_active_workers_in_inventory())
while (backups_complete < n_backups) and (len(available_workers) > 0):
backup_unit = available_workers.pop()
if backup_unit == get_unit_name():
continue
try:
rsync(
"-hz",
"--partial",
"--inplace",
output_file,
f"{backup_unit}:{output_file}",
)
except ErrorReturnCode:
logger.debug(
f"Unable to backup database to {backup_unit}. Is it online?",
exc_info=True,
)
logger.warning(
f"Unable to backup database to {backup_unit}. Is it online?"
)
else:
logger.debug(
f"Backed up database to {backup_unit}:{output_file}.")
backups_complete += 1
return
def od_normalization(
unit: str,
experiment: str,
n_samples: int = 35
) -> tuple[dict[PdChannel, float], dict[PdChannel, float]]:
from statistics import mean, variance
action_name = "od_normalization"
logger = create_logger(action_name)
logger.debug("Starting OD normalization.")
with publish_ready_to_disconnected_state(unit, experiment, action_name):
if (not (is_pio_job_running("od_reading"))
# but if test mode, ignore
and not is_testing_env()):
logger.error(
" OD Reading should be running. Run OD Reading first. Exiting."
)
raise exc.JobRequiredError(
"OD Reading should be running. Run OD Reading first. Exiting.")
# TODO: write tests for this
def yield_from_mqtt() -> Generator[dict, None, None]:
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{experiment}/od_reading/od_raw_batched",
allow_retained=False,
)
if msg is None:
continue
yield json.loads(msg.payload)
signal = yield_from_mqtt()
readings = defaultdict(list)
for count, batched_reading in enumerate(signal, start=1):
for (sensor, reading) in batched_reading["od_raw"].items():
readings[sensor].append(reading["voltage"])
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count // n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
if count == n_samples:
break
variances = {}
means = {}
autocorrelations = {} # lag 1
for sensor, od_reading_series in readings.items():
variances[sensor] = variance(
residuals_of_simple_linear_regression(list(
range(n_samples)), od_reading_series)) # see issue #206
means[sensor] = mean(od_reading_series)
autocorrelations[sensor] = correlation(od_reading_series[:-1],
od_reading_series[1:])
with local_persistant_storage("od_normalization_mean") as cache:
cache[experiment] = json.dumps(means)
with local_persistant_storage("od_normalization_variance") as cache:
cache[experiment] = json.dumps(variances)
logger.debug(f"measured mean: {means}")
logger.debug(f"measured variances: {variances}")
logger.debug(f"measured autocorrelations: {autocorrelations}")
logger.debug("OD normalization finished.")
if config.getboolean(
"data_sharing_with_pioreactor",
"send_od_statistics_to_Pioreactor",
fallback=False,
):
add_on = {
"ir_intensity": config["od_config"]["ir_intensity"],
}
pubsub.publish_to_pioreactor_cloud(
"od_normalization_variance",
json={
**variances,
**add_on,
}, # TODO: this syntax changed in a recent python version...
)
pubsub.publish_to_pioreactor_cloud(
"od_normalization_mean",
json={
**means,
**add_on
},
)
return means, variances
def od_blank(
od_angle_channel1,
od_angle_channel2,
n_samples: int = 30,
):
"""
Compute the sample average of the photodiodes attached.
Note that because of the sensitivity of the growth rate (and normalized OD) to the starting values,
we need a very accurate estimate of these statistics.
"""
from statistics import mean, variance
action_name = "od_blank"
logger = create_logger(action_name)
unit = get_unit_name()
experiment = get_latest_experiment_name()
testing_experiment = get_latest_testing_experiment_name()
logger.info(
"Starting reading of blank OD. This will take about a few minutes.")
with publish_ready_to_disconnected_state(unit, experiment, action_name):
# running this will mess with OD Reading - best to just not let it happen.
if (is_pio_job_running("od_reading")
# but if test mode, ignore
and not is_testing_env()):
logger.error(
"od_reading should not be running. Stop od_reading first. Exiting."
)
return
# turn on stirring if not already on
if not is_pio_job_running("stirring"):
# start stirring
st = start_stirring(
target_rpm=config.getint("stirring", "target_rpm"),
unit=unit,
experiment=testing_experiment,
)
st.block_until_rpm_is_close_to_target()
else:
# TODO: it could be paused, we should make sure it's running
...
sampling_rate = 1 / config.getfloat("od_config", "samples_per_second")
# start od_reading
start_od_reading(
od_angle_channel1,
od_angle_channel2,
sampling_rate=sampling_rate,
unit=unit,
experiment=testing_experiment,
fake_data=is_testing_env(),
)
def yield_from_mqtt():
while True:
msg = pubsub.subscribe(
f"pioreactor/{unit}/{testing_experiment}/od_reading/od_raw_batched"
)
yield json.loads(msg.payload)
signal = yield_from_mqtt()
readings = defaultdict(list)
for count, batched_reading in enumerate(signal, start=1):
for (channel, reading) in batched_reading["od_raw"].items():
readings[channel].append(reading["voltage"])
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count // n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
if count == n_samples:
break
means = {}
variances = {}
autocorrelations = {} # lag 1
for channel, od_reading_series in readings.items():
# measure the mean and publish. The mean will be used to normalize the readings in downstream jobs
means[channel] = mean(od_reading_series)
variances[channel] = variance(od_reading_series)
autocorrelations[channel] = correlation(od_reading_series[:-1],
od_reading_series[1:])
# warn users that a blank is 0 - maybe this should be an error instead? TODO: link this to a docs page.
if means[channel] == 0.0:
logger.warning(
f"OD reading for PD Channel {channel} is 0.0 - that shouldn't be. Is there a loose connection, or an extra channel in the configuration's [od_config.photodiode_channel] section?"
)
pubsub.publish(
f"pioreactor/{unit}/{experiment}/od_blank/{channel}",
json.dumps({
"timestamp": current_utc_time(),
"od_reading_v": means[channel]
}),
)
# store locally as the source of truth.
with local_persistant_storage(action_name) as cache:
cache[experiment] = json.dumps(means)
# publish to UI and database
pubsub.publish(
f"pioreactor/{unit}/{experiment}/{action_name}/mean",
json.dumps(means),
qos=pubsub.QOS.AT_LEAST_ONCE,
retain=True,
)
if config.getboolean(
"data_sharing_with_pioreactor",
"send_od_statistics_to_Pioreactor",
fallback=False,
):
to_share = {"mean": means, "variance": variances}
to_share["ir_intensity"] = config["od_config"]["ir_intensity"]
to_share["od_angle_channel1"] = od_angle_channel1
to_share["od_angle_channel2"] = od_angle_channel2
pubsub.publish_to_pioreactor_cloud("od_blank_mean", json=to_share)
logger.debug(f"measured mean: {means}")
logger.debug(f"measured variances: {variances}")
logger.debug(f"measured autocorrelations: {autocorrelations}")
logger.debug("OD normalization finished.")
return means
def stirring_calibration(min_dc: int, max_dc: int) -> None:
unit = get_unit_name()
experiment = get_latest_testing_experiment_name()
action_name = "stirring_calibration"
logger = create_logger(action_name)
with publish_ready_to_disconnected_state(unit, experiment, action_name):
logger.info("Starting stirring calibration.")
if is_pio_job_running("stirring"):
logger.error(
"Make sure Stirring job is off before running stirring calibration. Exiting."
)
return
measured_rpms = []
# go up and down to observe any hystersis.
dcs = (list(range(max_dc, min_dc, -3)) +
list(range(min_dc, max_dc, 4)) +
list(range(max_dc, min_dc, -5)))
with stirring.RpmFromFrequency() as rpm_calc, stirring.Stirrer(
target_rpm=0,
unit=unit,
experiment=experiment,
rpm_calculator=None,
) as st:
st.duty_cycle = dcs[0]
st.start_stirring()
time.sleep(8)
n_samples = len(dcs)
for count, dc in enumerate(dcs, start=1):
st.set_duty_cycle(dc)
time.sleep(8)
rpm = rpm_calc(4)
measured_rpms.append(rpm)
logger.debug(f"Detected {rpm=} RPM @ {dc=}%")
# log progress
publish(
f"pioreactor/{unit}/{experiment}/{action_name}/percent_progress",
count / n_samples * 100,
)
logger.debug(f"Progress: {count/n_samples:.0%}")
publish_to_pioreactor_cloud(action_name,
json=dict(zip(dcs, measured_rpms)))
logger.debug(list(zip(dcs, measured_rpms)))
# drop any 0 in RPM, too little DC
try:
filtered_dcs, filtered_measured_rpms = zip(
*filter(lambda d: d[1] > 0, zip(dcs, measured_rpms)))
except ValueError:
# the above can fail if all measured rpms are 0
logger.error("No RPMs were measured. Is the stirring spinning?")
return
# since in practice, we want a look up from RPM -> required DC, we
# set x=measure_rpms, y=dcs
(rpm_coef, rpm_coef_std), (intercept,
intercept_std) = simple_linear_regression(
filtered_measured_rpms, filtered_dcs)
logger.debug(
f"{rpm_coef=}, {rpm_coef_std=}, {intercept=}, {intercept_std=}")
if rpm_coef <= 0:
logger.warning(
"Something went wrong - detected negative correlation between RPM and stirring."
)
return
if intercept <= 0:
logger.warning(
"Something went wrong - the intercept should be greater than 0."
)
return
with local_persistant_storage(action_name) as cache:
cache["linear_v1"] = json.dumps({
"rpm_coef": rpm_coef,
"intercept": intercept,
"timestamp": current_utc_time(),
})
cache["stirring_calibration_data"] = json.dumps({
"timestamp":
current_utc_time(),
"data": {
"dcs": dcs,
"measured_rpms": measured_rpms
},
})