def _choose_and_ingest_run(self,
analysis_phase: str = "",
run_spec: dict = None):
if run_spec is not None:
run_rep = [None, None]
if "key" in run_spec:
run_key = run_spec["key"]
for run in getattr(self, analysis_phase + "_runs"):
if run[0].config.name == run_spec["key"]:
run_rep[0] = run[0]
else:
run_rep[0] = run_spec["run"]
cur_rep = run_spec["rep"]
elif self.sample_selection_mode == SampleSelectionMode.Random:
run_rep = random.choice(getattr(self, analysis_phase + "_runs"))
cur_rep = random.randint(0, run_rep[1] - 1)
elif self.sample_selection_mode == SampleSelectionMode.Alternating:
min_runs = np.where(
self._sample_selection[
self._prev_run,
self._sample_lists[self._sample_lists_selector]] ==
self._sample_selection[
self._prev_run,
self._sample_lists[self._sample_lists_selector]].min())[0]
min_runs = self._sample_lists[
self._sample_lists_selector][min_runs]
self._sample_lists_selector = not self._sample_lists_selector
elif self.sample_selection_mode == SampleSelectionMode.Uniform:
min_runs = np.where(
self._sample_selection[self._prev_run, :] ==
self._sample_selection[self._prev_run, :].min())[0]
if (self.sample_selection_mode == SampleSelectionMode.Alternating
or self.sample_selection_mode
== SampleSelectionMode.Uniform) and (run_spec is None):
run_idx = min_runs[self._sample_selection[min_runs, :].sum(
axis=1).argmin()]
self._sample_selection[self._prev_run, run_idx] += 1
self._prev_run = run_idx
run_rep = getattr(self, analysis_phase + "_runs")[run_idx]
cur_rep = random.choice(
np.where(self._repetition_selection[run_idx] ==
self._repetition_selection[run_idx].min())[0])
cur_run = run_rep[0]
ingest = False
try:
cur_run.load_ingestion_data(**self.ingest_args(
cur_rep, analysis_phase),
prefix=self.name + "_")
except:
ingest = True
if ingest:
get_root_logger().debug(
f"No ingestion data present, ingesting run {cur_run}")
cur_run.ingest(**self.ingest_args(cur_rep, analysis_phase))
cur_run.save_ingestion_data(prefix=self.name + "_")
return cur_run
def fan(self) -> t.Optional[t.Union[str, tuple]]:
"""Gets fan settings
Returns:
t.Optional[t.Union[str, tuple]]: The fan settings
"""
fan = self.fan_path
if not fan:
get_root_logger().warning(
"setting fan failed due to no fan path found")
return None
if fan["key"] == "thinkpad":
_ = self.backend.run(["cat", fan["path"], "|", "grep", "'^level'"])
return _.stdout.split(":")[1].strip() if _.ok else None
elif fan["key"] == "odroid-xu3" or fan["key"] == "odroid-2":
_ = self.backend.run(["cat", fan["path"] + "{fan_mode,pwm_duty}"])
return tuple(_.stdout.splitlines()) if _.ok else None
elif fan["key"] == "pwm-fan-hwmon":
_ = self.backend.run(["cat", fan["path"] + "{automatic,pwm1}"])
return tuple(_.stdout.splitlines()) if _.ok else None
elif fan["key"] == "pwm-fan":
_ = self.backend.run(["cat", fan["path"] + "cur_pwm"])
return _.stdout.strip() if _.ok else None
def reboot_device(self):
get_root_logger().info(
"Rebooting device, this will take roughly 2 minutes...")
self.backend.sudo("reboot")
sleep(
90
) # 1.5 minutes security wait time until the device is back online
def latency(self, value: t.Optional[int]) -> None:
existing_pgid = getattr(self, "_latency_setter_pgid", None)
if existing_pgid:
# must kill whole process group, the pgid of children processes
# should be the same as calling parent
if not self.unix_kill(existing_pgid, pgid=True):
_ = self.backend.history[-1]
get_root_logger().warning(
f"failed to kill CPU DMA setter, exit code {_.exited}, "
f"stderr {_.stderr!r}")
else:
delattr(self, "_latency_setter_pgid")
# setting to None will just kill the existing dma latency setter
if value is None:
return
elif not isinstance(value, int):
raise TypeError("latency accepts integer values")
# Accepts bit-endian values as byte strings or hex values (without 0x prefix)
value = value.to_bytes(4, "big").hex()
# CPU DMA latency will be set as long as a file descriptor is held, the value
# has to be written only once.
inner = "exec 4> /dev/cpu_dma_latency; echo -ne {} >&4; sleep 99999".format(
str(value))
outer = ["sudo", "/usr/bin/env", "bash", "-c", inner]
_ = self.persistent(outer)
self._latency_setter_pgid = _.pgid
def _adb_device_wrapper(self, cmd: Backend.CommandT, **kwargs):
if not isinstance(cmd, (str, t.List)):
raise TypeError(f"'cmd' ({type(cmd)}) not str or list")
ip, port = (kwargs.pop("ip", self.config.ip),
kwargs.pop("port", self.config.port))
is_usb = self._is_usb(ip)
pre = ["-s", f"{ip}:{port}" if not is_usb else ip]
if isinstance(cmd, t.List):
cmd = pre + cmd
else:
cmd = " ".join([list2cmdline(pre), cmd])
if not is_usb:
def connection_is_ok():
return self._adb_wrapper(pre + ["shell", "pwd"],
history=False).ok
if not connection_is_ok():
get_root_logger().critical(
f"TCP/IP ADB connection to {ip}:{port} went offline, reconnecting..."
)
# Make sure the adb server is running
self._spawn_server()
# Try to reconnect using exponentiall back-off waiting time
max_retries = 5
for retries in range(1, max_retries):
sleep(2**(retries) - 1)
self._adb_wrapper(["connect", f"{ip}:{port}"],
history=False)
if connection_is_ok():
break
else:
# Try to restart the adb connection. Sometimes even if the connection seems not ok
# we can still access the device as adb is simply messed up but not fully broken.
self._adb_wrapper([
"shell", "su", "-c", "setprop", "ctl.restart",
"adbd"
])
if retries == (max_retries - 1):
# As a last resort, try to reboot the phone
sleep(2)
self._adb_wrapper(["shell", "su", "-c", "reboot"])
sleep(
60 * 5
) # Wait for five minutes until the phone is back up
if not connection_is_ok():
raise BackendRuntimeError(
f"Could not re-establish TCP/IP ADB connection to {ip}:{port} after "
f"{retries} retries.")
return self._adb_wrapper(cmd, **kwargs)
def write_dataset(self) -> None:
"""Serialise a Dataset"""
self.save_mapping("train_set", path=self.path_dataset, prefix="train_")
self.save_mapping("test_set", path=self.path_dataset, prefix="test_")
self.save_mapping("verification_set",
path=self.path_dataset,
prefix="verif_")
with self.path_dataset.joinpath("_parameters.toml").open(
"w") as parameterfile:
toml.dump(self.dataset_parameters_to_dict(), parameterfile)
get_root_logger().debug(f"serialised dataset to '{self.path_dataset}'")
def dataset_parameters_from_dict(self, param_dict: dict) -> None:
not_set = list()
for key in param_dict:
try:
setattr(self, key, param_dict[key])
except:
# This might happen as some attributes don't have a setter. No big deal, but the developer should be informed.
not_set.append(key)
if len(not_set) > 0:
get_root_logger().warning(
f"Following attributes could not be set: {not_set}")
def can_connect(self) -> bool:
if not self.configured:
get_root_logger().critical("Driver not configured!")
return False
try:
self.keyfile = self.config.key
except MisconfiguredError:
get_root_logger().critical("Could not find key file " +
str(self.config.key))
return False
return True
def remove_dataset(self) -> None:
"""Serialise a Dataset"""
self.remove_mapping("train_set",
path=self.path_dataset,
prefix="train_")
self.remove_mapping("test_set", path=self.path_dataset, prefix="test_")
self.remove_mapping("verification_set",
path=self.path_dataset,
prefix="verif_")
if self.path_dataset.joinpath("_parameters.toml").exists():
os.remove(self.path_dataset.joinpath("_parameters.toml"))
get_root_logger().debug(
f"serialised dataset from path '{self.path_dataset}' removed")
def _stop_server(self) -> None:
# The client can kill the server regardless of its location
self._adb_wrapper(["kill-server"])
self._spawned = False
for adb_file in [
self.adb_params[_] for _ in ["key", "stdout", "stderr"]
]:
if self.gateway:
self.gateway.run("rm -f {}".format(quote(adb_file)))
else:
try:
delete(Path(os.path.expandvars(adb_file)))
except:
get_root_logger().warning(f"Could not delete {adb_file}")
def read_dataset(self) -> None:
"""Deserialise a Dataset"""
train_set = self.load_mapping(path=self.path_dataset, prefix="train_")
for key in train_set:
setattr(self, "train_" + key, train_set[key])
test_set = self.load_mapping(path=self.path_dataset, prefix="test_")
for key in test_set:
setattr(self, "test_" + key, test_set[key])
verif_set = self.load_mapping(path=self.path_dataset, prefix="verif_")
for key in verif_set:
setattr(self, "verif_" + key, verif_set[key])
with self.path_dataset.joinpath("_parameters.toml").open(
"r") as parameterfile:
param_dict = toml.load(parameterfile)
self.dataset_parameters_from_dict(param_dict)
get_root_logger().debug(f"serialised dataset to '{self.path_dataset}'")
def _query_state_framework(self) -> EXOT_APP_STATE:
"""Queries the state of a framework app process
Returns:
EXOT_APP_STATE: The process state
"""
query_intent = Intent(COMPONENT_NAME=self.component,
ACTION=EXOT_APP_ACTIONS["query"])
intent_time = self.driver.get_time()
self.driver.send_intent(query_intent)
logs = self.driver.get_logs(grep=r"handleActionQuery\(\): (\S+)",
since=intent_time)
if len(set(logs)) > 1:
get_root_logger().warning(
f"logs returned more than one state query output: {set(logs)}")
return EXOT_APP_STATE(logs[0]) if logs else EXOT_APP_STATE.INVALID
def can_connect(self) -> bool:
if self.configured:
# Will throw if the key is not available
self.keyfile = self.config.key
self._local_adb_version = self._get_adb_version()
if "gateway" in self.config:
try:
self._gateway = fabric.Connection(
os.path.expandvars(self.config.gateway),
config=self._fabric_config)
if self.gateway.run("pwd").exited != 0:
get_root_logger().critical("Gateway connection failed")
return False
self._remote_adb_version = self._get_adb_version(
self.gateway)
if self._local_adb_version != self._remote_adb_version:
get_root_logger().critical(
("The ADB version of the gateway server {} does "
"not match the client version {}!").format(
self._remote_adb_version,
self._local_adb_version))
return False
except Exception as e:
get_root_logger().critical("Couldn't connect to gateway",
e.args)
return False
return True
else:
return False
def _execute_handler(self, *args, **kwargs):
"""Prepare train and test samples."""
get_root_logger().info(
"Start preparation of the train, test and verification samples")
self.cleanup()
for phase in ["train", "test", "verif"]:
if getattr(self, "augmentation_" + phase):
get_root_logger().info(
"Using data augmenetation for phase %s" % phase)
self._prepare_with_augmentation(phase)
else:
get_root_logger().info(
"Not using data augmenetation for phase %s" % phase)
self._prepare_without_augmentation(phase)
get_root_logger().info(
"Finished preparation of the train, test and verification samples")
def bootstrap_analysis(self, analysis_name: str):
if "ANALYSES" in self.parent.config:
if analysis_name in self.parent.config.ANALYSES:
kwargs = self.parent.config.ANALYSES[analysis_name].copy()
kwargs.update(name=analysis_name)
args = [self.parent]
if "type" in kwargs:
if kwargs["type"] in self.analyses_classes:
new_analysis = self.analyses_classes[kwargs["type"]](*args, **kwargs)
self.analyses[new_analysis.name] = new_analysis
else:
get_root_logger().critical(
f"Analysis of type %s not available for channel {type(self)}"
% (kwargs["type"])
)
else:
get_root_logger().critical(
f"type not specified in config for analysis {analysis_name}"
)
else:
get_root_logger().critical(
f"Analysis {analysis_name} not specified in configuration!"
)
def bootstrap_analyses(self):
if "ANALYSES" in self.parent.config:
for analysis in self.parent.config.ANALYSES:
self.bootstrap_analysis(analysis)
else:
get_root_logger().info("No analyses specified")
def verify(self, best_model=None):
if best_model is not None:
self.checkpoint.restore(best_model)
get_root_logger().info("Restored model from " + str(best_model))
if self.verif_X is None:
get_root_logger().info("No Inference data, nothing to do...")
else:
(
pred,
Y,
X,
weights,
loss,
accuracy,
lev_abs,
lev_norm,
lev_len,
timing,
) = self.inference(train=False, epoch=None, debug=True)
overall_loss = np.nansum(loss)
overall_accuracy = np.nanmean(accuracy)
overall_lev_abs = np.nansum(lev_abs)
overall_lev_norm = np.nanmean(lev_norm)
overall_lev_len = np.nansum(lev_len)
overall_timing = np.nanmean(timing)
get_root_logger().info(
"Verification | loss: %03.3f | accuracy: %2.2f%% | lev_abs: %4i | lev_len: %4i | lev_norm: %.3f | timing: %.3f"
% (
overall_loss,
overall_accuracy,
overall_lev_abs,
overall_lev_len,
overall_lev_norm,
overall_timing,
)
)
self.write_debug_files(phase="verif", outputs=pred, prefix="prediction")
self.write_debug_files(phase="verif", outputs=Y, prefix="Y")
self.write_debug_files(phase="verif", outputs=self.test_set, prefix="")
self.write_debug_files(
phase="verif", outputs={"X": X, "weights": weights}, prefix="data"
)
self.write_debug_files(
phase="verif",
outputs={
"accuracy": accuracy,
"lev_abs": lev_abs,
"lev_norm": lev_norm,
"lev_len": lev_len,
"timing": timing,
"results": {
"loss": overall_loss,
"accuracy": overall_accuracy,
"lev_abs": overall_lev_abs,
"lev_norm": overall_lev_norm,
"timing": overall_timing,
},
},
prefix="metric",
as_txt=True,
)
self.checkpoint.save(file_prefix=self.path.joinpath("verif").joinpath("tf_ckpt"))
get_root_logger().info(
"Inference finished, saved files to " + str(self.path.joinpath("verif"))
)
def train(self):
get_root_logger().info(
"Epoch X | loss: (TRAIN/TEST) | accuracy: (TRAIN/TEST) | lev_abs: (TRAIN/TEST) | lev_len: (TRAIN/TEST) | lev_norm: (TRAIN/TEST) | timing: (TRAIN/TEST)"
)
self.best_lev_norm = np.inf
self.best_loss = np.inf
best_model = None
early_stop = False
trace_loss = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
trace_accuracy = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
trace_lev_abs = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
trace_lev_norm = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
trace_lev_len = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
trace_timing = {
"train": np.empty((math.ceil(self.n_epochs / self.debug_step))),
"test": np.empty((math.ceil(self.n_epochs / self.debug_step))),
}
loss = {"train": None, "test": None}
accuracy = {"train": None, "test": None}
lev_abs = {"train": None, "test": None}
lev_norm = {"train": None, "test": None}
lev_len = {"train": None, "test": None}
timing = {"train": None, "test": None}
pred = {"train": None, "test": None}
Y = {"train": None, "test": None}
X = {"train": None, "test": None}
weights = {"train": None, "test": None}
dbg_cnt = 0
for epoch in range(self.start_epoch + 1, self.n_epochs + 1):
appdetector_state = self.appdetector.initial_hidden_state
# Training
self.reshuffle_data("train", epoch)
debug = (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1
(
pred["train"],
Y["train"],
X["train"],
weights["train"],
loss["train"],
accuracy["train"],
lev_abs["train"],
lev_norm["train"],
lev_len["train"],
timing["train"],
) = self.inference(epoch, train=True, debug=debug)
if (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1:
(
pred["test"],
Y["test"],
X["test"],
weights["test"],
loss["test"],
accuracy["test"],
lev_abs["test"],
lev_norm["test"],
lev_len["test"],
timing["test"],
) = self.inference(
epoch, train=False, debug=debug
) # Test inference
for phase in ["train", "test"]:
trace_loss[phase][dbg_cnt] = np.nansum(loss[phase])
trace_accuracy[phase][dbg_cnt] = np.nanmean(accuracy[phase])
trace_lev_abs[phase][dbg_cnt] = np.nansum(lev_abs[phase])
trace_lev_norm[phase][dbg_cnt] = np.nanmean(lev_norm[phase])
trace_lev_len[phase][dbg_cnt] = np.nansum(lev_len[phase])
trace_timing[phase][dbg_cnt] = np.nanmean(timing[phase])
get_root_logger().info(
"Epoch %03i | loss: (%03.3f/%03.3f) | accuracy: (%2.2f%%/%2.2f%%) | lev_abs: (%4i/%4i) | lev_len: (%4i/%4i) | lev_norm: (%.3f/%.3f) | timing: (%.3f/%.3f)"
% (
epoch,
trace_loss["train"][dbg_cnt],
trace_loss["test"][dbg_cnt],
trace_accuracy["train"][dbg_cnt],
trace_accuracy["test"][dbg_cnt],
trace_lev_abs["train"][dbg_cnt],
trace_lev_abs["test"][dbg_cnt],
trace_lev_len["train"][dbg_cnt],
trace_lev_len["test"][dbg_cnt],
trace_lev_norm["train"][dbg_cnt],
trace_lev_norm["test"][dbg_cnt],
trace_timing["train"][dbg_cnt],
trace_timing["test"][dbg_cnt],
)
)
if trace_loss["test"][dbg_cnt] == 0.0:
train_violations = self.early_stopping_threshold + 1
if trace_loss["test"][dbg_cnt] < self.best_loss:
train_violations = 0
self.best_loss = trace_loss["test"][dbg_cnt]
for phase in ["train", "test"]:
self.write_debug_files(
phase=phase, outputs=pred[phase], prefix="prediction"
)
self.write_debug_files(phase=phase, outputs=Y[phase], prefix="Y")
self.write_debug_files(
phase=phase,
outputs={"X": X[phase], "weights": weights[phase]},
prefix="data",
)
self.write_debug_files(
phase=phase,
outputs={
"accuracy": accuracy[phase],
"lev_abs": lev_abs[phase],
"lev_norm": lev_norm[phase],
"lev_len": lev_len[phase],
"timing": timing[phase],
"results": {
"epoch": epoch,
"loss": trace_loss[phase][dbg_cnt],
"accuracy": trace_accuracy[phase][dbg_cnt],
"lev_abs": trace_lev_abs[phase][dbg_cnt],
"lev_norm": trace_lev_norm[phase][dbg_cnt],
"timing": trace_timing[phase][dbg_cnt],
},
},
prefix="metric",
as_txt=True,
)
best_model = self.checkpoint.save(file_prefix=self.path.joinpath("tf_ckpt"))
self.config.MODEL["best_model"] = best_model
with self.path.joinpath("model.toml").open("w") as toml_file:
toml.dump(self.config.MODEL, toml_file)
else:
train_violations += self.debug_step if epoch >= self.debug_step else 1
dbg_cnt += 1
if train_violations >= self.early_stopping_threshold:
if not early_stop:
new_lr = self.learning_rate / 10
get_root_logger().info(
"Starting second phase of training at %i epochs with learning rate %f!"
% (epoch, new_lr)
)
get_root_logger().info(
"Restoring previously best model from %s!" % str(best_model)
)
self.checkpoint.restore(best_model)
config = self.optimizer.get_config()
config["learning_rate"] = self.learning_rate / 10
self.optimizer = self.optimizer.from_config(config)
early_stop = True
train_violations = 0
else:
get_root_logger().info("Early stopping triggered after %i epochs!" % epoch)
break
for phase in ["train", "test"]:
self.write_debug_files(
phase=phase,
outputs={
"loss": trace_loss[phase],
"accuracy": trace_accuracy[phase],
"lev_abs": trace_lev_abs[phase],
"lev_norm": trace_lev_norm[phase],
"lev_len": trace_lev_len[phase],
"timing": trace_timing[phase],
},
prefix="trace",
as_txt=True,
)
get_root_logger().info(
"Training finished, saved files to "
+ str(self.path.joinpath("train"))
+ " and "
+ str(self.path.joinpath("test"))
)
return best_model
def _prepare_without_augmentation(self, analysis_phase: str):
"""
"""
get_root_logger().info(
"Starting data preparation without augmentation for phase %s" %
analysis_phase)
cnt_batch = 0
if analysis_phase not in ["train", "test", "verif"]:
raise Exception(f"Unknown analysis phase {analysis_phase}")
setattr(self.config.DATASET, "num_" + analysis_phase + "_batches", 0)
reshape_X = list(getattr(self, analysis_phase + "_X_shape"))
reshape_X[0] = 0
reshape_Y = list(getattr(self, analysis_phase + "_Y_shape"))
reshape_Y[0] = 0
X = np.zeros(reshape_X)
Y = np.full(reshape_Y,
self.label_mapping["UNKNOWN"]["int"] + 1,
dtype=np.int32)
actual_batch_len = list()
if analysis_phase != "verif":
weights = np.zeros(reshape_Y)
histlabels = np.zeros((self.num_labels, ))
for run_rep in getattr(self, analysis_phase + "_runs"):
for cur_rep in range(run_rep[1]):
cur_run = self._choose_and_ingest_run(analysis_phase, {
"run": run_rep[0],
"rep": cur_rep
})
_, num_batches = math.modf(cur_run.i_symstream.shape[0] /
self.batch_size_timesteps)
num_batches = int(num_batches) + 1
for batch_idx in range(num_batches):
idx_start = min([
(batch_idx + 0) * self.batch_size_timesteps,
cur_run.i_symstream.shape[0],
])
idx_end = min([
(batch_idx + 1) * self.batch_size_timesteps,
cur_run.i_symstream.shape[0],
])
x_shape = list(X.shape)
x_shape[0] = 1
y_shape = list(Y.shape)
y_shape[0] = 1
X = np.vstack([X, np.zeros(x_shape)])
Y = np.vstack([Y, np.zeros(y_shape)])
X[-1, :(idx_end - idx_start), :] = (
cur_run.i_symstream[idx_start:idx_end,
self.data_columns_idxes] +
self.T_meas_ambient)
Y[-1, :(idx_end - idx_start), :] = cur_run.i_symstream[
idx_start:idx_end, self.label_columns_idxes]
# Set length of each iteration sample
actual_batch_len.append(
cur_run.i_symstream[idx_start:idx_end,
self.data_columns_idxes].shape[0])
if analysis_phase != "verif":
weights = np.vstack([weights, np.zeros(y_shape)])
for cur_lbl_col in range(Y.shape[-1]):
hist, _ = np.histogram(
Y[batch_idx, :, cur_lbl_col],
bins=np.array(list(range(self.num_labels + 1)))
- 0.5,
density=False,
)
histlabels[:] += hist
get_root_logger().debug(
f"Processing run {cur_run} finished, processed {num_batches}")
actual_batch_len = np.array(actual_batch_len).reshape((-1, 1))
if analysis_phase == "verif":
self.num_verif_batches = X.shape[0]
setattr(self, "verif_X", X)
setattr(self, "verif_Y", Y.astype(int))
setattr(self, "verif_actual_batch_len", actual_batch_len)
else:
per_label_weights = self.compute_weights(
histlabels, mode=WeightCalculationMode.Proportional)
for label in per_label_weights:
weights[np.where(Y == label)] = per_label_weights[label]
Y[Y == self.label_mapping["UNKNOWN"]["int"] + 1] = 0
setattr(self, "num_" + analysis_phase + "_batches", X.shape[0])
shuffled_idxes = np.array(
shuffle(
list(
range(
getattr(self,
"num_" + analysis_phase + "_batches"))),
random_state=random.randint(
0,
len(getattr(self, analysis_phase + "_runs")) - 1),
),
dtype=int,
)
setattr(self, analysis_phase + "_X", X[shuffled_idxes])
setattr(self, analysis_phase + "_Y", Y[shuffled_idxes].astype(int))
setattr(self, analysis_phase + "_actual_batch_len",
actual_batch_len[shuffled_idxes, :])
setattr(self, analysis_phase + "_weights", weights[shuffled_idxes])
def _prepare_with_augmentation(self, analysis_phase: str):
"""
generate self.samples and split into training and test data
0) Init empty stream
1) Read stream
2) Resample and normalise stream
3) Augment and append stream
4) Repeat 1-3 until sample is finished
5) Repeat 0-4 until enough self.samples have been initialised
"""
# --------------------------------------------------------------------------------------------------
get_root_logger().info(
"Starting data preparation with augmentation for phase %s" %
analysis_phase)
sample_timestamps = np.linspace(
0,
(self.batch_size_timesteps - 1) * self.resampling_period_s,
self.batch_size_timesteps,
)
data_col_idx = range(1, min(self.label_columns_idxes))
self._init_sample_selection(analysis_phase)
X = np.zeros(getattr(self, analysis_phase + "_X_shape"))
Y = np.full(
getattr(self, analysis_phase + "_Y_shape"),
self.label_mapping["UNKNOWN"]["int"] + 1,
dtype=np.int32,
)
actual_batch_len = np.zeros(
getattr(self, analysis_phase + "_actual_batch_len_shape"))
if analysis_phase == "train" or analysis_phase == "test":
weights = np.zeros(getattr(self,
analysis_phase + "_weights_shape"))
histlabels = np.zeros((self.num_labels, ))
for batch_idx in range(
getattr(self, "num_" + analysis_phase + "_batches")):
T_0 = None
remaining_len = self.batch_size_timesteps
cur_batch_len = 0
# 1.) Generate the sequence
while remaining_len >= self.min_len_profile:
# 1.1.) Choose run and ingest to get the processed thermal and labelling trace
cur_run = self._choose_and_ingest_run(analysis_phase)
while cur_run.i_symstream.shape[0] <= 1:
cur_run = self._choose_and_ingest_run(analysis_phase)
# 1.2.) Crop the trace to the required length
if cur_run.i_symstream.shape[0] > self.min_len_profile:
if batch_idx < self.batch_size and analysis_phase != "verif":
# This statement should make sure that shorter snippets appear more often
max_snippet_len = random.choice([
(1 + batch_idx) * 2 * self.min_len_profile,
remaining_len,
cur_run.i_symstream.shape[0],
])
min_snippet_len = self.min_len_profile
cur_snippet_len = random.randint(
min_snippet_len,
min([
max_snippet_len, remaining_len,
cur_run.i_symstream.shape[0]
]),
)
else:
# Make sure that at least in the last batch long traces are shown
cur_snippet_len = min(
[remaining_len, cur_run.i_symstream.shape[0]])
else:
cur_snippet_len = cur_run.i_symstream.shape[0]
context_changes = (np.where(
np.diff(cur_run.i_symstream[
cur_snippet_len:, self.label_columns_idxes]) != 0)[0] +
cur_snippet_len)
if len(context_changes) > 0:
# Include context change such that the thermal feature of closing the app is still in the thermal trace
if cur_snippet_len + self.duration_context_change > remaining_len:
cur_snippet_len = remaining_len - self.duration_context_change
cur_profile_len = cur_snippet_len + (
context_changes[0] + self.duration_context_change -
context_changes[0])
trace = np.empty(
(cur_profile_len, len(self.data_columns_idxes)))
trace[:
cur_snippet_len, :] = cur_run.i_symstream[:
cur_snippet_len,
self.
data_columns_idxes]
trace[
cur_snippet_len:, :] = self.experiment.layers.lne.augment(
*[
cur_run.i_symstream[
context_changes[0]:context_changes[0] +
self.duration_context_change, 0, ] -
cur_run.i_symstream[context_changes[0], 0],
cur_run.i_symstream[
context_changes[0]:context_changes[0] +
self.duration_context_change,
self.data_columns_idxes, ],
cur_run.i_symstream[cur_snippet_len,
self.data_columns_idxes],
])
else:
cur_profile_len = cur_snippet_len
trace = np.empty(
(cur_profile_len, len(self.data_columns_idxes)))
trace[:
cur_profile_len, :] = cur_run.i_symstream[:
cur_profile_len,
self.
data_columns_idxes]
# 1.3.) Concatenate the traces
if T_0 is None:
T_0 = trace[0, :].reshape((1, -1))
else:
T_0[0, :] = X[batch_idx, cur_batch_len - 1, :]
Y[batch_idx, cur_batch_len:cur_batch_len +
cur_profile_len, :] = cur_run.i_symstream[:cur_profile_len,
self.
label_columns_idxes]
X[batch_idx, cur_batch_len:cur_batch_len +
cur_profile_len, :] = self.experiment.layers.lne.augment(
*[cur_run.i_symstream[:cur_profile_len, 0], trace, T_0])
remaining_len -= cur_profile_len
cur_batch_len += cur_profile_len
# 2. Add augmentation offset to the traces
sampling_period = np.mean(np.diff(cur_run.i_symstream[:, 0]))
time = np.linspace(
0,
(X[batch_idx, :cur_batch_len, :].shape[0] - 1) *
sampling_period,
X[batch_idx, :cur_batch_len, :].shape[0],
)
X[batch_idx, :cur_batch_len, :] += self._generate_thermal_offset(
time, X[batch_idx, :cur_batch_len, :].shape)
for cur_lbl_col in range(Y.shape[-1]):
hist, _ = np.histogram(
Y[batch_idx, :, cur_lbl_col],
bins=np.array(list(range(self.num_labels + 1))) - 0.5,
density=False,
)
histlabels[:] += hist
# Set length of each iteration sample
actual_batch_len[batch_idx, 0] = cur_batch_len
get_root_logger().info("%s batch %i of %i finished" % (
analysis_phase,
batch_idx,
getattr(self, "num_" + analysis_phase + "_batches"),
))
if analysis_phase == "train" or analysis_phase == "test":
per_label_weights = self.compute_weights(
histlabels, mode=WeightCalculationMode.Proportional)
for label in per_label_weights:
weights[np.where(Y == label)] = per_label_weights[label]
Y[Y == self.label_mapping["UNKNOWN"]["int"] + 1] = 0
setattr(self, analysis_phase + "_X", X)
setattr(self, analysis_phase + "_Y", Y.astype(int))
setattr(self, analysis_phase + "_actual_batch_len", actual_batch_len)
if analysis_phase == "train" or analysis_phase == "test":
setattr(self, analysis_phase + "_weights", weights)
self._serialise_sample_selection(analysis_phase)
def _spawn_server(self) -> None:
server_str = "on the gateway" if self.gateway else "locally"
runner = self.gateway if self.gateway else self.connection
spawn_new_server = True
# Copy the adb vendor key over to the gateway or to a local path
if self.gateway:
self.gateway.put(self.keyfile, self.adb_key)
else:
copy(self.keyfile, self.adb_key, replace=True)
# Check if ADB server is already running on the port systemwide
check_server_exists_cmd = runner.run("pgrep -f '^adb'")
if check_server_exists_cmd.exited == 0:
pids = check_server_exists_cmd.stdout.rstrip("\r\n").split("\n")
for server_pid in pids:
check_server_port_cmd = runner.run(
"ps --no-headers -o command {}".format(server_pid))
regex_match = re.match(r".*(?:-P |tcp:)(\d+)",
check_server_port_cmd.stdout)
if regex_match is None:
continue
_ = regex_match.groups()
running_server_port = int(_[0]) if _ else -1
if self.adb_port == running_server_port:
if not self._spawned:
self.adb_port += 1
else:
spawn_new_server = False
if len(pids) > 0:
if spawn_new_server:
get_root_logger().warning(
"ADB server already running pids: {pids!r}. Starting server on port {where}"
.format(pids=pids, where=self.adb_port))
else:
get_root_logger().warning(
"ADB server already running with pids: {pids!r}. Reusing server on port {where}"
.format(pids=pids, where=self.adb_port))
if spawn_new_server:
cmd = (
"env ADB_VENDOR_KEYS=$HOME/.adb_key nohup "
"adb -a -P {port} server nodaemon 1>{stdout} 2>{stderr} & disown"
).format(**self.adb_params)
# Start the ADB server with the specified command
start_server_cmd = runner.run(cmd)
if start_server_cmd.exited != 0:
raise BackendRuntimeError(
"Failed to start the ADB server {}".format(server_str),
start_server_cmd.stderr,
)
if self.gateway:
self.adb_ip = self.gateway.host
ps_status = runner.run(
"pgrep -f -u $USER '^adb.+{port}'".format(**self.adb_params))
if ps_status.exited != 0:
raise BackendRuntimeError(
"ADB server not running {}!".format(server_str))
self._spawned = True
def fan(self, value: t.Union[bool, str, int, tuple]) -> None:
"""Sets fan settings
Args:
value (t.Union[str, int, tuple]): Value appropriate for the specific fan endpoints
Returns:
None: Returns early if no fan path is available
Raises:
DriverTypeError: Wrong type supplied to the specific fan endpoint
DriverValueError: Wrong value supplied to the specific fan endpoint
"""
fan = self.fan_path
if not fan:
get_root_logger().warning(
"setting fan failed due to no fan path found")
return
if fan["key"] == "thinkpad":
if isinstance(value, bool):
value = "7" if value is True else "auto"
elif not isinstance(value, str):
raise DriverTypeError(
"thinkpad fan setting accepts a bool or a single str")
_ = self.backend.run([
"echo", "level",
quote(value), "|", "sudo", "tee", fan["path"]
])
if not _.ok:
get_root_logger().warning(
f"setting thinkpad fan to {value!r} failed, exit code: {_.exited}, "
f"stderr: {_.stderr}")
elif fan["key"] == "odroid-xu3" or fan["key"] == "odroid-2":
if isinstance(value, bool):
value = ("1", "255") if value is True else ("0", "")
elif not isinstance(value, (tuple, list)) and len(value) != 2:
raise DriverTypeError(
"odroid fan setting accept a bool or a 2-tuple")
assert all(isinstance(v, str)
for v in value), "values must be strings"
_ = self.backend.run(
f"echo {quote(value[0])} | sudo tee {fan['path'] + 'fan_mode'}"
)
if not _.ok:
get_root_logger().warning(
f"setting odroid fan MODE to {value[0]!r} failed, "
f"exit code: {_.exited}, stderr: {_.stderr}")
if value[1]:
_pwm = self.backend.run(
f"echo {quote(value[1])} | sudo tee {fan['path'] + 'pwm_duty'}"
)
if not _pwm.ok:
get_root_logger().warning(
f"setting odroid fan PWM to {value[1]!r} failed, "
f"exit code: {_pwm.exited}, stderr: {_pwm.stderr}")
elif fan["key"] == "pwm-fan-hwmon":
if isinstance(value, bool):
value = ("0", "255") if value is True else ("1", "0")
elif not isinstance(value, (tuple, list)) and len(value) != 2:
raise DriverTypeError(
"odroid fan setting accept a bool or a 2-tuple")
assert all(isinstance(v, str)
for v in value), "values must be strings"
_ = self.backend.run(
f"echo {quote(value[0])} | sudo tee {fan['path'] + 'automatic'}"
)
if not _.ok:
get_root_logger().warning(
f"setting pwm-fan-hwmon fan 'AUTOMATIC' to {value[0]!r} failed, "
f"exit code: {_.exited}, stderr: {_.stderr}")
_ = self.backend.run(
f"echo {quote(value[1])} | sudo tee {fan['path'] + 'pwm1'}")
if not _.ok:
get_root_logger().warning(
f"setting pwm-fan-hwmon fan 'PWM1' to {value[1]!r} failed, "
f"exit code: {_.exited}, stderr: {_.stderr}")
elif fan["key"] == "pwm-fan":
if isinstance(value, bool):
value = 255 if value is True else 0
elif isinstance(value, str):
try:
value = int(value)
except ValueError:
raise DriverValueError(
f"could not convert provided value to integer: {value}"
)
elif isinstance(value, int):
pass
else:
raise DriverTypeError(
f"pwm-fan accepts bool, int, and str; got: {type(value)}")
if value > 255 or value < 0:
raise DriverValueError(
f"pwm-fan accepts values in range [0, 255], got: {value}")
value = str(value)
_ = self.backend.run(
f"echo {quote(value)} | sudo tee {fan['path'] + 'target_pwm'}")
if not _.ok:
get_root_logger().warning(
f"setting pwm-fan to value {value} failed, "
f"exit code: {_.exited}, stderr: {_.stderr}")
