def _choose_and_ingest_run(self, analysis_phase: str = "", run_spec: dict = None): if run_spec is not None: run_rep = [None, None] if "key" in run_spec: run_key = run_spec["key"] for run in getattr(self, analysis_phase + "_runs"): if run[0].config.name == run_spec["key"]: run_rep[0] = run[0] else: run_rep[0] = run_spec["run"] cur_rep = run_spec["rep"] elif self.sample_selection_mode == SampleSelectionMode.Random: run_rep = random.choice(getattr(self, analysis_phase + "_runs")) cur_rep = random.randint(0, run_rep[1] - 1) elif self.sample_selection_mode == SampleSelectionMode.Alternating: min_runs = np.where( self._sample_selection[ self._prev_run, self._sample_lists[self._sample_lists_selector]] == self._sample_selection[ self._prev_run, self._sample_lists[self._sample_lists_selector]].min())[0] min_runs = self._sample_lists[ self._sample_lists_selector][min_runs] self._sample_lists_selector = not self._sample_lists_selector elif self.sample_selection_mode == SampleSelectionMode.Uniform: min_runs = np.where( self._sample_selection[self._prev_run, :] == self._sample_selection[self._prev_run, :].min())[0] if (self.sample_selection_mode == SampleSelectionMode.Alternating or self.sample_selection_mode == SampleSelectionMode.Uniform) and (run_spec is None): run_idx = min_runs[self._sample_selection[min_runs, :].sum( axis=1).argmin()] self._sample_selection[self._prev_run, run_idx] += 1 self._prev_run = run_idx run_rep = getattr(self, analysis_phase + "_runs")[run_idx] cur_rep = random.choice( np.where(self._repetition_selection[run_idx] == self._repetition_selection[run_idx].min())[0]) cur_run = run_rep[0] ingest = False try: cur_run.load_ingestion_data(**self.ingest_args( cur_rep, analysis_phase), prefix=self.name + "_") except: ingest = True if ingest: get_root_logger().debug( f"No ingestion data present, ingesting run {cur_run}") cur_run.ingest(**self.ingest_args(cur_rep, analysis_phase)) cur_run.save_ingestion_data(prefix=self.name + "_") return cur_run
def fan(self) -> t.Optional[t.Union[str, tuple]]: """Gets fan settings Returns: t.Optional[t.Union[str, tuple]]: The fan settings """ fan = self.fan_path if not fan: get_root_logger().warning( "setting fan failed due to no fan path found") return None if fan["key"] == "thinkpad": _ = self.backend.run(["cat", fan["path"], "|", "grep", "'^level'"]) return _.stdout.split(":")[1].strip() if _.ok else None elif fan["key"] == "odroid-xu3" or fan["key"] == "odroid-2": _ = self.backend.run(["cat", fan["path"] + "{fan_mode,pwm_duty}"]) return tuple(_.stdout.splitlines()) if _.ok else None elif fan["key"] == "pwm-fan-hwmon": _ = self.backend.run(["cat", fan["path"] + "{automatic,pwm1}"]) return tuple(_.stdout.splitlines()) if _.ok else None elif fan["key"] == "pwm-fan": _ = self.backend.run(["cat", fan["path"] + "cur_pwm"]) return _.stdout.strip() if _.ok else None
def reboot_device(self): get_root_logger().info( "Rebooting device, this will take roughly 2 minutes...") self.backend.sudo("reboot") sleep( 90 ) # 1.5 minutes security wait time until the device is back online
def latency(self, value: t.Optional[int]) -> None: existing_pgid = getattr(self, "_latency_setter_pgid", None) if existing_pgid: # must kill whole process group, the pgid of children processes # should be the same as calling parent if not self.unix_kill(existing_pgid, pgid=True): _ = self.backend.history[-1] get_root_logger().warning( f"failed to kill CPU DMA setter, exit code {_.exited}, " f"stderr {_.stderr!r}") else: delattr(self, "_latency_setter_pgid") # setting to None will just kill the existing dma latency setter if value is None: return elif not isinstance(value, int): raise TypeError("latency accepts integer values") # Accepts bit-endian values as byte strings or hex values (without 0x prefix) value = value.to_bytes(4, "big").hex() # CPU DMA latency will be set as long as a file descriptor is held, the value # has to be written only once. inner = "exec 4> /dev/cpu_dma_latency; echo -ne {} >&4; sleep 99999".format( str(value)) outer = ["sudo", "/usr/bin/env", "bash", "-c", inner] _ = self.persistent(outer) self._latency_setter_pgid = _.pgid
def _adb_device_wrapper(self, cmd: Backend.CommandT, **kwargs): if not isinstance(cmd, (str, t.List)): raise TypeError(f"'cmd' ({type(cmd)}) not str or list") ip, port = (kwargs.pop("ip", self.config.ip), kwargs.pop("port", self.config.port)) is_usb = self._is_usb(ip) pre = ["-s", f"{ip}:{port}" if not is_usb else ip] if isinstance(cmd, t.List): cmd = pre + cmd else: cmd = " ".join([list2cmdline(pre), cmd]) if not is_usb: def connection_is_ok(): return self._adb_wrapper(pre + ["shell", "pwd"], history=False).ok if not connection_is_ok(): get_root_logger().critical( f"TCP/IP ADB connection to {ip}:{port} went offline, reconnecting..." ) # Make sure the adb server is running self._spawn_server() # Try to reconnect using exponentiall back-off waiting time max_retries = 5 for retries in range(1, max_retries): sleep(2**(retries) - 1) self._adb_wrapper(["connect", f"{ip}:{port}"], history=False) if connection_is_ok(): break else: # Try to restart the adb connection. Sometimes even if the connection seems not ok # we can still access the device as adb is simply messed up but not fully broken. self._adb_wrapper([ "shell", "su", "-c", "setprop", "ctl.restart", "adbd" ]) if retries == (max_retries - 1): # As a last resort, try to reboot the phone sleep(2) self._adb_wrapper(["shell", "su", "-c", "reboot"]) sleep( 60 * 5 ) # Wait for five minutes until the phone is back up if not connection_is_ok(): raise BackendRuntimeError( f"Could not re-establish TCP/IP ADB connection to {ip}:{port} after " f"{retries} retries.") return self._adb_wrapper(cmd, **kwargs)
def write_dataset(self) -> None: """Serialise a Dataset""" self.save_mapping("train_set", path=self.path_dataset, prefix="train_") self.save_mapping("test_set", path=self.path_dataset, prefix="test_") self.save_mapping("verification_set", path=self.path_dataset, prefix="verif_") with self.path_dataset.joinpath("_parameters.toml").open( "w") as parameterfile: toml.dump(self.dataset_parameters_to_dict(), parameterfile) get_root_logger().debug(f"serialised dataset to '{self.path_dataset}'")
def dataset_parameters_from_dict(self, param_dict: dict) -> None: not_set = list() for key in param_dict: try: setattr(self, key, param_dict[key]) except: # This might happen as some attributes don't have a setter. No big deal, but the developer should be informed. not_set.append(key) if len(not_set) > 0: get_root_logger().warning( f"Following attributes could not be set: {not_set}")
def can_connect(self) -> bool: if not self.configured: get_root_logger().critical("Driver not configured!") return False try: self.keyfile = self.config.key except MisconfiguredError: get_root_logger().critical("Could not find key file " + str(self.config.key)) return False return True
def remove_dataset(self) -> None: """Serialise a Dataset""" self.remove_mapping("train_set", path=self.path_dataset, prefix="train_") self.remove_mapping("test_set", path=self.path_dataset, prefix="test_") self.remove_mapping("verification_set", path=self.path_dataset, prefix="verif_") if self.path_dataset.joinpath("_parameters.toml").exists(): os.remove(self.path_dataset.joinpath("_parameters.toml")) get_root_logger().debug( f"serialised dataset from path '{self.path_dataset}' removed")
def _stop_server(self) -> None: # The client can kill the server regardless of its location self._adb_wrapper(["kill-server"]) self._spawned = False for adb_file in [ self.adb_params[_] for _ in ["key", "stdout", "stderr"] ]: if self.gateway: self.gateway.run("rm -f {}".format(quote(adb_file))) else: try: delete(Path(os.path.expandvars(adb_file))) except: get_root_logger().warning(f"Could not delete {adb_file}")
def read_dataset(self) -> None: """Deserialise a Dataset""" train_set = self.load_mapping(path=self.path_dataset, prefix="train_") for key in train_set: setattr(self, "train_" + key, train_set[key]) test_set = self.load_mapping(path=self.path_dataset, prefix="test_") for key in test_set: setattr(self, "test_" + key, test_set[key]) verif_set = self.load_mapping(path=self.path_dataset, prefix="verif_") for key in verif_set: setattr(self, "verif_" + key, verif_set[key]) with self.path_dataset.joinpath("_parameters.toml").open( "r") as parameterfile: param_dict = toml.load(parameterfile) self.dataset_parameters_from_dict(param_dict) get_root_logger().debug(f"serialised dataset to '{self.path_dataset}'")
def _query_state_framework(self) -> EXOT_APP_STATE: """Queries the state of a framework app process Returns: EXOT_APP_STATE: The process state """ query_intent = Intent(COMPONENT_NAME=self.component, ACTION=EXOT_APP_ACTIONS["query"]) intent_time = self.driver.get_time() self.driver.send_intent(query_intent) logs = self.driver.get_logs(grep=r"handleActionQuery\(\): (\S+)", since=intent_time) if len(set(logs)) > 1: get_root_logger().warning( f"logs returned more than one state query output: {set(logs)}") return EXOT_APP_STATE(logs[0]) if logs else EXOT_APP_STATE.INVALID
def can_connect(self) -> bool: if self.configured: # Will throw if the key is not available self.keyfile = self.config.key self._local_adb_version = self._get_adb_version() if "gateway" in self.config: try: self._gateway = fabric.Connection( os.path.expandvars(self.config.gateway), config=self._fabric_config) if self.gateway.run("pwd").exited != 0: get_root_logger().critical("Gateway connection failed") return False self._remote_adb_version = self._get_adb_version( self.gateway) if self._local_adb_version != self._remote_adb_version: get_root_logger().critical( ("The ADB version of the gateway server {} does " "not match the client version {}!").format( self._remote_adb_version, self._local_adb_version)) return False except Exception as e: get_root_logger().critical("Couldn't connect to gateway", e.args) return False return True else: return False
def _execute_handler(self, *args, **kwargs): """Prepare train and test samples.""" get_root_logger().info( "Start preparation of the train, test and verification samples") self.cleanup() for phase in ["train", "test", "verif"]: if getattr(self, "augmentation_" + phase): get_root_logger().info( "Using data augmenetation for phase %s" % phase) self._prepare_with_augmentation(phase) else: get_root_logger().info( "Not using data augmenetation for phase %s" % phase) self._prepare_without_augmentation(phase) get_root_logger().info( "Finished preparation of the train, test and verification samples")
def bootstrap_analysis(self, analysis_name: str): if "ANALYSES" in self.parent.config: if analysis_name in self.parent.config.ANALYSES: kwargs = self.parent.config.ANALYSES[analysis_name].copy() kwargs.update(name=analysis_name) args = [self.parent] if "type" in kwargs: if kwargs["type"] in self.analyses_classes: new_analysis = self.analyses_classes[kwargs["type"]](*args, **kwargs) self.analyses[new_analysis.name] = new_analysis else: get_root_logger().critical( f"Analysis of type %s not available for channel {type(self)}" % (kwargs["type"]) ) else: get_root_logger().critical( f"type not specified in config for analysis {analysis_name}" ) else: get_root_logger().critical( f"Analysis {analysis_name} not specified in configuration!" )
def bootstrap_analyses(self): if "ANALYSES" in self.parent.config: for analysis in self.parent.config.ANALYSES: self.bootstrap_analysis(analysis) else: get_root_logger().info("No analyses specified")
def verify(self, best_model=None): if best_model is not None: self.checkpoint.restore(best_model) get_root_logger().info("Restored model from " + str(best_model)) if self.verif_X is None: get_root_logger().info("No Inference data, nothing to do...") else: ( pred, Y, X, weights, loss, accuracy, lev_abs, lev_norm, lev_len, timing, ) = self.inference(train=False, epoch=None, debug=True) overall_loss = np.nansum(loss) overall_accuracy = np.nanmean(accuracy) overall_lev_abs = np.nansum(lev_abs) overall_lev_norm = np.nanmean(lev_norm) overall_lev_len = np.nansum(lev_len) overall_timing = np.nanmean(timing) get_root_logger().info( "Verification | loss: %03.3f | accuracy: %2.2f%% | lev_abs: %4i | lev_len: %4i | lev_norm: %.3f | timing: %.3f" % ( overall_loss, overall_accuracy, overall_lev_abs, overall_lev_len, overall_lev_norm, overall_timing, ) ) self.write_debug_files(phase="verif", outputs=pred, prefix="prediction") self.write_debug_files(phase="verif", outputs=Y, prefix="Y") self.write_debug_files(phase="verif", outputs=self.test_set, prefix="") self.write_debug_files( phase="verif", outputs={"X": X, "weights": weights}, prefix="data" ) self.write_debug_files( phase="verif", outputs={ "accuracy": accuracy, "lev_abs": lev_abs, "lev_norm": lev_norm, "lev_len": lev_len, "timing": timing, "results": { "loss": overall_loss, "accuracy": overall_accuracy, "lev_abs": overall_lev_abs, "lev_norm": overall_lev_norm, "timing": overall_timing, }, }, prefix="metric", as_txt=True, ) self.checkpoint.save(file_prefix=self.path.joinpath("verif").joinpath("tf_ckpt")) get_root_logger().info( "Inference finished, saved files to " + str(self.path.joinpath("verif")) )
def train(self): get_root_logger().info( "Epoch X | loss: (TRAIN/TEST) | accuracy: (TRAIN/TEST) | lev_abs: (TRAIN/TEST) | lev_len: (TRAIN/TEST) | lev_norm: (TRAIN/TEST) | timing: (TRAIN/TEST)" ) self.best_lev_norm = np.inf self.best_loss = np.inf best_model = None early_stop = False trace_loss = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } trace_accuracy = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } trace_lev_abs = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } trace_lev_norm = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } trace_lev_len = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } trace_timing = { "train": np.empty((math.ceil(self.n_epochs / self.debug_step))), "test": np.empty((math.ceil(self.n_epochs / self.debug_step))), } loss = {"train": None, "test": None} accuracy = {"train": None, "test": None} lev_abs = {"train": None, "test": None} lev_norm = {"train": None, "test": None} lev_len = {"train": None, "test": None} timing = {"train": None, "test": None} pred = {"train": None, "test": None} Y = {"train": None, "test": None} X = {"train": None, "test": None} weights = {"train": None, "test": None} dbg_cnt = 0 for epoch in range(self.start_epoch + 1, self.n_epochs + 1): appdetector_state = self.appdetector.initial_hidden_state # Training self.reshuffle_data("train", epoch) debug = (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1 ( pred["train"], Y["train"], X["train"], weights["train"], loss["train"], accuracy["train"], lev_abs["train"], lev_norm["train"], lev_len["train"], timing["train"], ) = self.inference(epoch, train=True, debug=debug) if (epoch % self.debug_step) == 0 or epoch == self.n_epochs or epoch == 1: ( pred["test"], Y["test"], X["test"], weights["test"], loss["test"], accuracy["test"], lev_abs["test"], lev_norm["test"], lev_len["test"], timing["test"], ) = self.inference( epoch, train=False, debug=debug ) # Test inference for phase in ["train", "test"]: trace_loss[phase][dbg_cnt] = np.nansum(loss[phase]) trace_accuracy[phase][dbg_cnt] = np.nanmean(accuracy[phase]) trace_lev_abs[phase][dbg_cnt] = np.nansum(lev_abs[phase]) trace_lev_norm[phase][dbg_cnt] = np.nanmean(lev_norm[phase]) trace_lev_len[phase][dbg_cnt] = np.nansum(lev_len[phase]) trace_timing[phase][dbg_cnt] = np.nanmean(timing[phase]) get_root_logger().info( "Epoch %03i | loss: (%03.3f/%03.3f) | accuracy: (%2.2f%%/%2.2f%%) | lev_abs: (%4i/%4i) | lev_len: (%4i/%4i) | lev_norm: (%.3f/%.3f) | timing: (%.3f/%.3f)" % ( epoch, trace_loss["train"][dbg_cnt], trace_loss["test"][dbg_cnt], trace_accuracy["train"][dbg_cnt], trace_accuracy["test"][dbg_cnt], trace_lev_abs["train"][dbg_cnt], trace_lev_abs["test"][dbg_cnt], trace_lev_len["train"][dbg_cnt], trace_lev_len["test"][dbg_cnt], trace_lev_norm["train"][dbg_cnt], trace_lev_norm["test"][dbg_cnt], trace_timing["train"][dbg_cnt], trace_timing["test"][dbg_cnt], ) ) if trace_loss["test"][dbg_cnt] == 0.0: train_violations = self.early_stopping_threshold + 1 if trace_loss["test"][dbg_cnt] < self.best_loss: train_violations = 0 self.best_loss = trace_loss["test"][dbg_cnt] for phase in ["train", "test"]: self.write_debug_files( phase=phase, outputs=pred[phase], prefix="prediction" ) self.write_debug_files(phase=phase, outputs=Y[phase], prefix="Y") self.write_debug_files( phase=phase, outputs={"X": X[phase], "weights": weights[phase]}, prefix="data", ) self.write_debug_files( phase=phase, outputs={ "accuracy": accuracy[phase], "lev_abs": lev_abs[phase], "lev_norm": lev_norm[phase], "lev_len": lev_len[phase], "timing": timing[phase], "results": { "epoch": epoch, "loss": trace_loss[phase][dbg_cnt], "accuracy": trace_accuracy[phase][dbg_cnt], "lev_abs": trace_lev_abs[phase][dbg_cnt], "lev_norm": trace_lev_norm[phase][dbg_cnt], "timing": trace_timing[phase][dbg_cnt], }, }, prefix="metric", as_txt=True, ) best_model = self.checkpoint.save(file_prefix=self.path.joinpath("tf_ckpt")) self.config.MODEL["best_model"] = best_model with self.path.joinpath("model.toml").open("w") as toml_file: toml.dump(self.config.MODEL, toml_file) else: train_violations += self.debug_step if epoch >= self.debug_step else 1 dbg_cnt += 1 if train_violations >= self.early_stopping_threshold: if not early_stop: new_lr = self.learning_rate / 10 get_root_logger().info( "Starting second phase of training at %i epochs with learning rate %f!" % (epoch, new_lr) ) get_root_logger().info( "Restoring previously best model from %s!" % str(best_model) ) self.checkpoint.restore(best_model) config = self.optimizer.get_config() config["learning_rate"] = self.learning_rate / 10 self.optimizer = self.optimizer.from_config(config) early_stop = True train_violations = 0 else: get_root_logger().info("Early stopping triggered after %i epochs!" % epoch) break for phase in ["train", "test"]: self.write_debug_files( phase=phase, outputs={ "loss": trace_loss[phase], "accuracy": trace_accuracy[phase], "lev_abs": trace_lev_abs[phase], "lev_norm": trace_lev_norm[phase], "lev_len": trace_lev_len[phase], "timing": trace_timing[phase], }, prefix="trace", as_txt=True, ) get_root_logger().info( "Training finished, saved files to " + str(self.path.joinpath("train")) + " and " + str(self.path.joinpath("test")) ) return best_model
def _prepare_without_augmentation(self, analysis_phase: str): """ """ get_root_logger().info( "Starting data preparation without augmentation for phase %s" % analysis_phase) cnt_batch = 0 if analysis_phase not in ["train", "test", "verif"]: raise Exception(f"Unknown analysis phase {analysis_phase}") setattr(self.config.DATASET, "num_" + analysis_phase + "_batches", 0) reshape_X = list(getattr(self, analysis_phase + "_X_shape")) reshape_X[0] = 0 reshape_Y = list(getattr(self, analysis_phase + "_Y_shape")) reshape_Y[0] = 0 X = np.zeros(reshape_X) Y = np.full(reshape_Y, self.label_mapping["UNKNOWN"]["int"] + 1, dtype=np.int32) actual_batch_len = list() if analysis_phase != "verif": weights = np.zeros(reshape_Y) histlabels = np.zeros((self.num_labels, )) for run_rep in getattr(self, analysis_phase + "_runs"): for cur_rep in range(run_rep[1]): cur_run = self._choose_and_ingest_run(analysis_phase, { "run": run_rep[0], "rep": cur_rep }) _, num_batches = math.modf(cur_run.i_symstream.shape[0] / self.batch_size_timesteps) num_batches = int(num_batches) + 1 for batch_idx in range(num_batches): idx_start = min([ (batch_idx + 0) * self.batch_size_timesteps, cur_run.i_symstream.shape[0], ]) idx_end = min([ (batch_idx + 1) * self.batch_size_timesteps, cur_run.i_symstream.shape[0], ]) x_shape = list(X.shape) x_shape[0] = 1 y_shape = list(Y.shape) y_shape[0] = 1 X = np.vstack([X, np.zeros(x_shape)]) Y = np.vstack([Y, np.zeros(y_shape)]) X[-1, :(idx_end - idx_start), :] = ( cur_run.i_symstream[idx_start:idx_end, self.data_columns_idxes] + self.T_meas_ambient) Y[-1, :(idx_end - idx_start), :] = cur_run.i_symstream[ idx_start:idx_end, self.label_columns_idxes] # Set length of each iteration sample actual_batch_len.append( cur_run.i_symstream[idx_start:idx_end, self.data_columns_idxes].shape[0]) if analysis_phase != "verif": weights = np.vstack([weights, np.zeros(y_shape)]) for cur_lbl_col in range(Y.shape[-1]): hist, _ = np.histogram( Y[batch_idx, :, cur_lbl_col], bins=np.array(list(range(self.num_labels + 1))) - 0.5, density=False, ) histlabels[:] += hist get_root_logger().debug( f"Processing run {cur_run} finished, processed {num_batches}") actual_batch_len = np.array(actual_batch_len).reshape((-1, 1)) if analysis_phase == "verif": self.num_verif_batches = X.shape[0] setattr(self, "verif_X", X) setattr(self, "verif_Y", Y.astype(int)) setattr(self, "verif_actual_batch_len", actual_batch_len) else: per_label_weights = self.compute_weights( histlabels, mode=WeightCalculationMode.Proportional) for label in per_label_weights: weights[np.where(Y == label)] = per_label_weights[label] Y[Y == self.label_mapping["UNKNOWN"]["int"] + 1] = 0 setattr(self, "num_" + analysis_phase + "_batches", X.shape[0]) shuffled_idxes = np.array( shuffle( list( range( getattr(self, "num_" + analysis_phase + "_batches"))), random_state=random.randint( 0, len(getattr(self, analysis_phase + "_runs")) - 1), ), dtype=int, ) setattr(self, analysis_phase + "_X", X[shuffled_idxes]) setattr(self, analysis_phase + "_Y", Y[shuffled_idxes].astype(int)) setattr(self, analysis_phase + "_actual_batch_len", actual_batch_len[shuffled_idxes, :]) setattr(self, analysis_phase + "_weights", weights[shuffled_idxes])
def _prepare_with_augmentation(self, analysis_phase: str): """ generate self.samples and split into training and test data 0) Init empty stream 1) Read stream 2) Resample and normalise stream 3) Augment and append stream 4) Repeat 1-3 until sample is finished 5) Repeat 0-4 until enough self.samples have been initialised """ # -------------------------------------------------------------------------------------------------- get_root_logger().info( "Starting data preparation with augmentation for phase %s" % analysis_phase) sample_timestamps = np.linspace( 0, (self.batch_size_timesteps - 1) * self.resampling_period_s, self.batch_size_timesteps, ) data_col_idx = range(1, min(self.label_columns_idxes)) self._init_sample_selection(analysis_phase) X = np.zeros(getattr(self, analysis_phase + "_X_shape")) Y = np.full( getattr(self, analysis_phase + "_Y_shape"), self.label_mapping["UNKNOWN"]["int"] + 1, dtype=np.int32, ) actual_batch_len = np.zeros( getattr(self, analysis_phase + "_actual_batch_len_shape")) if analysis_phase == "train" or analysis_phase == "test": weights = np.zeros(getattr(self, analysis_phase + "_weights_shape")) histlabels = np.zeros((self.num_labels, )) for batch_idx in range( getattr(self, "num_" + analysis_phase + "_batches")): T_0 = None remaining_len = self.batch_size_timesteps cur_batch_len = 0 # 1.) Generate the sequence while remaining_len >= self.min_len_profile: # 1.1.) Choose run and ingest to get the processed thermal and labelling trace cur_run = self._choose_and_ingest_run(analysis_phase) while cur_run.i_symstream.shape[0] <= 1: cur_run = self._choose_and_ingest_run(analysis_phase) # 1.2.) Crop the trace to the required length if cur_run.i_symstream.shape[0] > self.min_len_profile: if batch_idx < self.batch_size and analysis_phase != "verif": # This statement should make sure that shorter snippets appear more often max_snippet_len = random.choice([ (1 + batch_idx) * 2 * self.min_len_profile, remaining_len, cur_run.i_symstream.shape[0], ]) min_snippet_len = self.min_len_profile cur_snippet_len = random.randint( min_snippet_len, min([ max_snippet_len, remaining_len, cur_run.i_symstream.shape[0] ]), ) else: # Make sure that at least in the last batch long traces are shown cur_snippet_len = min( [remaining_len, cur_run.i_symstream.shape[0]]) else: cur_snippet_len = cur_run.i_symstream.shape[0] context_changes = (np.where( np.diff(cur_run.i_symstream[ cur_snippet_len:, self.label_columns_idxes]) != 0)[0] + cur_snippet_len) if len(context_changes) > 0: # Include context change such that the thermal feature of closing the app is still in the thermal trace if cur_snippet_len + self.duration_context_change > remaining_len: cur_snippet_len = remaining_len - self.duration_context_change cur_profile_len = cur_snippet_len + ( context_changes[0] + self.duration_context_change - context_changes[0]) trace = np.empty( (cur_profile_len, len(self.data_columns_idxes))) trace[: cur_snippet_len, :] = cur_run.i_symstream[: cur_snippet_len, self. data_columns_idxes] trace[ cur_snippet_len:, :] = self.experiment.layers.lne.augment( *[ cur_run.i_symstream[ context_changes[0]:context_changes[0] + self.duration_context_change, 0, ] - cur_run.i_symstream[context_changes[0], 0], cur_run.i_symstream[ context_changes[0]:context_changes[0] + self.duration_context_change, self.data_columns_idxes, ], cur_run.i_symstream[cur_snippet_len, self.data_columns_idxes], ]) else: cur_profile_len = cur_snippet_len trace = np.empty( (cur_profile_len, len(self.data_columns_idxes))) trace[: cur_profile_len, :] = cur_run.i_symstream[: cur_profile_len, self. data_columns_idxes] # 1.3.) Concatenate the traces if T_0 is None: T_0 = trace[0, :].reshape((1, -1)) else: T_0[0, :] = X[batch_idx, cur_batch_len - 1, :] Y[batch_idx, cur_batch_len:cur_batch_len + cur_profile_len, :] = cur_run.i_symstream[:cur_profile_len, self. label_columns_idxes] X[batch_idx, cur_batch_len:cur_batch_len + cur_profile_len, :] = self.experiment.layers.lne.augment( *[cur_run.i_symstream[:cur_profile_len, 0], trace, T_0]) remaining_len -= cur_profile_len cur_batch_len += cur_profile_len # 2. Add augmentation offset to the traces sampling_period = np.mean(np.diff(cur_run.i_symstream[:, 0])) time = np.linspace( 0, (X[batch_idx, :cur_batch_len, :].shape[0] - 1) * sampling_period, X[batch_idx, :cur_batch_len, :].shape[0], ) X[batch_idx, :cur_batch_len, :] += self._generate_thermal_offset( time, X[batch_idx, :cur_batch_len, :].shape) for cur_lbl_col in range(Y.shape[-1]): hist, _ = np.histogram( Y[batch_idx, :, cur_lbl_col], bins=np.array(list(range(self.num_labels + 1))) - 0.5, density=False, ) histlabels[:] += hist # Set length of each iteration sample actual_batch_len[batch_idx, 0] = cur_batch_len get_root_logger().info("%s batch %i of %i finished" % ( analysis_phase, batch_idx, getattr(self, "num_" + analysis_phase + "_batches"), )) if analysis_phase == "train" or analysis_phase == "test": per_label_weights = self.compute_weights( histlabels, mode=WeightCalculationMode.Proportional) for label in per_label_weights: weights[np.where(Y == label)] = per_label_weights[label] Y[Y == self.label_mapping["UNKNOWN"]["int"] + 1] = 0 setattr(self, analysis_phase + "_X", X) setattr(self, analysis_phase + "_Y", Y.astype(int)) setattr(self, analysis_phase + "_actual_batch_len", actual_batch_len) if analysis_phase == "train" or analysis_phase == "test": setattr(self, analysis_phase + "_weights", weights) self._serialise_sample_selection(analysis_phase)
def _spawn_server(self) -> None: server_str = "on the gateway" if self.gateway else "locally" runner = self.gateway if self.gateway else self.connection spawn_new_server = True # Copy the adb vendor key over to the gateway or to a local path if self.gateway: self.gateway.put(self.keyfile, self.adb_key) else: copy(self.keyfile, self.adb_key, replace=True) # Check if ADB server is already running on the port systemwide check_server_exists_cmd = runner.run("pgrep -f '^adb'") if check_server_exists_cmd.exited == 0: pids = check_server_exists_cmd.stdout.rstrip("\r\n").split("\n") for server_pid in pids: check_server_port_cmd = runner.run( "ps --no-headers -o command {}".format(server_pid)) regex_match = re.match(r".*(?:-P |tcp:)(\d+)", check_server_port_cmd.stdout) if regex_match is None: continue _ = regex_match.groups() running_server_port = int(_[0]) if _ else -1 if self.adb_port == running_server_port: if not self._spawned: self.adb_port += 1 else: spawn_new_server = False if len(pids) > 0: if spawn_new_server: get_root_logger().warning( "ADB server already running pids: {pids!r}. Starting server on port {where}" .format(pids=pids, where=self.adb_port)) else: get_root_logger().warning( "ADB server already running with pids: {pids!r}. Reusing server on port {where}" .format(pids=pids, where=self.adb_port)) if spawn_new_server: cmd = ( "env ADB_VENDOR_KEYS=$HOME/.adb_key nohup " "adb -a -P {port} server nodaemon 1>{stdout} 2>{stderr} & disown" ).format(**self.adb_params) # Start the ADB server with the specified command start_server_cmd = runner.run(cmd) if start_server_cmd.exited != 0: raise BackendRuntimeError( "Failed to start the ADB server {}".format(server_str), start_server_cmd.stderr, ) if self.gateway: self.adb_ip = self.gateway.host ps_status = runner.run( "pgrep -f -u $USER '^adb.+{port}'".format(**self.adb_params)) if ps_status.exited != 0: raise BackendRuntimeError( "ADB server not running {}!".format(server_str)) self._spawned = True
def fan(self, value: t.Union[bool, str, int, tuple]) -> None: """Sets fan settings Args: value (t.Union[str, int, tuple]): Value appropriate for the specific fan endpoints Returns: None: Returns early if no fan path is available Raises: DriverTypeError: Wrong type supplied to the specific fan endpoint DriverValueError: Wrong value supplied to the specific fan endpoint """ fan = self.fan_path if not fan: get_root_logger().warning( "setting fan failed due to no fan path found") return if fan["key"] == "thinkpad": if isinstance(value, bool): value = "7" if value is True else "auto" elif not isinstance(value, str): raise DriverTypeError( "thinkpad fan setting accepts a bool or a single str") _ = self.backend.run([ "echo", "level", quote(value), "|", "sudo", "tee", fan["path"] ]) if not _.ok: get_root_logger().warning( f"setting thinkpad fan to {value!r} failed, exit code: {_.exited}, " f"stderr: {_.stderr}") elif fan["key"] == "odroid-xu3" or fan["key"] == "odroid-2": if isinstance(value, bool): value = ("1", "255") if value is True else ("0", "") elif not isinstance(value, (tuple, list)) and len(value) != 2: raise DriverTypeError( "odroid fan setting accept a bool or a 2-tuple") assert all(isinstance(v, str) for v in value), "values must be strings" _ = self.backend.run( f"echo {quote(value[0])} | sudo tee {fan['path'] + 'fan_mode'}" ) if not _.ok: get_root_logger().warning( f"setting odroid fan MODE to {value[0]!r} failed, " f"exit code: {_.exited}, stderr: {_.stderr}") if value[1]: _pwm = self.backend.run( f"echo {quote(value[1])} | sudo tee {fan['path'] + 'pwm_duty'}" ) if not _pwm.ok: get_root_logger().warning( f"setting odroid fan PWM to {value[1]!r} failed, " f"exit code: {_pwm.exited}, stderr: {_pwm.stderr}") elif fan["key"] == "pwm-fan-hwmon": if isinstance(value, bool): value = ("0", "255") if value is True else ("1", "0") elif not isinstance(value, (tuple, list)) and len(value) != 2: raise DriverTypeError( "odroid fan setting accept a bool or a 2-tuple") assert all(isinstance(v, str) for v in value), "values must be strings" _ = self.backend.run( f"echo {quote(value[0])} | sudo tee {fan['path'] + 'automatic'}" ) if not _.ok: get_root_logger().warning( f"setting pwm-fan-hwmon fan 'AUTOMATIC' to {value[0]!r} failed, " f"exit code: {_.exited}, stderr: {_.stderr}") _ = self.backend.run( f"echo {quote(value[1])} | sudo tee {fan['path'] + 'pwm1'}") if not _.ok: get_root_logger().warning( f"setting pwm-fan-hwmon fan 'PWM1' to {value[1]!r} failed, " f"exit code: {_.exited}, stderr: {_.stderr}") elif fan["key"] == "pwm-fan": if isinstance(value, bool): value = 255 if value is True else 0 elif isinstance(value, str): try: value = int(value) except ValueError: raise DriverValueError( f"could not convert provided value to integer: {value}" ) elif isinstance(value, int): pass else: raise DriverTypeError( f"pwm-fan accepts bool, int, and str; got: {type(value)}") if value > 255 or value < 0: raise DriverValueError( f"pwm-fan accepts values in range [0, 255], got: {value}") value = str(value) _ = self.backend.run( f"echo {quote(value)} | sudo tee {fan['path'] + 'target_pwm'}") if not _.ok: get_root_logger().warning( f"setting pwm-fan to value {value} failed, " f"exit code: {_.exited}, stderr: {_.stderr}")