def _setup(self, config): # controls how often workers report model performance self.batches_per_step = config.get("batches_per_step", 1) self.max_validation_steps = config.get("max_validation_steps", 5) # removes channels from data to increase performance self.use_only_first_channel = config.get("use_only_first_channel", False) self.train_data = get_pinned_object(config.get("train_data")) self.val_data = get_pinned_object(config.get("val_data")) # GPU or CPU? use_cuda = config.get("use_gpu") and torch.cuda.is_available() print("CUDA:", use_cuda) self.device = torch.device("cuda" if use_cuda else "cpu") self.batch_size = config.get("batch_size", 2) self.verbose = config.get("verbose", False) # Abstract: implement self.model optimizer = config.get("optimizer", "Adam") if optimizer == "SGD": self.optimizer = optim.SGD( self.model.parameters(), lr=config.get("lr", 0.01), momentum=config.get("momentum", 0.9), ) elif optimizer == "Adam": self.optimizer = optim.Adam(self.model.parameters()) else: raise NotImplemented() self.validation_loss_F = F.mse_loss loss_name = config.get("training_loss", "mse_loss") self.train_loss_F = mappings.losses[loss_name]
def _setup(self, config: dict): self._dataset, self._col_labels, self._y_label = ( get_pinned_object(data_id), None, "Value", ) self.rf_model = None self._build(config)
def objective(config): #get objects from central tune storage labeler = get_pinned_object(labeler_id) price_array = get_pinned_object(price_array_id) calculate_profit = get_pinned_object(calculate_profit_id) #get the labels from the specific method label_list = labeler(config) #create the array tha gets passed to the profit calculator label_array = pd.concat(label_list, axis=0).to_numpy() label_array = np.expand_dims(label_array, axis=1) array = np.concatenate([price_array, label_array], axis=1) specific_profit, _ = calculate_profit(array, self.trading_fee) tune.report(specific_profit=specific_profit) time.sleep(0.1)
def _setup(self, config): [X_train, X_test, y_train, y_test] = get_pinned_object(data_id) self.cuml_model = curfc(n_estimators=config.get("estimators", 40), max_depth=config.get("depth", 16), max_features=1.0) self.X_cudf_train = cudf.DataFrame.from_pandas(X_train) self.X_cudf_test = cudf.DataFrame.from_pandas(X_test) self.y_cudf_train = cudf.Series(y_train.values) self.y_test = y_test
def _validation_loss(self): self.model.eval() val_loss = [] with torch.no_grad(): count = tqdm( range(self.max_validation_steps), desc="Validation", disable=not self.verbose, ) for data, i in zip(get_pinned_object(self.val_data), count): if self.use_only_first_channel: data = data.narrow(1, 0, 1) data = data.to(self.device) pred = self.model(data) val_loss.append(float(self.validation_loss_F(pred, data))) return np.mean(val_loss)
def _train_batches(self, train_for_n_batches=1): losses = [] self.model.train() count = tqdm( range(train_for_n_batches), desc="Training", disable=not self.verbose, leave=True ) data_train = get_pinned_object(self.get_train_pin()) for data, i in zip(data_train, count): if self.batch_size and self.batch_size < data['data'].shape[0]: data = data['data'][:self.batch_size].to(self.device) else: data = data['data'].to(self.device) self.optimizer.zero_grad() pred = self.model(data) loss = self.train_loss_F(pred, data) loss.backward() self.optimizer.step() losses.append(float(loss)) return losses
def _validation_loss(self): self.model.eval() val_loss = [] truths = [] with torch.no_grad(): data_val = get_pinned_object(self.get_val_pin()) for data,_ in zip(data_val, range(self.max_validation_steps)): wf = data['data'].to(self.device) truth = (data['MC_type']>0) pred = self.model(wf) val_loss.append(self.validation_loss_F(pred, wf).cpu().numpy()) truths.append(np.array(truth)) try: pred = np.hstack(np.asarray(val_loss)) truth = np.hstack(truths) fpr, tpr, _ = metrics.roc_curve(truth, pred) auc = metrics.auc(fpr, tpr) except ValueError: return 0 return auc, np.mean(pred)
def train(config, reporter): get_pinned_object(X) reporter(timesteps_total=100, done=True)
def f(): return get_pinned_object(X)