def __init__(self, X, y, features, samples_id, depth, max_depth, lambdap=0.01, gamma=0.01, loss='mse'): self.lambdap = lambdap self.gamma = gamma self.loss_metric = loss if loss == 'mse': self.loss = MSE() elif loss == 'log': self.loss = LogLoss() super().__init__(X, y, features, samples_id, depth, max_depth)
def build_model(): model = Sequential(MSE(), input_size=2) model.add_layer(Linear(2, 25)) model.add_layer(ReLU(25)) model.add_layer(Linear(25, 25)) model.add_layer(ReLU(25)) model.add_layer(Linear(25, 25)) model.add_layer(Tanh(25)) model.add_layer(Linear(25, 2)) return model
def evaluate(self, X, Y): if (X.ndim == 1): X = X.reshape(len(X), 1) bias = np.ones(len(X)).reshape(len(X), 1) X = np.concatenate((X, bias), axis=1) Y_pred = X.dot(self.Weights) loss = MSE(Y, Y_pred) print("Test MSE loss : ", loss)
def compile(self, loss='mse', lr=0.001): global LOSS_LIST if (loss not in LOSS_LIST and not isinstance(loss, Loss)): raise Exception("Invalid loss") if (isinstance(loss, str)): if (loss == 'mse'): self.loss = MSE() else: self.loss = loss self.lr = lr
def train(self): dataset = self.dataset(self.args.data, self.args.gt, self.args.val_size, crop_div=8) out_dir = self._create_out_dir() train_dataset = dataset.get_dataset(data_type='train', batch_size=self.args.batch, repeat_count=None, crop_params=('random', self.args.crop)) valid_dataset = dataset.get_dataset(data_type='valid', batch_size=1, repeat_count=None, crop_params=('center', self.args.crop)) optimizer = tf.keras.optimizers.Adam(self.args.lr, beta_1=.5) self.model.compile(loss=MSE(), optimizer=optimizer, metrics=[PSNR(), SSIM()]) early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_PSNR', patience=10, mode='max') csv_log = tf.keras.callbacks.CSVLogger( os.path.join(out_dir, f'train_{self.args.model}.log')) checkpoints_path = os.path.join(out_dir, self.args.model) saver = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoints_path + '_{epoch:03d}-{val_PSNR:.2f}.h5', monitor='val_PSNR') callbacks = [csv_log, saver] if not self.args.no_early_stop: callbacks.append(early_stop) self.model.fit(train_dataset, epochs=self.args.epochs, callbacks=callbacks, validation_data=valid_dataset, verbose=1, validation_steps=1, steps_per_epoch=dataset.numel * self.args.repeat // self.args.batch)
def __init__(self, n_iters=10000, hidden_activation=Sigmoid(), output_activation=Linear(), learning_rate=1e-2, n_hidden=10, loss=MSE(), mini_batch=10): self.n_iters = n_iters self.hidden_activation = hidden_activation self.output_activation = output_activation self.learning_rate = learning_rate self.W = None self.W0 = None self.V = None self.V0 = None self.mini_batch = mini_batch self.loss = loss self.X, self.y = None, None self.n_hidden = n_hidden
def fit(self, X_train, Y_train, showfig=False): self.X = X_train.astype(dtype=np.float32) self.Y = Y_train self.X = self.X.reshape(len(self.X), 1) bias = np.ones(len(self.X)).reshape(len(self.X), 1) self.X = np.concatenate((self.X, bias), axis=1) self.Weights = np.random.rand(self.X.ndim + 1) self.Weights = np.linalg.inv(self.X.T.dot(self.X)).dot( self.X.T.dot(self.Y)) Y_pred = self.X.dot(self.Weights) loss = MSE(self.Y, Y_pred) print("MSE loss : ", loss) if (showfig): plt.scatter(self.X[:, :-1], self.Y, c='b') plt.plot(self.X[:, :-1], Y_pred, c='r') plt.show()
output_layer = None if type_of_assigment == "regression": p_train = RegressionProvider(train_df, batch_size=json_parser.batch_size) p_test = RegressionProvider(test_df, batch_size=json_parser.batch_size) output_layer = "linear" elif type_of_assigment == "classification": p_train = ClassifierProvider(train_df, batch_size=json_parser.batch_size) p_test = ClassifierProvider(test_df, batch_size=json_parser.batch_size) output_layer = "sigmoid" hidden = json_parser.layers_size act = json_parser.layers_activations act.append(output_layer) seed = json_parser.seed loss = MSE() number_of_iterations = json_parser.number_of_iterations nn = NeuralNet(inputs=p_train.number_of_inputs, hidden=hidden, outputs=p_train.number_of_outputs, activations=act, loss=loss, seed=seed) summary = train(nn, number_of_iterations, p_train, p_test, print_weights) summary.show() visualization = VisualizationProvider(test_df, nn, type_of_assigment) x = visualization.show() visualization.show_map() print("Accuracy: {}".format(x))
W1 = np.array([[0.15, 0.20], [0.25, 0.30]]) b1 = 0.35 W2 = np.array([[0.4, 0.45], [0.50, 0.55]]) b2 = 0.60 y_true = np.array([[0.01, 0.99]]) dense = Dense(2, W1, b1) sigmoid = Activations('Sigmoid') swish1 = Activations('Swish') dense2 = Dense(2, W2, b2) swish2 = Activations('Swish') activation2 = Activations('Sigmoid') loss_fn = MSE() z1 = dense.forward(x) sig1 = sigmoid.forward(z1) z2 = dense2.forward(sig1) y_pred = activation2.forward(z2) sw1 = swish1.forward(z1) sw2 = dense2.forward(sw1) y_pre = swish2.forward(sw2) # loss = loss_fn.loss(y_true, y_pred) # print("loss: ", loss) # print("loss's mean: ",np.mean(loss)) # sigloss = loss_fn.loss(y_true, y_pre)
def quantization(): if args.quant_mode != 'test' and args.deploy: args.deploy = False warnings.warn( 'Exporting xmodel needs to be done in quantization test mode, turn off it in this running!', UserWarning) if args.quant_mode == 'test' and (args.batch_size != 1 or args.subset_len != 1): warnings.warn( 'Exporting xmodel needs batch size to be 1 and only 1 iteration of inference, they\'ll be changed automatically!', UserWarning) args.batch_size = 1 args.subset_len = 1 p = Path(args.checkpoint_dir) / args.model_name model = FFN(args.input_size) model = preprocessors.load_from_state_dict(model, p) if args.quant_mode == 'float': quant_model = deepcopy(model) else: rand_input = torch.randn([args.batch_size, args.input_size]) quantizer = torch_quantizer(args.quant_mode, module=deepcopy(model), input_args=rand_input, bitwidth=8, mix_bit=False, qat_proc=False, device=set_seed.DEVICE) quant_model = quantizer.quant_model if args.fast_finetune: ft_loader = preprocessors.make_dataloader(data_dir=args.data_dir, data_file=args.calib_data, subset_len=args.subset_len) if args.quant_mode == 'calib': loss_fn = MSE().to(set_seed.DEVICE) quantizer.fast_finetune(eval_loss, (quant_model, ft_loader, loss_fn)) elif args.quant_mode == 'test': quantizer.load_ft_param() if args.evaluate: valid_loader = preprocessors.make_dataloader( data_dir=args.data_dir, data_file=args.calib_data, batch_size=args.batch_size) cr1 = CustomRunner(model=model, device=set_seed.DEVICE, input_key='features', input_target_key='targets', evaluate=True, loaders={'test': valid_loader}) print('Evaluation completed!') print('Initial model results:') pprint.pprint(cr1.logs, width=5) if args.quant_mode != 'float': cr2 = CustomRunner(model=quant_model, device=set_seed.DEVICE, input_key='features', input_target_key='targets', evaluate=True, loaders={'test': valid_loader}) print('Quantized model results:') pprint.pprint(cr2.logs, width=5) if args.quant_mode == 'calib': quantizer.export_quant_config() if args.deploy: quantizer.export_xmodel(deploy_check=True)
def train_model(model, lr, batch_size, momentum, epochs, train_input, train_target, test_input, test_target, loss_log=False): losses = [] validation_input = test_input valdidation_target = test_target validation_loss = [] train_loss = [] validation_acc = [] train_acc = [] optimizer = SGD(model.layers, learning_rate=lr, momentum=momentum) criterion = MSE() batch_size = batch_size for epoch in range(epochs): acc_loss = 0 for b in range(0, train_input.size(0), batch_size): if b + batch_size > train_input.size(0): mini_batch_size = train_input.size(0) - b else: mini_batch_size = batch_size #model forward pass pred = model.forward(train_input.narrow(0, b, mini_batch_size)) ## computing loss loss = criterion.compute_loss( pred, train_target.narrow(0, b, mini_batch_size)) acc_loss += loss.item() #compute gradient of loss grad_wrt_outputs = criterion.backward() #backpropagate with gradient of loss model.backward(grad_wrt_outputs) #update parameters of model optimizer.step() #reset gradient matrices to 0 model.zero_grad() batches = int(train_input.size(0) / mini_batch_size ) if train_input.size(0) % mini_batch_size == 0 else int( train_input.size(0) / mini_batch_size) + 1 #keeping track of loss accuracy after one epoch if loss_log: print(epoch, acc_loss / batches) pred_train = model.forward(train_input) pred_test = model.forward(test_input) train_loss.append(MSE().compute_loss(pred_train, train_target).item()) validation_loss.append(MSE().compute_loss(pred_test, test_target).item()) train_acc.append(computeAccuracy(model, train_input, train_target)) validation_acc.append( computeAccuracy(model, test_input, test_target)) model.zero_grad() return train_loss, validation_loss, train_acc, validation_acc
W2 = np.array([[.40, .45], [.50, .55]]) b1 = .35 b2 = 0.60 y_true = np.array([[.01, .99]]) #Layers Generation dense = Dense(2, W1, b1) dense2 = Dense(2, W2, b2) activation1 = Sigmoid() # activation2=Sigmoid() activation2 = Activation("sigmoid") loss_func = MSE() #Forward Pass # Dense -> Activation -> Dense -> Activation -> y_pred z1 = dense.forward(x) a1 = activation1.forward(z1) print("Activation Value:", a1) z2 = dense2.forward(a1) a2 = activation2.forward(z2) y_pred = a2 loss = loss_func.loss(y_true, y_pred) print("Individual Loss:", loss)
if __name__ == '__main__': fcnn = FCNN() times = torch.linspace(-5, 5, 100) training_times = torch.linspace(-5, 5, 100).unsqueeze(-1) print(training_times.shape) training_values = g(training_times, True) values = g(times) optimiser = AdamHD(fcnn.parameters(), alpha_lr=1e-8) #optimiser = Adam(fcnn.parameters) epochs = 500 for _ in range(epochs): predictions = fcnn(training_times) loss = MSE(training_values, predictions) loss.backward() optimiser.step() optimiser.zero_grad() print(f"MSE : {loss}") plotting = True if plotting: plt.figure() with torch.no_grad(): for i in range(5): predictions = fcnn(torch.unsqueeze(times, -1), training=True).squeeze() if i == 0: plt.plot(times,