def train_with_loader(network,
training_data,
config_dict,
validation_data=None,
loss_fn=torch.nn.MSELoss()):
print('------- TRAINING WITH LOADER ---------')
# set configurations
# optimizer = set_optimizer(network, config_dict)
optimizer = torch.optim.Adam(network.params_groups(config_dict),
betas=config_dict['betas'])
config_dict["save_dir"] = create_directory_timestamp(
config_dict["save_dir"], config_dict["name"])
save_configs(
config_dict,
os.path.join(config_dict["save_dir"], "training_configs.json"))
# Define variables
regularization = torch.tensor(config_dict['cv_penalty'],
device=TorchUtils.get_accelerator_type())
costs = np.zeros((config_dict["nr_epochs"],
2)) # training and validation costs per epoch
accuracy_over_epochs = np.zeros((config_dict["nr_epochs"], 2))
# grads_epochs = np.zeros((config_dict["nr_epochs"], config_dict["nr_nodes"]))
looper = trange(config_dict["nr_epochs"], desc='Initialising')
for epoch in looper:
costs[epoch, 0], accuracy_over_epochs[epoch, 0] = batch_training(
training_data, network, optimizer, loss_fn, regularization)
# grads_epochs[epoch] = np.asarray([np.abs(p.grad.detach().cpu().numpy()).max()
# for p in network.dnpu_layer.parameters() if p.requires_grad])
# Evaluate Validation error
if validation_data is not None:
costs[epoch, 1], accuracy_over_epochs[epoch, 1] = evaluate(
network, loss_fn, validation_data)
else:
costs[epoch, 1], accuracy_over_epochs[epoch, 1] = np.nan, np.nan
if 'save_interval' in config_dict.keys(
) and epoch % config_dict["save_interval"] == 10:
save_model(network, config_dict["save_dir"], f'checkpoint_{epoch}')
if np.isnan(costs[epoch, 0]):
costs[epoch:, 0] = np.nan
costs[epoch:, 1] = np.nan
print('--------- Training interrupted value was Nan!! ---------')
break
looper.set_description(
f' Epoch: {epoch} | Training Error:{costs[epoch, 0]:7.3f} | Val. Error:{costs[epoch, 1]:7.3f}'
)
save_model(network, config_dict["save_dir"], 'final_model')
save_parameters_as_numpy(network, config_dict["save_dir"])
np.savez(os.path.join(config_dict["save_dir"], 'training_history'),
costs=costs,
accuracy=accuracy)
print('------------DONE-------------')
return costs, accuracy_over_epochs
def load_file(self, data_dir, file_type):
if file_type == 'pt':
state_dict = torch.load(data_dir, map_location=TorchUtils.get_accelerator_type())
info = state_dict['info']
del state_dict['info']
info['smg_configs'] = self._info_consistency_check(info['smg_configs'])
if 'amplification' not in info['data_info']['processor'].keys():
info['data_info']['processor']['amplification'] = 1
elif file_type == 'json':
state_dict = None
# TODO: Implement loading from a json file
raise NotImplementedError(f"Loading file from a json file in TorchModel has not been implemented yet. ")
# info = model_info loaded from a json file
return info, state_dict
def get_simulation_architecture(configs):
if configs['processor_type'] == 'nn':
raise NotImplementedError(
f"{configs['processor_type']} 'processor_type' nn configuration is not implemented yet. "
)
if configs['processor_type'] == 'surrogate':
return get_processor_architecture(configs)
elif configs['processor_type'] == 'dnpu':
return get_dnpu_architecture(configs).to(
device=TorchUtils.get_accelerator_type())
else:
raise NotImplementedError(
f"{configs['processor_type']} 'processor_type' configuration is not recognised. The simulation type has to be defined as 'nn', 'surrogate' or 'dpnu'. "
)
def load_data(base_dir):
import os
import torch
import pickle
from bspyalgo.utils.io import load_configs
model_dir = os.path.join(base_dir, 'reproducibility', 'model.pt')
results_dir = os.path.join(base_dir, 'reproducibility', 'results.pickle')
configs_dir = os.path.join(base_dir, 'reproducibility', 'configs.json')
model = torch.load(model_dir,
map_location=TorchUtils.get_accelerator_type())
results = pickle.load(open(results_dir, "rb"))
configs = load_configs(configs_dir)
configs['results_base_dir'] = base_dir
return model, results, configs
def choose_loss_function(loss_fn_name):
'''Gets the fitness function used in GD from the module losses.
The loss functions must take two arguments, the outputs of the black-box and the target
and must return a torch array of scores of size len(outputs).
'''
if loss_fn_name == 'corrsig':
return corrsig
elif loss_fn_name == 'sqrt_corrsig':
return sqrt_corrsig
elif loss_fn_name == 'fisher':
return fisher
elif loss_fn_name == 'fisher_added_corr':
return fisher_added_corr
elif loss_fn_name == 'fisher_multipled_corr':
return fisher_multipled_corr
elif loss_fn_name == 'bce':
bce = BCELossWithSigmoid()
bce.cuda(TorchUtils.get_accelerator_type()).to(TorchUtils.data_type)
return bce
else:
raise NotImplementedError(
f"Loss function {loss_fn_name} is not recognized!")
import torch.nn as nn
import matplotlib.pyplot as plt
from bspyalgo.utils.io import load_configs
from bspyproc.architectures.dnpu.modules import DNPU_Layer
from bspyproc.utils.pytorch import TorchUtils
# Generate model
NODE_CONFIGS = load_configs(
'/home/hruiz/Documents/PROJECTS/DARWIN/Code/packages/brainspy/brainspy-processors/configs/configs_nn_model.json'
)
nr_nodes = 5
input_list = [[0, 3, 4]] * nr_nodes
data_dim = 20
linear_layer = nn.Linear(data_dim,
len(input_list[0]) * nr_nodes).to(
device=TorchUtils.get_accelerator_type())
dnpu_layer = DNPU_Layer(input_list, NODE_CONFIGS)
model = nn.Sequential(linear_layer, dnpu_layer)
# Generate data
nr_train_samples = 50
nr_val_samples = 10
x = TorchUtils.format_tensor(
torch.rand(nr_train_samples + nr_val_samples, data_dim))
y = TorchUtils.format_tensor(
5. * torch.ones(nr_train_samples + nr_val_samples, nr_nodes))
inp_train = x[:nr_train_samples]
t_train = y[:nr_train_samples]
inp_val = x[nr_train_samples:]
t_val = y[nr_train_samples:]
return x.view(-1, self.out_size, self.out_size)
if __name__ == '__main__':
from bspyalgo.utils.io import load_configs
import matplotlib.pyplot as plt
import time
NODE_CONFIGS = load_configs(
'/home/hruiz/Documents/PROJECTS/DARWIN/Code/packages/brainspy/brainspy-processors/configs/configs_nn_model.json'
)
node = TorchModel(NODE_CONFIGS)
# linear_layer = nn.Linear(20, 3).to(device=TorchUtils.get_accelerator_type())
# dnpu_layer = DNPU_Channels([[0, 3, 4]] * 1000, node)
linear_layer = nn.Linear(20,
300).to(device=TorchUtils.get_accelerator_type())
dnpu_layer = DNPU_Layer([[0, 3, 4]] * 100, node)
model = nn.Sequential(linear_layer, dnpu_layer)
data = torch.rand((200, 20)).to(device=TorchUtils.get_accelerator_type())
start = time.time()
output = model(data)
end = time.time()
# print([param.shape for param in model.parameters() if param.requires_grad])
print(
f'(inputs,outputs) = {output.shape} of layer evaluated in {end-start} seconds'
)
print(f'Output range : [{output.min()},{output.max()}]')