def FNN(linear_feature_columns,
dnn_feature_columns,
embedding_size=8,
dnn_hidden_units=(128, 128),
l2_reg_embedding=1e-5,
l2_reg_dnn=0,
init_std=0.0001,
seed=1024,
dnn_dropout=0.0,
dnn_activation='relu',
task='binary'):
# 将所有特征转化成tensor，返回的是OrderedDict
features = inputs.build_input_features(linear_feature_columns +
dnn_feature_columns)
# 从OrderedDict中获取所有的value，即获取所有的tensor
inputs_list = list(features.values())
# 得到输入dnn的特征list，在fnn中不区分两者，因为dnn_feature_columns = linear_feature_columns
sparse_embedding_list, dense_value_list = inputs.input_from_feature_columns(
features, dnn_feature_columns, embedding_size, l2_reg_embedding,
init_std, seed)
dnn_input = inputs.combined_dnn_input(sparse_embedding_list,
dense_value_list)
deep_out = DNN.DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn,
dnn_dropout, False, seed)(dnn_input)
deep_logit = tf.keras.layers.Dense(1, use_bias=False,
activation=None)(deep_out)
output = PredictionLayer(task)(deep_logit)
model = tf.keras.models.Model(inputs=inputs_list, outputs=output)
return model
def main(argv=None):
model = DNN(
train_images_dir=FLAGS.train_images_dir,
val_images_dir=FLAGS.val_images_dir,
test_images_dir=FLAGS.test_images_dir,
num_epochs=FLAGS.num_epochs,
train_batch_size=FLAGS.train_batch_size,
val_batch_size=FLAGS.val_batch_size,
test_batch_size=FLAGS.test_batch_size,
height_of_image=FLAGS.height_of_image,
width_of_image=FLAGS.width_of_image,
num_channels=FLAGS.num_channels,
# num_classes=FLAGS.num_classes,
learning_rate=FLAGS.learning_rate,
base_dir=FLAGS.base_dir,
max_to_keep=FLAGS.max_to_keep,
model_name=FLAGS.model_name,
)
if FLAGS.train:
model.create_network(model_type="train")
model.initialize_network()
model.train_model(FLAGS.display_step, FLAGS.validation_step,
FLAGS.checkpoint_step, FLAGS.summary_step)
else:
model.create_network(model_type="test")
model.initialize_network()
model.test_model()
def main(argv=None):
model = DNN(train_images_dir='data/train/',
val_images_dir='data/val/',
test_images_dir='data/test/',
num_epochs=40,
train_batch_size=1000,
val_batch_size=1000,
test_batch_size=10000,
height_of_image=28,
width_of_image=28,
num_channels=1,
num_classes=10,
learning_rate=0.001,
base_dir='results',
max_to_keep=2,
model_name="DNN",
model='DNN')
model.create_network()
model.initialize_network()
if False:
model.train_model(1, 1, 1, 4)
else:
model.test_model()
def main(argv=None):
model = DNN(
train_spec_dir=FLAGS.train_spec_dir,
val_spec_dir=FLAGS.val_spec_dir,
test_spec_dir=FLAGS.test_spec_dir,
num_epochs=FLAGS.num_epochs,
train_batch_size=FLAGS.train_batch_size,
val_batch_size=FLAGS.val_batch_size,
test_batch_size=FLAGS.test_batch_size,
sequence_length=FLAGS.sequence_length,
fft_length=FLAGS.fft_length,
learning_rate=FLAGS.learning_rate,
base_dir=FLAGS.base_dir,
max_to_keep=FLAGS.max_to_keep,
model_name=FLAGS.model_name,
)
model.create_network()
print('[*] Network created')
model.initialize_network()
print('[*] Network initialized')
if FLAGS.train:
model.train_model(FLAGS.display_step, FLAGS.validation_step, FLAGS.checkpoint_step, FLAGS.summary_step)
print('[*] Model trained')
else:
model.test_model()
print('[*] Model tested')
def restore_dnn(self, model):
if not model:
params = {
"offline_model_dir": PROJECT_ROOT+"/ltr/weights/dnn",
# deep part score fn
"fc_type": "fc",
"fc_dim": 32,
"fc_dropout": 0.,
}
params.update(self.params_common)
model = DNN("ranking", params, self.logger, training=False)
model.restore_session()
return model
def train_dnn(self, cid):
params = {
"offline_model_dir": PROJECT_ROOT+"/ltr/weights/dnn",
# deep part score fn
"fc_type": "fc",
"fc_dim": 32,
"fc_dropout": 0.,
}
params.update(self.params_common)
X_train, X_valid = self.load_data_by_id("train", cid), self.load_data_by_id("vali", cid)
model = DNN("ranking", params, self.logger)
model.fit(X_train, validation_data=X_valid)
model.save_session()
return model
def run_forrest_run(dataset_list, activation_list, modelname):
for dataset_name in dataset_list:
for name in activation_list:
for model in modelname:
if model == "DNN":
dataset = Datasets()
if (dataset_name == 'MNIST'):
x_train, x_test, y_train, y_test = dataset.get_mnist(
"DNN")
num_classes = dataset.num_classes
input_shape = dataset.input_shape
elif (dataset_name == 'Fashion-MNIST'):
x_train, x_test, y_train, y_test = dataset.get_fashion_mnist(
"DNN")
num_classes = dataset.num_classes
input_shape = dataset.input_shape
dnn = DNN(name)
score, history = dnn.run_model(input_shape, x_train,
x_test, y_train, y_test, 1)
else:
dataset = Datasets()
if (dataset_name == 'MNIST'):
x_train, x_test, y_train, y_test = dataset.get_mnist(
"CNN")
elif (dataset_name == 'Fashion-MNIST'):
x_train, x_test, y_train, y_test = dataset.get_fashion_mnist(
"CNN")
num_classes = dataset.num_classes
input_shape = dataset.input_shape
if model == "CNN":
cnn = CNN(name)
score, history = cnn.run_model(input_shape, x_train,
x_test, y_train, y_test)
elif model == "CAE":
cae = CAE(name)
score, history = cae.run_model(input_shape, x_train,
x_test, y_train, y_test)
score, history = cnn.run_model(input_shape, x_train,
x_test, y_train, y_test)
plot_model(history, name, model, dataset_name)
def main():
#print the config args
print(config.transfer_learning)
print(config.mode)
print(config.input_size)
# Fix Seed for Reproducibility #
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(config.seed)
# Samples, Weights, and Plots Path #
paths = [config.weights_path, config.plots_path, config.numpy_path]
for path in paths:
make_dirs(path)
# Prepare Data #
data = load_data(config.combined_path, config.which_data, config.preprocess, config.resample)
# id = config.which_data.split('_')[0]
id = 12 #BOON added
print("Data of {} is successfully Loaded!".format(config.which_data))
print(type(data))
print(data.shape)
# Plot Time-series Data #
if config.plot:
plot_full(config.plots_path, data, id, config.feature)
plot_split(config.plots_path, data, id, config.valid_start, config.test_start, config.feature)
# Min-Max Scaler #
scaler = MinMaxScaler()
data.iloc[:,:] = scaler.fit_transform(data)
print(type(data))
# Split the Dataset #
train_X, train_Y, val_X, val_Y, test_X, test_Y, test_shifted = \
get_time_series_data_(data, config.valid_start, config.test_start, config.feature, config.label, config.window)
print(train_X.shape)
print(train_Y.shape)
# Get Data Loader #
train_loader, val_loader, test_loader = \
get_data_loader(train_X, train_Y, val_X, val_Y, test_X, test_Y, config.batch_size)
# Constants #
best_val_loss = 100
best_val_improv = 0
# Lists #
train_losses, val_losses = list(), list()
val_maes, val_mses, val_rmses, val_mapes, val_mpes, val_r2s = list(), list(), list(), list(), list(), list()
# Prepare Network #
if config.network == 'dnn':
model = DNN(config.window, config.hidden_size, config.output_size).to(device)
elif config.network == 'cnn':
model = CNN(config.window, config.hidden_size, config.output_size).to(device)
elif config.network == 'rnn':
model = RNN(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'lstm':
model = LSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
elif config.network == 'gru':
model = GRU(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'recursive':
model = RecursiveLSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'attentional':
model = AttentionalLSTM(config.input_size, config.key, config.query, config.value, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
else:
raise NotImplementedError
if config.mode == 'train':
# If fine-tuning #
print('config.TL = {}'.format(config.transfer_learning))
if config.transfer_learning:
print('config.TL = {}'.format(config.transfer_learning))
print('TL: True')
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_12.pkl'.format(config.network))))
for param in model.parameters():
param.requires_grad = True
# Loss Function #
criterion = torch.nn.MSELoss()
# Optimizer #
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr, betas=(0.5, 0.999))
optimizer_scheduler = get_lr_scheduler(config.lr_scheduler, optimizer, config)
# Train and Validation #
print("Training {} started with total epoch of {} using Driver ID of {}.".format(config.network, config.num_epochs, id))
for epoch in range(config.num_epochs):
# Train #
for i, (data, label) in enumerate(train_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Calculate Loss #
train_loss = criterion(pred, label)
# Back Propagation and Update #
optimizer.zero_grad()
train_loss.backward()
optimizer.step()
# Add items to Lists #
train_losses.append(train_loss.item())
print("Epoch [{}/{}]".format(epoch+1, config.num_epochs))
print("Train")
print("Loss : {:.4f}".format(np.average(train_losses)))
optimizer_scheduler.step()
# Validation #
with torch.no_grad():
for i, (data, label) in enumerate(val_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_val = model(data)
# Calculate Loss #
val_loss = criterion(pred_val, label)
val_mae = mean_absolute_error(label.cpu(), pred_val.cpu())
val_mse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=True)
val_rmse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=False)
val_mpe = mean_percentage_error(label.cpu(), pred_val.cpu())
val_mape = mean_absolute_percentage_error(label.cpu(), pred_val.cpu())
val_r2 = r2_score(label.cpu(), pred_val.cpu())
# Add item to Lists #
val_losses.append(val_loss.item())
val_maes.append(val_mae.item())
val_mses.append(val_mse.item())
val_rmses.append(val_rmse.item())
val_mpes.append(val_mpe.item())
val_mapes.append(val_mape.item())
val_r2s.append(val_r2.item())
# Print Statistics #
print("Validation")
print("Loss : {:.4f}".format(np.average(val_losses)))
print(" MAE : {:.4f}".format(np.average(val_maes)))
print(" MSE : {:.4f}".format(np.average(val_mses)))
print("RMSE : {:.4f}".format(np.average(val_rmses)))
print(" MPE : {:.4f}".format(np.average(val_mpes)))
print("MAPE : {:.4f}".format(np.average(val_mapes)))
print(" R^2 : {:.4f}".format(np.average(val_r2s)))
# Save the model only if validation loss decreased #
curr_val_loss = np.average(val_losses)
if curr_val_loss < best_val_loss:
best_val_loss = min(curr_val_loss, best_val_loss)
# if config.transfer_learning:
# torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer.pkl'.format(config.network, id)))
# else:
# torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}.pkl'.format(config.network, id)))
if config.transfer_learning:
torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer_BOON_reshaped.pkl'.format(config.network, id)))
else:
torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_BOON_reshaped.pkl'.format(config.network, id)))
print("Best model is saved!\n")
best_val_improv = 0
elif curr_val_loss >= best_val_loss:
best_val_improv += 1
print("Best Validation has not improved for {} epochs.\n".format(best_val_improv))
if best_val_improv == 10:
break
elif config.mode == 'test':
# Prepare Network #
if config.transfer_learning:
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer_BOON_reshaped.pkl'.format(config.network, id))))
else:
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_BOON_reshaped.pkl'.format(config.network, id))))
print("{} for Device ID {} is successfully loaded!".format((config.network).upper(), id))
with torch.no_grad():
pred_test, labels = list(), list()
for i, (data, label) in enumerate(test_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Add items to Lists #
pred_test += pred
labels += label
# Derive Metric and Plot #
if config.transfer_learning:
pred, actual = test(config.plots_path, id, config.network, scaler, pred_test, labels, test_shifted, transfer_learning=True)
else:
pred, actual = test(config.plots_path, id, config.network, scaler, pred_test, labels, test_shifted)
def main():
# parser
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model_config", required=False,
default="deep_dnn.json",
help="Model Architecture .json")
parser.add_argument('-s', "--setup", default="agressive_setup.json",
help="Experimental Setup .json")
args = parser.parse_args()
# config paths
model_config = 'configs/model_architecture/' + args.model_config
setup_path = "configs/experimental_setup/" + args.setup
print(model_config)
# Hyper Parameters
setup = load_config(setup_path)
train_setup = setup["Train"]
prune_setup = setup["Prune"]
batch_size = train_setup["batch_size"]
epochs = train_setup["training_epochs"]
lr = train_setup["learning_rate"]
datatype = train_setup["datatype"]
feat_size = train_setup["feature_size"]
n_samples = train_setup["n_samples"]
n_classes = train_setup["n_classes"]
val_ratio = train_setup["val_ratio"]
test_ratio = train_setup["test_ratio"]
labels = 1
if datatype == "multilabel":
labels = train_setup["labels_per_sample"]
# CUDA for PyTorch
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
# Dataloaders
train_loader, val_loader, test_loader = \
CreateRandomDataset(datatype,
feat_size,
n_samples,
n_classes,
val_ratio,
test_ratio,
batch_size,
labels).get_dataloaders()
data_loaders = {"train": train_loader,
"val": val_loader,
"test": test_loader}
# Init model
model = DNN(config=model_config,
in_features=feat_size,
n_classes=n_classes)
if torch.cuda.is_available():
print('CUDA enabled.')
model.cuda()
print("--- DNN network initialized ---")
print(model)
# Criterion/Optimizer/Pruner
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
weight_pruner = WeightPruner(model)
# Train the model from scratch
print("--- Training DNN ---")
train_losses, val_losses = \
train_eval(model, criterion, optimizer, epochs, train_loader, val_loader)
test_acc, test_loss = test(model, test_loader, criterion)
learn_curves(train_losses, val_losses, "loss_fig.png")
iterative_pruning(model, weight_pruner, criterion, data_loaders, prune_setup)
def main():
parser = argparse.ArgumentParser(description='Spoken Language Idenfication')
parser.add_argument('--dropout', type=float, default=0.2, help='dropout rate in training (default: 0.2')
parser.add_argument('--batch_size', type=int, default=64, help='batch size in training (default: 32')
parser.add_argument('--workers', type=int, default=4, help='number of workers in dataset loader (default: 4)')
parser.add_argument('--max_epochs', type=int, default=5, help='number of max epochs in training (default: 10)')
parser.add_argument('--lr', type=float, default=1e-04, help='learning rate (default: 0.0001)')
parser.add_argument('--n_class', type=int, default=2, help='number of class')
parser.add_argument('--no_cuda', action='store_true', default=False, help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)')
parser.add_argument('--nn_type', type=str, default='crnn', help='type of neural networks')
parser.add_argument('--save_name', type=str, default='model', help='the name of model')
parser.add_argument('--mode', type=str, default='train')
args = parser.parse_args()
random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
args.cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device('cuda' if args.cuda else 'cpu')
model = DNN.DNN()
model = nn.DataParallel(model).to(device)
optimizer = optim.Adam(model.module.parameters(), lr=args.lr)
criterion = nn.CrossEntropyLoss(reduction='sum').to(device)
if args.mode != 'train':
return
download_data()
kor_db_list = []
search('dataset/train/train_data', kor_db_list)
train_wav_paths = np.loadtxt("dataset/TRAIN_list.csv", delimiter=',', dtype=np.unicode)
valid_wav_paths = np.loadtxt("dataset/TEST_developmentset_list.csv", delimiter=',', dtype=np.unicode)
test_wav_paths = np.loadtxt("dataset/TEST_coreset_list.csv", delimiter=',', dtype=np.unicode)
train_wav_paths = list(map(lambda x: "dataset/TIMIT/{}.WAV".format(x), train_wav_paths))
valid_wav_paths = list(map(lambda x: "dataset/TIMIT/{}.WAV".format(x), valid_wav_paths))
test_wav_paths = list(map(lambda x: "dataset/TIMIT/{}.WAV".format(x), test_wav_paths))
min_loss = 100000
begin_epoch = 0
loss_acc = [[], [], [], []]
train_batch_num, train_dataset_list, valid_dataset, test_dataset = \
split_dataset(args, train_wav_paths, valid_wav_paths, test_wav_paths, kor_db_list)
logger.info('start')
train_begin = time.time()
for epoch in range(begin_epoch, args.max_epochs):
train_queue = queue.Queue(args.workers * 2)
train_loader = MultiLoader(train_dataset_list, train_queue, args.batch_size, args.workers, args.nn_type)
train_loader.start()
train_loss, train_acc = train(model, train_batch_num, train_queue, criterion, optimizer, device, train_begin, args.workers, 10)
logger.info('Epoch %d (Training) Loss %0.4f Acc %0.4f' % (epoch, train_loss, train_acc))
train_loader.join()
loss_acc[0].append(train_loss)
loss_acc[1].append(train_acc)
valid_queue = queue.Queue(args.workers * 2)
valid_loader = BaseDataLoader(valid_dataset, valid_queue, args.batch_size, 0, args.nn_type)
valid_loader.start()
eval_loss, eval_acc = evaluate(model, valid_loader, valid_queue, criterion, device)
logger.info('Epoch %d (Evaluate) Loss %0.4f Acc %0.4f' % (epoch, eval_loss, eval_acc))
valid_loader.join()
loss_acc[2].append(eval_loss)
loss_acc[3].append(eval_acc)
best_model = (eval_loss < min_loss)
if best_model:
min_loss = eval_loss
torch.save(model.state_dict(), './save_model/best_model.pt')
save_epoch = epoch
model.load_state_dict(torch.load('./save_model/best_model.pt'))
test_queue = queue.Queue(args.workers * 2)
test_loader = BaseDataLoader(test_dataset, test_queue, args.batch_size, 0, args.nn_type)
test_loader.start()
confusion_matrix = torch.zeros((args.n_class, args.n_class))
test_loss, test_acc = evaluate(model, test_loader, test_queue, criterion, device, confusion_matrix)
logger.info('Epoch %d (Test) Loss %0.4f Acc %0.4f' % (save_epoch, test_loss, test_acc))
test_loader.join()
save_data(loss_acc, test_loss, test_acc, confusion_matrix.to('cpu').numpy())
plot_data(loss_acc, test_loss, test_acc)
return 0
def main(args):
# Fix Seed for Reproducibility #
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
# Samples, Weights, and Plots Path #
paths = [args.weights_path, args.plots_path, args.numpy_path]
for path in paths:
make_dirs(path)
# Prepare Data #
data = load_data(args.combined_path, args.which_data, args.preprocess,
args.resample)[[args.feature]]
id = args.which_data.split('_')[0]
print("Data of {} is successfully Loaded!".format(args.which_data))
# Plot Time-series Data #
if args.plot:
plot_full(args.plots_path, data, id, args.feature)
plot_split(args.plots_path, data, id, args.valid_start,
args.test_start, args.feature)
# Min-Max Scaler #
scaler = MinMaxScaler()
data[args.feature] = scaler.fit_transform(data)
# Split the Dataset #
copied_data = data.copy()
if args.multi_step:
X, y = split_sequence_multi_step(copied_data, args.window,
args.output_size)
else:
X, y = split_sequence_uni_step(copied_data, args.window)
# Get Data Loader #
train_loader, val_loader, test_loader = get_data_loader(
X, y, args.train_split, args.test_split, args.batch_size)
# Constants #
best_val_loss = 100
best_val_improv = 0
# Lists #
train_losses, val_losses = list(), list()
val_maes, val_mses, val_rmses, val_mapes, val_mpes, val_r2s = list(), list(
), list(), list(), list(), list()
test_maes, test_mses, test_rmses, test_mapes, test_mpes, test_r2s = list(
), list(), list(), list(), list(), list()
# Prepare Network #
if args.network == 'dnn':
model = DNN(args.window, args.hidden_size, args.output_size).to(device)
elif args.network == 'cnn':
model = CNN(args.window, args.hidden_size, args.output_size).to(device)
elif args.network == 'rnn':
model = RNN(args.input_size, args.hidden_size, args.num_layers,
args.output_size).to(device)
elif args.network == 'lstm':
model = LSTM(args.input_size, args.hidden_size, args.num_layers,
args.output_size, args.bidirectional).to(device)
elif args.network == 'gru':
model = GRU(args.input_size, args.hidden_size, args.num_layers,
args.output_size).to(device)
elif args.network == 'recursive':
model = RecursiveLSTM(args.input_size, args.hidden_size,
args.num_layers, args.output_size).to(device)
elif args.network == 'attentional':
model = AttentionalLSTM(args.input_size, args.qkv, args.hidden_size,
args.num_layers, args.output_size,
args.bidirectional).to(device)
else:
raise NotImplementedError
if args.mode == 'train':
# If fine-tuning #
if args.transfer_learning:
model.load_state_dict(
torch.load(
os.path.join(
args.weights_path, 'BEST_{}_Device_ID_12.pkl'.format(
model.__class__.__name__))))
for param in model.parameters():
param.requires_grad = True
# Loss Function #
criterion = torch.nn.MSELoss()
# Optimizer #
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
betas=(0.5, 0.999))
optimizer_scheduler = get_lr_scheduler(optimizer, args)
# Train and Validation #
print(
"Training {} started with total epoch of {} using Driver ID of {}."
.format(model.__class__.__name__, args.num_epochs, id))
for epoch in range(args.num_epochs):
# Train #
for i, (data, label) in enumerate(train_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Calculate Loss #
train_loss = criterion(pred, label)
# Back Propagation and Update #
optimizer.zero_grad()
train_loss.backward()
optimizer.step()
# Add items to Lists #
train_losses.append(train_loss.item())
print("Epoch [{}/{}]".format(epoch + 1, args.num_epochs))
print("Train")
print("Loss : {:.4f}".format(np.average(train_losses)))
optimizer_scheduler.step()
# Validation #
with torch.no_grad():
for i, (data, label) in enumerate(val_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_val = model(data)
# Calculate Loss #
val_loss = criterion(pred_val, label)
val_mae = mean_absolute_error(label.cpu(), pred_val.cpu())
val_mse = mean_squared_error(label.cpu(),
pred_val.cpu(),
squared=True)
val_rmse = mean_squared_error(label.cpu(),
pred_val.cpu(),
squared=False)
# val_mpe = mean_percentage_error(label.cpu(), pred_val.cpu())
# val_mape = mean_absolute_percentage_error(label.cpu(), pred_val.cpu())
val_r2 = r2_score(label.cpu(), pred_val.cpu())
# Add item to Lists #
val_losses.append(val_loss.item())
val_maes.append(val_mae.item())
val_mses.append(val_mse.item())
val_rmses.append(val_rmse.item())
# val_mpes.append(val_mpe.item())
# val_mapes.append(val_mape.item())
val_r2s.append(val_r2.item())
# Print Statistics #
print("Validation")
print("Loss : {:.4f}".format(np.average(val_losses)))
print(" MAE : {:.4f}".format(np.average(val_maes)))
print(" MSE : {:.4f}".format(np.average(val_mses)))
print("RMSE : {:.4f}".format(np.average(val_rmses)))
# print(" MPE : {:.4f}".format(np.average(val_mpes)))
# print("MAPE : {:.4f}".format(np.average(val_mapes)))
print(" R^2 : {:.4f}".format(np.average(val_r2s)))
# Save the model only if validation loss decreased #
curr_val_loss = np.average(val_losses)
if curr_val_loss < best_val_loss:
best_val_loss = min(curr_val_loss, best_val_loss)
if args.transfer_learning:
torch.save(
model.state_dict(),
os.path.join(
args.weights_path,
'BEST_{}_Device_ID_{}_transfer.pkl'.format(
model.__class__.__name__, id)))
else:
torch.save(
model.state_dict(),
os.path.join(
args.weights_path,
'BEST_{}_Device_ID_{}.pkl'.format(
model.__class__.__name__, id)))
print("Best model is saved!\n")
best_val_improv = 0
elif curr_val_loss >= best_val_loss:
best_val_improv += 1
print("Best Validation has not improved for {} epochs.\n".
format(best_val_improv))
if best_val_improv == 10:
break
elif args.mode == 'test':
# Prepare Network #
if args.transfer_learning:
model.load_state_dict(
torch.load(
os.path.join(
args.weights_path, 'BEST_{}_Device_ID_{}.pkl'.format(
model.__class__.__name__, id))))
else:
model.load_state_dict(
torch.load(
os.path.join(
args.weights_path, 'BEST_{}_Device_ID_{}.pkl'.format(
model.__class__.__name__, id))))
print("{} for Device ID {} is successfully loaded!".format(
model.__class__.__name__, id))
with torch.no_grad():
for i, (data, label) in enumerate(test_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_test = model(data)
# Convert to Original Value Range #
pred_test = pred_test.data.cpu().numpy()
label = label.data.cpu().numpy()
if not args.multi_step:
label = label.reshape(-1, 1)
pred_test = scaler.inverse_transform(pred_test)
label = scaler.inverse_transform(label)
# Calculate Loss #
test_mae = mean_absolute_error(label, pred_test)
test_mse = mean_squared_error(label, pred_test, squared=True)
test_rmse = mean_squared_error(label, pred_test, squared=False)
# test_mpe = mean_percentage_error(label, pred_test)
# test_mape = mean_absolute_percentage_error(label, pred_test)
test_r2 = r2_score(label, pred_test)
# Add item to Lists #
test_maes.append(test_mae.item())
test_mses.append(test_mse.item())
test_rmses.append(test_rmse.item())
# test_mpes.append(test_mpe.item())
# test_mapes.append(test_mape.item())
test_r2s.append(test_r2.item())
# Print Statistics #
print("Test {}".format(model.__class__.__name__))
print(" MAE : {:.4f}".format(np.average(test_maes)))
print(" MSE : {:.4f}".format(np.average(test_mses)))
print("RMSE : {:.4f}".format(np.average(test_rmses)))
# print(" MPE : {:.4f}".format(np.average(test_mpes)))
# print("MAPE : {:.4f}".format(np.average(test_mapes)))
print(" R^2 : {:.4f}".format(np.average(test_r2s)))
# Derive Metric and Plot #
if args.transfer_learning:
test_plot(pred_test,
label,
args.plots_path,
args.feature,
id,
model,
transfer_learning=False)
else:
test_plot(pred_test,
label,
args.plots_path,
args.feature,
id,
model,
transfer_learning=False)
def main(args):
# Fix Seed #
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
# Weights and Plots Path #
paths = [args.weights_path, args.plots_path, args.numpy_path]
for path in paths:
make_dirs(path)
# Prepare Data #
data = load_data(args.which_data)[[args.feature]]
data = data.copy()
# Plot Time-Series Data #
if args.plot_full:
plot_full(args.plots_path, data, args.feature)
scaler = MinMaxScaler()
data[args.feature] = scaler.fit_transform(data)
# Split the Dataset #
copied_data = data.copy().values
if args.multi_step:
X, y = split_sequence_multi_step(copied_data, args.seq_length,
args.output_size)
step = 'MultiStep'
else:
X, y = split_sequence_uni_step(copied_data, args.seq_length)
step = 'SingleStep'
train_loader, val_loader, test_loader = data_loader(
X, y, args.train_split, args.test_split, args.batch_size)
# Lists #
train_losses, val_losses = list(), list()
val_maes, val_mses, val_rmses, val_mapes, val_mpes, val_r2s = list(), list(
), list(), list(), list(), list()
test_maes, test_mses, test_rmses, test_mapes, test_mpes, test_r2s = list(
), list(), list(), list(), list(), list()
pred_tests, labels = list(), list()
# Constants #
best_val_loss = 100
best_val_improv = 0
# Prepare Network #
if args.model == 'dnn':
model = DNN(args.seq_length, args.hidden_size,
args.output_size).to(device)
elif args.model == 'cnn':
model = CNN(args.seq_length, args.batch_size,
args.output_size).to(device)
elif args.model == 'rnn':
model = RNN(args.input_size, args.hidden_size, args.num_layers,
args.output_size).to(device)
elif args.model == 'lstm':
model = LSTM(args.input_size, args.hidden_size, args.num_layers,
args.output_size, args.bidirectional).to(device)
elif args.model == 'gru':
model = GRU(args.input_size, args.hidden_size, args.num_layers,
args.output_size).to(device)
elif args.model == 'attentional':
model = AttentionalLSTM(args.input_size, args.qkv, args.hidden_size,
args.num_layers, args.output_size,
args.bidirectional).to(device)
else:
raise NotImplementedError
# Loss Function #
criterion = torch.nn.MSELoss()
# Optimizer #
optim = torch.optim.Adam(model.parameters(),
lr=args.lr,
betas=(0.5, 0.999))
optim_scheduler = get_lr_scheduler(args.lr_scheduler, optim)
# Train and Validation #
if args.mode == 'train':
# Train #
print("Training {} using {} started with total epoch of {}.".format(
model.__class__.__name__, step, args.num_epochs))
for epoch in range(args.num_epochs):
for i, (data, label) in enumerate(train_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Calculate Loss #
train_loss = criterion(pred, label)
# Initialize Optimizer, Back Propagation and Update #
optim.zero_grad()
train_loss.backward()
optim.step()
# Add item to Lists #
train_losses.append(train_loss.item())
# Print Statistics #
if (epoch + 1) % args.print_every == 0:
print("Epoch [{}/{}]".format(epoch + 1, args.num_epochs))
print("Train Loss {:.4f}".format(np.average(train_losses)))
# Learning Rate Scheduler #
optim_scheduler.step()
# Validation #
with torch.no_grad():
for i, (data, label) in enumerate(val_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_val = model(data)
# Calculate Loss #
val_loss = criterion(pred_val, label)
if args.multi_step:
pred_val = np.mean(pred_val.detach().cpu().numpy(),
axis=1)
label = np.mean(label.detach().cpu().numpy(), axis=1)
else:
pred_val, label = pred_val.cpu(), label.cpu()
# Calculate Metrics #
val_mae = mean_absolute_error(label, pred_val)
val_mse = mean_squared_error(label, pred_val, squared=True)
val_rmse = mean_squared_error(label,
pred_val,
squared=False)
val_mpe = mean_percentage_error(label, pred_val)
val_mape = mean_absolute_percentage_error(label, pred_val)
val_r2 = r2_score(label, pred_val)
# Add item to Lists #
val_losses.append(val_loss.item())
val_maes.append(val_mae.item())
val_mses.append(val_mse.item())
val_rmses.append(val_rmse.item())
val_mpes.append(val_mpe.item())
val_mapes.append(val_mape.item())
val_r2s.append(val_r2.item())
if (epoch + 1) % args.print_every == 0:
# Print Statistics #
print("Val Loss {:.4f}".format(np.average(val_losses)))
print(" MAE : {:.4f}".format(np.average(val_maes)))
print(" MSE : {:.4f}".format(np.average(val_mses)))
print("RMSE : {:.4f}".format(np.average(val_rmses)))
print(" MPE : {:.4f}".format(np.average(val_mpes)))
print("MAPE : {:.4f}".format(np.average(val_mapes)))
print(" R^2 : {:.4f}".format(np.average(val_r2s)))
# Save the model only if validation loss decreased #
curr_val_loss = np.average(val_losses)
if curr_val_loss < best_val_loss:
best_val_loss = min(curr_val_loss, best_val_loss)
torch.save(
model.state_dict(),
os.path.join(
args.weights_path, 'BEST_{}_using_{}.pkl'.format(
model.__class__.__name__, step)))
print("Best model is saved!\n")
best_val_improv = 0
elif curr_val_loss >= best_val_loss:
best_val_improv += 1
print("Best Validation has not improved for {} epochs.\n".
format(best_val_improv))
elif args.mode == 'test':
# Load the Model Weight #
model.load_state_dict(
torch.load(
os.path.join(
args.weights_path,
'BEST_{}_using_{}.pkl'.format(model.__class__.__name__,
step))))
# Test #
with torch.no_grad():
for i, (data, label) in enumerate(test_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_test = model(data)
# Convert to Original Value Range #
pred_test, label = pred_test.detach().cpu().numpy(
), label.detach().cpu().numpy()
pred_test = scaler.inverse_transform(pred_test)
label = scaler.inverse_transform(label)
if args.multi_step:
pred_test = np.mean(pred_test, axis=1)
label = np.mean(label, axis=1)
pred_tests += pred_test.tolist()
labels += label.tolist()
# Calculate Loss #
test_mae = mean_absolute_error(label, pred_test)
test_mse = mean_squared_error(label, pred_test, squared=True)
test_rmse = mean_squared_error(label, pred_test, squared=False)
test_mpe = mean_percentage_error(label, pred_test)
test_mape = mean_absolute_percentage_error(label, pred_test)
test_r2 = r2_score(label, pred_test)
# Add item to Lists #
test_maes.append(test_mae.item())
test_mses.append(test_mse.item())
test_rmses.append(test_rmse.item())
test_mpes.append(test_mpe.item())
test_mapes.append(test_mape.item())
test_r2s.append(test_r2.item())
# Print Statistics #
print("Test {} using {}".format(model.__class__.__name__, step))
print(" MAE : {:.4f}".format(np.average(test_maes)))
print(" MSE : {:.4f}".format(np.average(test_mses)))
print("RMSE : {:.4f}".format(np.average(test_rmses)))
print(" MPE : {:.4f}".format(np.average(test_mpes)))
print("MAPE : {:.4f}".format(np.average(test_mapes)))
print(" R^2 : {:.4f}".format(np.average(test_r2s)))
# Plot Figure #
plot_pred_test(pred_tests[:args.time_plot],
labels[:args.time_plot], args.plots_path,
args.feature, model, step)
# Save Numpy files #
np.save(
os.path.join(
args.numpy_path,
'{}_using_{}_TestSet.npy'.format(model.__class__.__name__,
step)),
np.asarray(pred_tests))
np.save(
os.path.join(args.numpy_path,
'TestSet_using_{}.npy'.format(step)),
np.asarray(labels))
else:
raise NotImplementedError
def main():
parser = argparse.ArgumentParser(
description='Emotion recognition on CMU-MOSEI')
parser.add_argument('--seed', default=5566, type=int, help="Random seed")
parser.add_argument('--save_dir', default='./results', type=str, help="")
parser.add_argument('--num_epochs',
default=20,
type=int,
help="Number of training epochs")
parser.add_argument('--dropout', default=0.5, type=float, help="")
parser.add_argument('--min_ir',
default=2,
type=float,
help="Minimum imbalance ratio")
parser.add_argument('--lr', default=0.5, type=float, help="")
parser.add_argument('--activation', default='relu', type=str, help="")
parser.add_argument('--batch_size', default=32, type=int, help="")
parser.add_argument('--layers',
default='512.512.256.256.128.128',
type=str,
help="Comma-separted list of hidden dimensions")
parser.add_argument('--gamma',
default=1,
type=float,
help="Weight for negative class")
parser.add_argument('--dataset', default=None, type=str, help="Dataset")
parser.add_argument('--verbose',
default=False,
action='store_true',
help="Verbose")
args = parser.parse_args()
options['num_epochs'] = args.num_epochs
options['dropout'] = args.dropout
options['activation'] = args.activation
options['batch_size'] = args.batch_size
options['layers'] = [int(x) for x in args.layers.split('.')]
options['gamma'] = args.gamma
options['lr'] = args.lr
options['min_ir'] = args.min_ir
grad_clip_value = 10.0
CUDA = True
import torch
torch.manual_seed(args.seed)
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam, Adagrad
from torch.utils.data import Dataset, DataLoader
from losses import WeightedBCELoss
from models import DNN, mosei_dataset
n_epochs = options['num_epochs']
batch_size = options['batch_size']
verbose = args.verbose
save_dir = args.save_dir
ckpt_path = os.path.join(save_dir, "checkpoint.pt")
if not os.path.exists(save_dir):
os.makedirs(save_dir)
with open(os.path.join(save_dir, 'options.json'), 'w') as fout:
fout.write(json.dumps(options))
class_weight = torch.Tensor(options['class_weights'])
class_weight = class_weight / torch.sum(class_weight)
gamma = torch.Tensor([options['gamma']])
if CUDA:
class_weight = class_weight.cuda()
gamma = gamma.cuda()
model = DNN(options)
if CUDA:
model.cuda()
if verbose: print(model)
model.train()
dataset = pickle.load(open(args.dataset, "rb"))
train_dataset = mosei_dataset(dataset,
splits=train_split,
oversample=True,
min_ir=options['min_ir'])
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=4)
if verbose: print(f"Train set: {len(train_dataset)} samples")
val_dataset = mosei_dataset(dataset, splits=dev_split)
val_loader = DataLoader(val_dataset,
batch_size=256,
shuffle=False,
num_workers=4)
if verbose: print(f"Val set: {len(val_dataset)} samples")
test_dataset = mosei_dataset(dataset, splits=test_split)
test_loader = DataLoader(test_dataset,
batch_size=256,
shuffle=False,
num_workers=4)
if verbose: print(f"Test set: {len(test_dataset)} samples")
optimizer = Adam(
[param for param in model.parameters() if param.requires_grad],
lr=options['lr'])
criterion = WeightedBCELoss(class_weight=class_weight,
PosWeightIsDynamic=True,
gamma=gamma)
best_val = np.Inf
best_metric = 0
train_labels = []
for epoch_no in range(n_epochs):
total_pos = 0
model.train()
for batch_no, batch in enumerate(train_loader, start=1):
embeddings, labels = batch
if CUDA:
embeddings, labels = embeddings.cuda(), labels.cuda()
y_hat = model(embeddings)
loss = criterion(y_hat, labels)
loss.backward()
# torch.nn.utils.clip_grad_norm_([param for param in model.parameters() if param.requires_grad], grad_clip_value)
optimizer.step()
optimizer.zero_grad()
if batch_no % 200 == 0:
if verbose: print(f"Training loss: {loss.item():.5f}")
y_true, y_pred, val_loss = [], [], []
model.eval()
for batch in val_loader:
embeddings, labels = batch
if CUDA:
embeddings, labels = embeddings.cuda(), labels.cuda()
y_hat = model(embeddings)
loss = criterion(y_hat, labels)
val_loss.append(loss.item())
y_true.append(labels.detach().cpu().numpy())
y_pred.append(y_hat.detach().cpu().numpy())
assert not np.any(np.isnan(val_loss))
y_true = np.concatenate(y_true, axis=0).squeeze()
y_pred = np.concatenate(y_pred, axis=0).squeeze()
y_true_bin = y_true > 0
y_pred_bin = y_pred > 0
val_loss = np.average(val_loss)
f1score = [
f1_score(t, p, average="weighted")
for t, p in zip(y_true_bin.T, y_pred_bin.T)
]
wa = np.average(weighted_accuracy(y_true_bin, y_pred_bin))
val_metric = np.average(f1score) + np.average(wa)
f1score = [f'{x*100:.2f}' for x in f1score]
if verbose: print(f"Validation loss: {val_loss:.3f}")
if best_metric < val_metric:
if verbose: print("Validation metric improved")
best_metric = val_metric
checkpoint = {
'options': options,
'model': model,
'epoch': epoch_no
}
torch.save(checkpoint, ckpt_path)
model.train()
# ====================================================================================================
# Final Validation
# ====================================================================================================
checkpoint = torch.load(ckpt_path)
model = checkpoint['model']
if verbose:
print("Loaded best model from epoch {}".format(checkpoint['epoch']))
model.eval()
val_true = []
val_pred = []
for batch in val_loader:
embeddings, labels = batch
if CUDA:
embeddings, labels = embeddings.cuda(), labels.cuda()
val_hat = model(embeddings)
val_true.append(labels.detach().cpu().numpy())
val_pred.append(val_hat.detach().cpu().numpy())
val_true = np.concatenate(val_true, axis=0).squeeze()
val_pred = np.concatenate(val_pred, axis=0).squeeze()
val_true_bin = val_true > 0
val_pred_bin = val_pred > 0
wa = [
weighted_accuracy(t, p) * 100
for t, p in zip(val_true_bin.T, val_pred_bin.T)
]
f1score = [
f1_score(t, p, average="weighted") * 100
for t, p in zip(val_true_bin.T, val_pred_bin.T)
]
if verbose:
print(f"Val WA, {reformat_array(wa)} Avg: {np.average(wa):.2f}")
if verbose:
print(
f"Val F1, {reformat_array(f1score)} Avg: {np.average(f1score):.2f} "
)
# ====================================================================================================
# Final Test
# ====================================================================================================
test_true = []
test_pred = []
for batch in test_loader:
embeddings, labels = batch
if CUDA:
embeddings, labels = embeddings.cuda(), labels.cuda()
pred = model(embeddings)
test_true.append(labels.detach().cpu().numpy())
test_pred.append(pred.detach().cpu().numpy())
test_true = np.concatenate(test_true, axis=0).squeeze()
test_pred = np.concatenate(test_pred, axis=0).squeeze()
test_true_bin = test_true > 0 # Binarized
test_pred_bin = test_pred > 0 # Logit outputs
test_wa = [
weighted_accuracy(t, p)
for t, p in zip(test_true_bin.T, test_pred_bin.T)
]
test_wa = reorder_labels(test_wa)
test_f1score = [
f1_score(t, p, average="weighted")
for t, p in zip(test_true_bin.T, test_pred_bin.T)
]
test_f1score = reorder_labels(test_f1score)
test_wa_str = [f'{x*100:.2f}' for x in test_wa]
test_f1score_str = [f'{x*100:.2f}' for x in test_f1score]
print(f"Test WA: {test_wa_str} Avg: {np.average(test_wa)*100:2.1f}")
print(
f"Test F1: {test_f1score_str} Avg: {np.average(test_f1score)*100:2.1f}"
)
combined = [f" {x} & {y} " for x, y in zip(test_wa_str, test_f1score_str)]
combined.append(
f" {np.average(test_wa)*100:2.1f} & {np.average(test_f1score)*100:2.1f}"
)
if verbose: print("&".join(combined))
model.compile(loss='categorical_crossentropy',
optimizer=opt(lr=lr),
metrics=['accuracy'] + grad_son.metrics)
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
callbacks=[grad_son])
# Train DNN
fname = '_'.join(['dnn', opt.__name__.lower(), str(lr), act])
fname = path + fname + '.wav'
print(fname)
model = DNN(input_shape=(28 * 28, ), activation=act)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=opt(lr=lr),
metrics=['accuracy'])
grad_son = GradientSonification(path=fname,
model=model,
fs=fs,
duration=duration,
freq=freq)
model.compile(loss='categorical_crossentropy',
optimizer=opt(lr=lr),
metrics=['accuracy'] + grad_son.metrics)
def load_models(self, dance_model_path):
dance_model = DNN()
dance_model.setup(dance_model_path)
self.dance_model = dance_model
def main(config):
# Fix Seed #
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed(config.seed)
# Weights and Plots Path #
paths = [config.weights_path, config.plots_path]
for path in paths:
make_dirs(path)
# Prepare Data #
data = load_data(config.which_data)[[config.feature]]
data = data.copy()
# Plot Time-Series Data #
if config.plot_full:
plot_full(config.plots_path, data, config.feature)
scaler = MinMaxScaler()
data[config.feature] = scaler.fit_transform(data)
train_loader, val_loader, test_loader = \
data_loader(data, config.seq_length, config.train_split, config.test_split, config.batch_size)
# Lists #
train_losses, val_losses = list(), list()
val_maes, val_mses, val_rmses, val_mapes, val_mpes, val_r2s = list(), list(), list(), list(), list(), list()
test_maes, test_mses, test_rmses, test_mapes, test_mpes, test_r2s = list(), list(), list(), list(), list(), list()
# Constants #
best_val_loss = 100
best_val_improv = 0
# Prepare Network #
if config.network == 'dnn':
model = DNN(config.seq_length, config.hidden_size, config.output_size).to(device)
elif config.network == 'cnn':
model = CNN(config.seq_length, config.batch_size).to(device)
elif config.network == 'rnn':
model = RNN(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'lstm':
model = LSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
elif config.network == 'gru':
model = GRU(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'recursive':
model = RecursiveLSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'attention':
model = AttentionLSTM(config.input_size, config.key, config.query, config.value, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
else:
raise NotImplementedError
# Loss Function #
criterion = torch.nn.MSELoss()
# Optimizer #
optim = torch.optim.Adam(model.parameters(), lr=config.lr, betas=(0.5, 0.999))
optim_scheduler = get_lr_scheduler(config.lr_scheduler, optim)
# Train and Validation #
if config.mode == 'train':
# Train #
print("Training {} started with total epoch of {}.".format(model.__class__.__name__, config.num_epochs))
for epoch in range(config.num_epochs):
for i, (data, label) in enumerate(train_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Calculate Loss #
train_loss = criterion(pred, label)
# Initialize Optimizer, Back Propagation and Update #
optim.zero_grad()
train_loss.backward()
optim.step()
# Add item to Lists #
train_losses.append(train_loss.item())
# Print Statistics #
if (epoch+1) % config.print_every == 0:
print("Epoch [{}/{}]".format(epoch+1, config.num_epochs))
print("Train Loss {:.4f}".format(np.average(train_losses)))
# Learning Rate Scheduler #
optim_scheduler.step()
# Validation #
with torch.no_grad():
for i, (data, label) in enumerate(val_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_val = model(data)
# Calculate Loss #
val_loss = criterion(pred_val, label)
val_mae = mean_absolute_error(label.cpu(), pred_val.cpu())
val_mse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=True)
val_rmse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=False)
val_mpe = mean_percentage_error(label.cpu(), pred_val.cpu())
val_mape = mean_absolute_percentage_error(label.cpu(), pred_val.cpu())
val_r2 = r2_score(label.cpu(), pred_val.cpu())
# Add item to Lists #
val_losses.append(val_loss.item())
val_maes.append(val_mae.item())
val_mses.append(val_mse.item())
val_rmses.append(val_rmse.item())
val_mpes.append(val_mpe.item())
val_mapes.append(val_mape.item())
val_r2s.append(val_r2.item())
if (epoch + 1) % config.print_every == 0:
# Print Statistics #
print("Val Loss {:.4f}".format(np.average(val_losses)))
print("Val MAE : {:.4f}".format(np.average(val_maes)))
print("Val MSE : {:.4f}".format(np.average(val_mses)))
print("Val RMSE : {:.4f}".format(np.average(val_rmses)))
print("Val MPE : {:.4f}".format(np.average(val_mpes)))
print("Val MAPE : {:.4f}".format(np.average(val_mapes)))
print("Val R^2 : {:.4f}".format(np.average(val_r2s)))
# Save the model Only if validation loss decreased #
curr_val_loss = np.average(val_losses)
if curr_val_loss < best_val_loss:
best_val_loss = min(curr_val_loss, best_val_loss)
torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}.pkl'.format(model.__class__.__name__)))
print("Best model is saved!\n")
best_val_improv = 0
elif curr_val_loss >= best_val_loss:
best_val_improv += 1
print("Best Validation has not improved for {} epochs.\n".format(best_val_improv))
elif config.mode == 'test':
# Load the Model Weight #
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}.pkl'.format(model.__class__.__name__))))
# Test #
with torch.no_grad():
for i, (data, label) in enumerate(test_loader):
# Prepare Data #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_test = model(data)
# Convert to Original Value Range #
pred_test = pred_test.data.cpu().numpy()
label = label.data.cpu().numpy().reshape(-1, 1)
pred_test = scaler.inverse_transform(pred_test)
label = scaler.inverse_transform(label)
# Calculate Loss #
test_mae = mean_absolute_error(label, pred_test)
test_mse = mean_squared_error(label, pred_test, squared=True)
test_rmse = mean_squared_error(label, pred_test, squared=False)
test_mpe = mean_percentage_error(label, pred_test)
test_mape = mean_absolute_percentage_error(label, pred_test)
test_r2 = r2_score(label, pred_test)
# Add item to Lists #
test_maes.append(test_mae.item())
test_mses.append(test_mse.item())
test_rmses.append(test_rmse.item())
test_mpes.append(test_mpe.item())
test_mapes.append(test_mape.item())
test_r2s.append(test_r2.item())
# Print Statistics #
print("Test {}".format(model.__class__.__name__))
print("Test MAE : {:.4f}".format(np.average(test_maes)))
print("Test MSE : {:.4f}".format(np.average(test_mses)))
print("Test RMSE : {:.4f}".format(np.average(test_rmses)))
print("Test MPE : {:.4f}".format(np.average(test_mpes)))
print("Test MAPE : {:.4f}".format(np.average(test_mapes)))
print("Test R^2 : {:.4f}".format(np.average(test_r2s)))
# Plot Figure #
plot_pred_test(pred_test, label, config.plots_path, config.feature, model)