def train_nn(n_epochs=2, log_interval=10):
df = pd.read_csv('./data/df_super.csv')
train_df, val_df, test_df = data.data_split(data.create_nn_dataset(df))
model = LSTM(output_size=64)
model, history = train_loop(model,
train_df,
val_df,
n_epochs=n_epochs,
log_interval=log_interval)
torch.save({'model_state_dict': model.state_dict()}, 'nn.hdf5')
def main():
args = parser.parse_args()
print args
corpus_file = 'data/android/corpus.tsv.gz'
dataset = AndroidDataset(corpus_file)
corpus = dataset.get_corpus()
if args.embedding == 'askubuntu':
embedding_file = 'data/askubuntu/vector/vectors_pruned.200.txt.gz'
else:
embedding_file = 'data/glove/glove.pruned.txt.gz'
embedding_iter = Embedding.iterator(embedding_file)
embedding = Embedding(args.embed, embedding_iter)
print 'Embeddings loaded.'
corpus_ids = embedding.corpus_to_ids(corpus)
padding_id = embedding.vocab_ids['<padding>']
dev_pos_file = 'data/android/dev.pos.txt'
dev_neg_file = 'data/android/dev.neg.txt'
dev_data = dataset.read_annotations(dev_pos_file, dev_neg_file)
test_pos_file = 'data/android/test.pos.txt'
test_neg_file = 'data/android/test.neg.txt'
test_data = dataset.read_annotations(test_pos_file, test_neg_file)
dev_batches = batch_utils.generate_eval_batches(corpus_ids, dev_data,
padding_id)
test_batches = batch_utils.generate_eval_batches(corpus_ids, test_data,
padding_id)
if os.path.isfile(args.load):
if args.model == 'lstm':
model = LSTM(args.embed, args.hidden)
else:
model = CNN(args.embed, args.hidden)
checkpoint = torch.load(args.load)
model.load_state_dict(checkpoint['state_dict'])
else:
print 'No checkpoint found here.'
print 'Evaluating on dev set.'
train_utils.evaluate_auc(args, model, embedding, dev_batches, padding_id)
print 'Evaluating on test set.'
train_utils.evaluate_auc(args, model, embedding, test_batches, padding_id)
return
def __init__(self,
Data,
lstm_neurons,
epochs,
ax,
gradient=0.9056,
dense_neruons=3):
self.lstm = LSTM(lstm_neurons, dense_neruons)
self.lstm.create_model()
self.lstm.fit_model(epochs, Data)
self.lstm.print_stats()
self.states = States(4000, 23000)
self.states.create_unperturbed(self.lstm, Data)
self.states.create_perturbed(self.lstm, Data)
self.Lyapunov = Lyapunov(self.states)
self.Lyapunov.plot_exponent(ax, gradient)
def __init__(self, state_shape, action_shape, stats):
# self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.device = torch.device("cpu")
self.state_shape = state_shape
self.action_shape = action_shape
self.stats = stats
self.learn_rate = 3e-4
self.num_epochs = 8
self.entropy_weight = 0.001
self.kl_clip = 0.1
self.deterministic_test_mode = False
self.hidden_state_size = 16
self.lstm = LSTM(self.state_shape, self.hidden_state_size)
self.actor = Actor(self.hidden_state_size,
self.action_shape).to(self.device)
self.critic = Critic(self.hidden_state_size).to(self.device)
self.optimizer = torch.optim.Adam(list(self.actor.parameters()) +
list(self.critic.parameters()),
lr=self.learn_rate)
def init_model(self, hparams):
if self.config.exp.model == 'xgboost':
model = XGBoost(hparams)
elif self.config.exp.model == 'lstm':
model = LSTM(hparams)
elif self.config.exp.model == 'transformer':
model = TransformerNet(hparams)
return model
def train(args):
src_root = args.src_root
sr = args.sample_rate
dt = args.delta_time
batch_size = args.batch_size
model_type = args.model_type
params = {'N_CLASSES': len(os.listdir(args.src_root)), 'SR': sr, 'DT': dt}
models = {
'conv1d': Conv1D(**params),
'conv2d': Conv2D(**params),
'lstm': LSTM(**params)
}
assert model_type in models.keys(), '{} not an available model'.format(
model_type)
csv_path = os.path.join('logs', '{}_history.csv'.format(model_type))
wav_paths = glob('{}/**'.format(src_root), recursive=True)
wav_paths = [x.replace(os.sep, '/') for x in wav_paths if '.wav' in x]
classes = sorted(os.listdir(args.src_root))
le = LabelEncoder()
le.fit(classes)
labels = [os.path.split(x)[0].split('/')[-1] for x in wav_paths]
labels = le.transform(labels)
wav_train, wav_val, label_train, label_val = train_test_split(
wav_paths, labels, test_size=0.1, random_state=0)
assert len(
label_train
) >= args.batch_size, 'Anzahl der trainierten Dateien muss >= batch_size sein'
tg = DataGenerator(wav_train,
label_train,
sr,
dt,
len(set(label_train)),
batch_size=batch_size)
vg = DataGenerator(wav_val,
label_val,
sr,
dt,
len(set(label_val)),
batch_size=batch_size)
model = models[model_type]
cp = ModelCheckpoint('models/{}.h5'.format(model_type),
monitor='val_loss',
save_best_only=True,
save_weights_only=False,
mode='auto',
save_freq='epoch',
verbose=1)
csv_logger = CSVLogger(csv_path, append=False)
model.fit(tg,
validation_data=vg,
epochs=30,
verbose=1,
callbacks=[csv_logger, cp])
class Predict:
def __init__(self, Data, lstm_neurons, epochs, dense_neurons=3):
self.lstm = LSTM(Data, lstm_neurons, epochs)
self.lstm.fit_model(False)
self.PredStates()
print(self.unperturbed.shape)
def PredStates(self):
old_state = tf.expand_dims(
self.lstm.Data.datapoints[200000:200000 +
self.lstm.Data.time_steps], 0)
predicted_states = []
for i in range(300):
new_state = self.lstm.model.predict(
old_state, batch_size=self.lstm.Data.batch_size) + old_state
old_state = new_state
predicted_states.append(np.squeeze(np.squeeze(old_state, 0), 0))
self.unperturbed = np.array(predicted_states)
def main(data_dir):
BATCH_SIZE = 16
N_SAMPLES_IN = 40 # divide by 10 to get # hours the sequence covers
N_SAMPLES_OUT = 5
generator = MyData(
data_dir,
BATCH_SIZE,
N_SAMPLES_IN,
N_SAMPLES_OUT
)
lstm = LSTM(N_SAMPLES_IN, N_SAMPLES_OUT, 1, 1)
lstm.build()
lstm.compile()
lstm.model.fit(generator,
epochs=6,
shuffle=True,
verbose=1,
steps_per_epoch=len(generator))
lstm.model.save('models/lstm_%d_%d_%d.h5' % (
BATCH_SIZE, N_SAMPLES_IN, N_SAMPLES_OUT
))
def train_model(data_pack, num_epochs, learning_rate, num_words, dim_embedding, num_classes, model_name):
train_X, train_y, valid_X, valid_y, test_X, test_y = data_pack
if model_name == "Baseline-BoW":
model = Bag_of_Words(num_words, num_classes)
elif model_name == "Baseline-AvEmbedding":
model = Baseline(num_words, dim_embedding, num_classes)
elif model_name == "Shallow-CNN":
n_filters = [40, 40]
model = CNN(num_words, dim_embedding, num_classes, n_filters)
elif model_name == "Deep-CNN":
n_filters = [40, 48, 72, 48]
model = CNN_Deep(num_words, dim_embedding, num_classes, n_filters)
elif model_name == "Shallow-LSTM":
memory_size = 100
model = LSTM(num_words, dim_embedding, num_classes, memory_size)
elif model_name == "Deep-LSTM":
memory_size = 100
model = LSTM_Deep(num_words, dim_embedding, num_classes, memory_size)
elif model_name == "Shallow-CNN-CE":
n_filters = [40, 40]
model = CE_CNN(dim_embedding, num_classes, n_filters)
elif model_name == "Deep-CNN-CE":
n_filters = [40, 48, 72, 48]
model = CE_CNN_Deep(dim_embedding, num_classes, n_filters)
elif model_name == "Block-CNN-CE":
n_filters = [64, 128, 256, 512]
model = CE_CNN_Block(dim_embedding, num_classes, n_filters)
elif model_name == "ResNet-CE":
n_filters = [64, 128, 256, 512]
model = CE_ResNet(dim_embedding, num_classes, n_filters)
model.cuda()
# n_filters = [15, 20, 40]
# model = CNN_Deep(num_words, dim_embedding, num_classes, n_filters)
max_train, max_val, max_test = 0, 0, 0
min_train, min_val, min_test = 10, 10, 10
model = torch.load(model_name + ".pt")
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
model.eval()
criterion = torch.nn.CrossEntropyLoss()
a = []
batch_x_one = torch.FloatTensor(batch_size, test_X[0].shape[1], dim_embedding)
t_acc, output_results = run_example_set(model, criterion, test_X, test_y, batch_x_one=batch_x_one)
print(output_results)
results = open("results/example_set_prediction.txt", "w")
for e in output_results:
results.write(str(e) + "\n")
print(str(t_acc))
def main(args):
if args.model == 'LSTM':
model = LSTM(input_dim=args.input_dim, lstm_hidden_dim=args.lstm_hidden_dim, time_step=args.time_step)
else:
raise ValueError
model.cuda()
if args.optim == 'SGD':
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
elif args.optim == 'SGD_momentum':
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.weight_decay)
elif args.optim == 'Adagrad':
optimizer = torch.optim.Adagrad(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
elif args.optim == 'RMSprop':
optimizer = torch.optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.999, eps=1e-8, weight_decay=args.weight_decay)
elif args.optim == 'Adam':
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-8, weight_decay=args.weight_decay)
else:
raise ValueError
lr_scheduler = None
if args.load_path:
if args.recover:
load_model(model, args.load_path, strict=True)
print('load model state dict in {}'.format(args.load_path))
map_file_path = 'divide.csv'
data_file_path = 'processed_data.txt'
social_economical_path = '2010-2016.csv'
if args.dataset == 'NaiveDataset':
train_set = NaiveDataset(data_file_path, map_file_path)
elif args.dataset == 'AdvancedDataset':
train_set = AdvancedDataset(data_file_path, map_file_path, social_economical_path)
else:
raise ValueError
train_dataloader = DataLoader(train_set, batch_size=args.batch_size,
shuffle=True, num_workers=args.num_workers,
pin_memory=True)
if args.evaluate:
validate(train_dataloader, model)
return
train(train_dataloader, train_dataloader, model, optimizer, lr_scheduler, args)
def evaluate(args):
label_map = load_label_map(args.dataset)
n_classes = 50
if args.dataset == "include":
n_classes = 263
if args.use_cnn:
dataset = FeaturesDatset(
features_dir=os.path.join(args.data_dir,
f"{args.dataset}_test_features"),
label_map=label_map,
mode="test",
)
else:
dataset = KeypointsDataset(
keypoints_dir=os.path.join(args.data_dir,
f"{args.dataset}_test_keypoints"),
use_augs=False,
label_map=label_map,
mode="test",
max_frame_len=169,
)
dataloader = data.DataLoader(
dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
)
if args.model == "lstm":
config = LstmConfig()
if args.use_cnn:
config.input_size = CnnConfig.output_dim
model = LSTM(config=config, n_classes=n_classes)
else:
config = TransformerConfig(size=args.transformer_size)
if args.use_cnn:
config.input_size = CnnConfig.output_dim
model = Transformer(config=config, n_classes=n_classes)
model = model.to(device)
if args.use_pretrained == "evaluate":
model, _, _ = load_pretrained(args, n_classes, model)
print("### Model loaded ###")
else:
exp_name = get_experiment_name(args)
model_path = os.path.join(args.save_path, exp_name) + ".pth"
ckpt = torch.load(model_path)
model.load_state_dict(ckpt["model"])
print("### Model loaded ###")
test_loss, test_acc = validate(dataloader, model, device)
print("Evaluation Results:")
print(f"Loss: {test_loss}, Accuracy: {test_acc}")
def create_model():
if args.model_type == 'cnn_net':
model = CNN(args.vocab_size, args.padding_size)
elif args.model_type == 'bow_net':
model = BOW(args.vocab_size, args.padding_size)
elif args.model_type == 'lstm_net':
model = LSTM(args.vocab_size, args.padding_size)
elif args.model_type == 'gru_net':
model = GRU(args.vocab_size, args.padding_size)
elif args.model_type == 'bigru_net':
model = BiGRU(args.vocab_size, args.batch_size, args.padding_size)
else:
raise ValueError("Unknown model type!")
return model
def __init__(self, window_size=3) -> None:
self.window_size = window_size
self.dataset = SupervisedDataset(mode='test',
window_size=self.window_size,
log_reg=False)
self.loader = DataLoader(self.dataset, batch_size=64, shuffle=False)
self.checkpoint_path = "./model_params/val/blstm_bs64_lr1e-3_ws40_hs128_nl2_dout50/val_lstm_epoch91_acc81.9593.pth"
#self.checkpoint_path = "./model_params/logistic_regression/val/val_logreg_epoch100_acc35.8229.pth"
self.checkpoint = torch.load(self.checkpoint_path)
self.model = LSTM(input_size=78,
hidden_size=128,
num_classes=170,
n_layers=2).to(device=torch.device('cuda:0'))
#self.model = LogisticRegression(num_keypoints=78, num_features=2, num_classes=170).to(device=torch.device('cuda:0'))
self.model.load_state_dict(self.checkpoint, strict=True)
self.model.eval()
#self.criterion = nn.BCEWithLogitsLoss() # Use this for Logistic Regression training
self.criterion = nn.BCELoss(
) # Use this for LSTM training (with Softmax)
def train(args):
src_root = args.src_root
sr = args.sample_rate
dt = args.delta_time
batch_size = args.batch_size
model_type = args.model_type
models = {'conv1d': Conv1D(), 'conv2d': Conv2D(), 'lstm': LSTM()}
assert model_type in models.keys(), '{} not an available model'.format(
model_type)
wav_paths = glob('{}/**'.format(src_root), recursive=True)
wav_paths = [x for x in wav_paths if '.wav' in x]
classes = sorted(os.listdir(args.src_root))
le = LabelEncoder()
le.fit(classes)
labels = [get_class(x, src_root) for x in wav_paths]
labels = le.transform(labels)
wav_train, wav_val, label_train, label_val = train_test_split(
wav_paths, labels, test_size=0.1, random_state=0)
tg = DataGenerator(wav_train,
label_train,
sr,
dt,
len(set(label_train)),
batch_size=batch_size)
vg = DataGenerator(wav_val,
label_val,
sr,
dt,
len(set(label_val)),
batch_size=batch_size)
model = models[model_type]
cp = ModelCheckpoint('models/{}.h5'.format(model_type),
monitor='val_loss',
save_best_only=True,
save_weights_only=False,
mode='auto',
save_freq='epoch',
verbose=1)
csv_logger = CSVLogger(os.path.join('logs',
'{}_history.csv'.format(model_type)),
append=True)
model.fit(tg,
validation_data=vg,
epochs=30,
verbose=1,
callbacks=[csv_logger, cp])
def train(args):
src_root = args.src_root
sr = args.sample_rate
dt = args.delta_time
batch_size = args.batch_size
model_type = args.model_type
params = {'N_CLASSES':len(os.listdir(args.src_root)),
'SR':sr,
'DT':dt}
models = {'conv1d':Conv1D(**params),
'conv2d':Conv2D(**params),
'lstm': LSTM(**params),
'transformer': Transformer(**params),
'ViT': ViT(**params)}
assert model_type in models.keys(), '{} not an available model'.format(model_type)
csv_path = os.path.join('logs', '{}_history.csv'.format(model_type))
wav_paths = glob('{}/**'.format(src_root), recursive=True)
wav_paths = [x.replace(os.sep, '/') for x in wav_paths if '.wav' in x]
classes = sorted(os.listdir(args.src_root))
le = LabelEncoder()
le.fit(classes)
labels = [os.path.split(x)[0].split('/')[-1] for x in wav_paths]
labels = le.transform(labels)
wav_train, wav_val, label_train, label_val = train_test_split(wav_paths,
labels,
test_size=0.1,
random_state=0)
assert len(label_train) >= args.batch_size, 'Number of train samples must be >= batch_size'
if len(set(label_train)) != params['N_CLASSES']:
warnings.warn('Found {}/{} classes in training data. Increase data size or change random_state.'.format(len(set(label_train)), params['N_CLASSES']))
if len(set(label_val)) != params['N_CLASSES']:
warnings.warn('Found {}/{} classes in validation data. Increase data size or change random_state.'.format(len(set(label_val)), params['N_CLASSES']))
tg = DataGenerator(wav_train, label_train, sr, dt,
params['N_CLASSES'], batch_size=batch_size)
vg = DataGenerator(wav_val, label_val, sr, dt,
params['N_CLASSES'], batch_size=batch_size)
model = models[model_type]
model.summary()
cp = ModelCheckpoint('models/{}.h5'.format(model_type), monitor='val_loss',
save_best_only=True, save_weights_only=False,
mode='auto', save_freq='epoch', verbose=1)
csv_logger = CSVLogger(csv_path, append=False)
model.fit(tg, validation_data=vg,
epochs=40, verbose=1,
callbacks=[csv_logger])
def run():
if len(sys.argv) < 2:
print("Usage: imitate.py <file>")
return
with open(sys.argv[1], "r") as f:
data = f.buffer.read()
charset = set(data)
world = Bytes(data, max_steps=100, charset=charset)
print("Charset size: %d" % len(charset))
model = Input(len(charset))
model = LSTM(model)
model = Affine(model, len(charset))
model = Softmax(model)
train(world, model)
for _ in range(10):
world.render(model)
def make_lstm(self,
embedd_dim: int = None,
hidden_size: int = None,
clone=None):
assert ((embedd_dim is not None) and
(hidden_size is not None)) or (clone is not None)
if clone is not None:
model = LSTM(clone.embedding_dim,
clone.hidden_dim,
vocab_size=self.vocab_size,
tagset_size=self.vocab_size)
model.load_state_dict(clone.state_dict())
else:
model = LSTM(embedd_dim,
hidden_size,
vocab_size=self.vocab_size,
tagset_size=self.vocab_size)
model = model.to(self.device)
return model
def main(args):
word_dict = WordDict(args.train_file)
train_set = DataSet(args.train_file, word_dict, sentence_len)
train_loader = data.DataLoader(train_set, batch_size=128)
test_set = DataSet(args.test_file, word_dict, sentence_len)
test_loader = data.DataLoader(test_set, batch_size=128)
print('finish loading the data')
if args.model == 'fasttext':
model = FastText(train_set.vocab_size(), embed_size, 14)
elif args.model == 'cnn':
model = CNN(train_set.vocab_size(), embed_size, 14)
else:
model = LSTM(train_set.vocab_size(), embed_size, 14)
optimizer = optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss().to(device)
for epoch in range(epochs):
print('Epoch ', epoch)
train(model, optimizer, criterion, train_loader)
test(model, test_loader)
def fit(args):
exp_name = get_experiment_name(args)
logging_path = os.path.join(args.save_path, exp_name) + ".log"
logging.basicConfig(filename=logging_path,
level=logging.INFO,
format="%(message)s")
seed_everything(args.seed)
label_map = load_label_map(args.dataset)
if args.use_cnn:
train_dataset = FeaturesDatset(
features_dir=os.path.join(args.data_dir,
f"{args.dataset}_train_features"),
label_map=label_map,
mode="train",
)
val_dataset = FeaturesDatset(
features_dir=os.path.join(args.data_dir,
f"{args.dataset}_val_features"),
label_map=label_map,
mode="val",
)
else:
train_dataset = KeypointsDataset(
keypoints_dir=os.path.join(args.data_dir,
f"{args.dataset}_train_keypoints"),
use_augs=args.use_augs,
label_map=label_map,
mode="train",
max_frame_len=169,
)
val_dataset = KeypointsDataset(
keypoints_dir=os.path.join(args.data_dir,
f"{args.dataset}_val_keypoints"),
use_augs=False,
label_map=label_map,
mode="val",
max_frame_len=169,
)
train_dataloader = data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True,
)
val_dataloader = data.DataLoader(
val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
)
n_classes = 50
if args.dataset == "include":
n_classes = 263
if args.model == "lstm":
config = LstmConfig()
if args.use_cnn:
config.input_size = CnnConfig.output_dim
model = LSTM(config=config, n_classes=n_classes)
else:
config = TransformerConfig(size=args.transformer_size)
if args.use_cnn:
config.input_size = CnnConfig.output_dim
model = Transformer(config=config, n_classes=n_classes)
model = model.to(device)
optimizer = torch.optim.AdamW(model.parameters(),
lr=args.learning_rate,
weight_decay=0.01)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="max",
factor=0.2)
if args.use_pretrained == "resume_training":
model, optimizer, scheduler = load_pretrained(args, n_classes, model,
optimizer, scheduler)
model_path = os.path.join(args.save_path, exp_name) + ".pth"
es = EarlyStopping(patience=15, mode="max")
for epoch in range(args.epochs):
print(f"Epoch: {epoch+1}/{args.epochs}")
train_loss, train_acc = train(train_dataloader, model, optimizer,
device)
val_loss, val_acc = validate(val_dataloader, model, device)
logging.info(
"Epoch: {}, train loss: {}, train acc: {}, val loss: {}, val acc: {}"
.format(epoch + 1, train_loss, train_acc, val_loss, val_acc))
scheduler.step(val_acc)
es(
model_path=model_path,
epoch_score=val_acc,
model=model,
optimizer=optimizer,
scheduler=scheduler,
)
if es.early_stop:
print("Early stopping")
break
print("### Training Complete ###")
def training(train_loader, test_loader, opt):
net_list = [
LSTM.LSTM(opt),
BiLSTM.BiLSTM(opt),
OriTextCNN.OriTextCNN(opt),
DilaTextCNN.DilaTextCNN(opt),
TextCNNInc.TextCNNInc(opt)
]
net = net_list[4]
best_acc = 0
# best_top_accs = []
best_acc_loss = 0
NUM_TRAIN = len(train_loader) * opt.BATCH_SIZE
PRE_EPOCH = 0
NUM_TRAIN_PER_EPOCH = len(train_loader)
NET_PREFIX = opt.NET_SAVE_PATH + net.model_name + "/"
print('==> Loading Model ...')
temp_model_name = "temp_model.dat"
model_name = "best_model.dat"
if not os.path.exists(NET_PREFIX):
os.mkdir(NET_PREFIX)
if not os.path.exists('./source/log/' + net.model_name):
os.mkdir('./source/log/' + net.model_name)
if os.path.exists(NET_PREFIX + temp_model_name) and opt.RE_TRAIN == False:
try:
net, PRE_EPOCH, best_acc = net.load(NET_PREFIX + temp_model_name)
print("Load existing model: %s" % (NET_PREFIX + temp_model_name))
except:
pass
if opt.USE_CUDA: net.cuda()
criterion = nn.CrossEntropyLoss()
if opt.OPTIMIZER == 'Adam':
optimizer = torch.optim.Adam(net.parameters(), lr=opt.LEARNING_RATE)
elif opt.OPTIMIZER == 'SGD':
optimizer = torch.optim.SGD(net.parameters(), lr=opt.LEARNING_RATE)
elif opt.OPTIMIZER == 'RMSP':
optimizer = torch.optim.RMSprop(net.parameters(), lr=opt.LEARNING_RATE)
else:
raise NameError("This optimizer isn't defined")
train_recorder = {'loss': [], 'acc': []}
test_recorder = {'loss': [], 'acc': []}
t = datetime.datetime.now().strftime("%m%d_%H:%M:%S")
log_file_name = "%s_%s.txt" % (net.model_name, t)
for epoch in range(opt.NUM_EPOCHS):
train_loss = 0
train_acc = 0
# Start training
net.train()
for i, data in tqdm(enumerate(train_loader),
desc="Training",
total=NUM_TRAIN_PER_EPOCH,
leave=False,
unit='b'):
inputs, labels, sent = data
if opt.USE_CUDA:
inputs, labels = Variable(inputs.cuda()), Variable(
labels.cuda())
else:
inputs, labels = Variable(inputs), Variable(labels)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# Do statistics for training
train_loss += loss.data[0]
_, predicts = torch.max(outputs, 1)
num_correct = (predicts == labels).sum()
train_acc += num_correct.data[0]
# Start testing
ave_test_loss, ave_test_acc, topnacc = testing(test_loader, net, opt)
ave_train_loss = float(train_loss) / NUM_TRAIN
ave_train_acc = float(train_acc) / NUM_TRAIN
# Do recording for each epoch
train_recorder['loss'].append(ave_train_loss)
train_recorder['acc'].append(ave_train_acc)
test_recorder['loss'].append(ave_test_loss)
test_recorder['acc'].append(ave_test_acc)
# Write log to files
with open('./source/log/' + net.model_name + '/' + log_file_name,
'w+') as fp:
json.dump(
{
'train_recorder': train_recorder,
'test_recorder': test_recorder
}, fp)
# Output results
print(
'Epoch [%d/%d], Train Loss: %.4f, Train Acc: %.4f, Test Loss: %.4f, Test Acc: %.4f'
% (epoch + 1 + PRE_EPOCH, opt.NUM_EPOCHS + PRE_EPOCH,
ave_train_loss, ave_train_acc, ave_test_loss, ave_test_acc))
if ave_test_acc > best_acc:
best_acc = ave_test_acc
best_top_accs = topnacc
best_acc_loss = ave_test_loss
net.save((epoch + PRE_EPOCH), best_acc, model_name)
# Save a temp model
net.save((epoch + PRE_EPOCH), best_acc, temp_model_name)
summary_info = {
'total_epoch': (epoch + PRE_EPOCH),
'best_acc': best_acc,
'best_acc_loss': best_acc_loss,
'ave_test_acc': ave_test_acc,
'ave_test_loss': ave_test_loss,
'ave_train_acc': ave_train_acc,
'ave_train_loss': ave_train_loss,
'best_top_accs': best_top_accs
}
write_summary(net, opt, summary_info)
print(
'==> Training Finished. Current model is %s. The highest test acc is %.4f'
% (net.model_name, best_acc))
return net
i = i.strip()
i = i.split("\t")
vocab[i[0]] = int(i[1])
vocab[unk_string] = len(vocab)
return vocab
assert args.ngrams == 0
assert args.share_vocab == 1
data = h5py.File(args.data_file, 'r')
data = data['data']
vocab = load_vocab(args.vocab_file)
vocab_fr = None
if args.load_file is not None:
model, epoch = load_model(data, args)
print("Loaded model at epoch {0} and resuming training.".format(epoch))
model.train_epochs(start_epoch=epoch)
else:
if args.model == "avg":
model = Averaging(data, args, vocab, vocab_fr)
elif args.model == "lstm":
model = LSTM(data, args, vocab, vocab_fr)
print(" ".join(sys.argv))
print("Num examples:", len(data))
print("Num words:", len(vocab))
model.train_epochs()
num_layers = 1
init_forget_bias = 1
lr = 0.00016654418947982137
weight_decay = 7.040822706204121e-05
dropout = 0.18404592540409914
clip = 4389.748805208904
# モデルのインスタンス作成
if model_name == 'sru':
model = SRU(input_size, phi_size, r_size, cell_out_size, output_size, dropout=dropout, gpu=gpu)
model.initWeight()
elif model_name == 'gru':
model = GRU(input_size, hidden_size, output_size, num_layers, dropout, gpu=gpu)
model.initWeight(init_forget_bias)
elif model_name == 'lstm':
model = LSTM(input_size, hidden_size, output_size, num_layers, dropout, gpu=gpu)
model.initWeight(init_forget_bias)
if gpu == True:
model.cuda()
# loss, optimizerの定義
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
''' 訓練 '''
n_epochs = 400
n_batches = train_X.shape[0]//batch_size
n_batches_test = test_X.shape[0]//batch_size
all_acc = []
class SupervisedTest:
def __init__(self, window_size=3) -> None:
self.window_size = window_size
self.dataset = SupervisedDataset(mode='test',
window_size=self.window_size,
log_reg=False)
self.loader = DataLoader(self.dataset, batch_size=64, shuffle=False)
self.checkpoint_path = "./model_params/val/blstm_bs64_lr1e-3_ws40_hs128_nl2_dout50/val_lstm_epoch91_acc81.9593.pth"
#self.checkpoint_path = "./model_params/logistic_regression/val/val_logreg_epoch100_acc35.8229.pth"
self.checkpoint = torch.load(self.checkpoint_path)
self.model = LSTM(input_size=78,
hidden_size=128,
num_classes=170,
n_layers=2).to(device=torch.device('cuda:0'))
#self.model = LogisticRegression(num_keypoints=78, num_features=2, num_classes=170).to(device=torch.device('cuda:0'))
self.model.load_state_dict(self.checkpoint, strict=True)
self.model.eval()
#self.criterion = nn.BCEWithLogitsLoss() # Use this for Logistic Regression training
self.criterion = nn.BCELoss(
) # Use this for LSTM training (with Softmax)
#self.writer_text = SummaryWriter('./Tensorboard/test_text/')
#self.writer_avg_test_loss = SummaryWriter('./Tensorboard/test_loss/')
#self.writer_hparams = SummaryWriter('./Tensorboard/test_hparams/')
def _start_batch_test(self) -> None:
current_iter = 0
running_loss = 0.0
average_loss = 0.0
num_data = 0
running_correct_preds = 0
running_correct_classwise_preds = [0] * 170
running_false_classwise_preds = [0] * 170
running_all_classwise_gt_labels = [0] * 170
with torch.no_grad():
for batch_window, batch_label in self.loader:
current_iter += 1
outs = self.model(batch_window)
loss = self.criterion(outs, batch_label)
running_loss += loss.item()
average_loss = running_loss / current_iter
pred_confidence, pred_index = torch.max(outs, dim=1)
gt_confidence, gt_index = torch.max(batch_label, dim=1)
#batch_correct_preds = torch.eq(pred_index, gt_index).long().sum().item()
#batch_accuracy = (batch_correct_preds / batch_window.shape[0]) * 100
num_data += batch_window.shape[0]
batch_accuracy, batch_correct_preds, classwise_correct_preds, classwise_false_preds, classwise_gt_labels = self._calculate_batch_accuracy(
outs, batch_label)
running_correct_preds += batch_correct_preds
running_correct_classwise_preds = self._add_lists_elementwise(
running_correct_classwise_preds, classwise_correct_preds)
running_false_classwise_preds = self._add_lists_elementwise(
running_false_classwise_preds, classwise_false_preds)
running_all_classwise_gt_labels = self._add_lists_elementwise(
running_all_classwise_gt_labels, classwise_gt_labels)
if current_iter % 1 == 0:
print(
f"\nITER#{current_iter} BATCH TEST ACCURACY: {batch_accuracy:.4f}, RUNNING TEST LOSS: {loss.item():.8f}"
)
print(f"Predicted / GT index:\n{pred_index}\n{gt_index}\n")
#epoch_accuracy = (running_correct_preds / num_data) * 100
epoch_accuracy, classwise_accuracy = self._calculate_epoch_accuracy(
running_correct_preds, running_correct_classwise_preds,
running_all_classwise_gt_labels, num_data)
print(
f"\n\nTEST WINDOW-WISE ACCURACY: {epoch_accuracy:.4f}, AVERAGE TEST LOSS: {average_loss:.8f}\n\n"
)
correct_vid = 0
false_vid = 0
for i in range(len(running_correct_classwise_preds)):
print(
f"Person{i:03d} | Number of correct/all predictions: {running_correct_classwise_preds[i]:<3d}/{running_all_classwise_gt_labels[i]:<5d} | Accuracy: {classwise_accuracy[i]:.2f}%"
)
if (running_correct_classwise_preds[i] +
running_false_classwise_preds[i]) != 0:
if classwise_accuracy[i] >= 50:
correct_vid += 1
else:
false_vid += 1
videowise_accuracy = (correct_vid /
(correct_vid + false_vid)) * 100
print(
f"\n\nTEST VIDEO-WISE ACCURACY: {videowise_accuracy:.4f}%\n\n")
def start(self, mode="batch") -> None:
if mode == "batch":
self._start_batch_test()
def _calculate_batch_accuracy(self, predictions, annotations):
pred_confidence, pred_index = torch.max(predictions, dim=1)
gt_confidence, gt_index = torch.max(annotations, dim=1)
person1 = 1
if gt_index[0] == 0:
person1 += gt_index.shape[0]
batch_correct_preds = torch.eq(pred_index,
gt_index).long().sum().item()
batch_accuracy = (batch_correct_preds / predictions.shape[0]) * 100
# Calculating number of classwise correct/false predictions
classwise_correct_preds = torch.zeros(170).long()
classwise_false_preds = torch.zeros(170).long()
classwise_gt_labels = torch.zeros(170).long()
correct_preds_class = pred_index[torch.eq(pred_index, gt_index)].long()
false_preds_class = pred_index[torch.ne(pred_index, gt_index)].long()
for element in correct_preds_class:
classwise_correct_preds[element] += 1
for element in false_preds_class:
classwise_false_preds[element] += 1
for element in gt_index:
classwise_gt_labels[element] += 1
classwise_correct_preds = classwise_correct_preds.tolist()
classwise_false_preds = classwise_false_preds.tolist()
return batch_accuracy, batch_correct_preds, classwise_correct_preds, classwise_false_preds, classwise_gt_labels
def _add_lists_elementwise(self, list1, list2):
array1 = np.array(list1)
array2 = np.array(list2)
sum_list = (array1 + array2).tolist()
return sum_list
def _calculate_epoch_accuracy(self, running_correct_preds,
running_correct_classwise_preds,
running_all_classwise_gt_labels, num_data):
epoch_accuracy = (running_correct_preds / num_data) * 100
classwise_accuracy = [
0
] * 170 #(((np.array(running_correct_classwise_preds) / (running_correct_classwise_preds + running_false_classwise_preds))) * 100).tolist()
for i in range(len(running_correct_classwise_preds)):
if (running_all_classwise_gt_labels[i]) == 0:
classwise_accuracy[i] = 0
else:
classwise_accuracy[i] = (
running_correct_classwise_preds[i] /
running_all_classwise_gt_labels[i]) * 100
return epoch_accuracy, classwise_accuracy
shuffle = True, random_seed = random_seed)
inputs, labels = next(iter(train_dataloader))
[batch_size, type_size, step_size, fea_size] = inputs.size()
importlib.reload(utils)
from utils import TrainModel, TestModel
# LSTM
importlib.reload(models)
from models import LSTM
importlib.reload(utils)
from utils import TrainModel, TestModel
model_name = 'LSTM'
print(model_name)
model = LSTM(A.shape[0])
model, train_result = TrainModel(model,
train_dataloader,
valid_dataloader,
learning_rate=learning_rate,
patience=5)
test_result = TestModel(model, test_dataloader, max_speed)
StoreData(result_dict, model_name, train_result, test_result, directory,
model, random_seed, save_model)
# LSTM-I
importlib.reload(models)
from models import LSTM
importlib.reload(utils)
from utils import TrainModel, TestModel
model_name = 'LSTMI'
def main():
torch.manual_seed(42)
# Random
#params = {'batch_size': 32, 'dropout': 0, 'hidden_dim': 128, 'learning_rate': 0.01, 'num_epochs': 5, 'num_layers': 2, 'oversample': False, 'soft_labels': False}
# Glove
params = {
'batch_size': 32,
'dropout': 0,
'hidden_dim': 128,
'learning_rate': 0.001,
'num_epochs': 5,
'num_layers': 2,
'oversample': False,
'soft_labels': False
}
# Random
#params = {'batch_size': 32, 'dropout': 0, 'hidden_dim': 256, 'learning_rate': 0.0001, 'num_epochs': 5, 'num_layers': 3, 'oversample': False, 'soft_labels': False}
#some params
experiment_number = 1
test_percentage = 0.1
val_percentage = 0.2
batch_size = params["batch_size"]
num_epochs = 5 #params["num_epochs"]
dropout = params["dropout"]
embedding_dim = 300
model_name = "CNN" #'Bert' #"CNN" #"LSTM"
unsupervised = True
embedding = "Glove" #"Random" ##"Glove" # "Both" #
soft_labels = False
combine = embedding == "Both"
# LSTM parameters
if model_name == "LSTM":
hidden_dim = params["hidden_dim"]
num_layers = params["num_layers"]
# Bert parameter
num_warmup_steps = 100
num_total_steps = 1000
if model_name == "Bert":
embedding = "None"
if embedding == "Both":
combine = True
embedding = "Random"
else:
combine = False
learning_rate = params["learning_rate"] #5e-5, 3e-5, 2e-5
oversample_bool = False
weighted_loss = True
# load data
dataset = Dataset("../data/cleaned_tweets_orig.csv",
use_embedding=embedding,
embedd_dim=embedding_dim,
combine=combine,
for_bert=(model_name == "Bert"))
#dataset.oversample()
train_data, val_test_data = split_dataset(dataset,
test_percentage + val_percentage)
val_data, test_data = split_dataset(
val_test_data, test_percentage / (test_percentage + val_percentage))
# print(len(train_data))
#save_data(train_data, 'train')
#save_data(test_data, 'test')
#define loaders
if oversample_bool:
weights, targets = get_loss_weights(train_data, return_targets=True)
class_sample_count = [
1024 / 20, 13426, 2898 / 2
] # dataset has 10 class-1 samples, 1 class-2 samples, etc.
oversample_weights = 1 / torch.Tensor(class_sample_count)
oversample_weights = oversample_weights[targets]
# oversample_weights = torch.tensor([0.9414, 0.2242, 0.8344]) #torch.ones((3))-
sampler = torch.utils.data.sampler.WeightedRandomSampler(
oversample_weights, len(oversample_weights))
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
collate_fn=my_collate,
sampler=sampler)
else:
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
collate_fn=my_collate)
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=batch_size,
collate_fn=my_collate)
#define model
if model_name == "CNN":
vocab_size = len(dataset.vocab)
model = CNN(vocab_size, embedding_dim, combine=combine)
elif model_name == "LSTM":
vocab_size = len(dataset.vocab)
model = LSTM(vocab_size,
embedding_dim,
batch_size=batch_size,
hidden_dim=hidden_dim,
lstm_num_layers=num_layers,
combine=combine,
dropout=dropout)
elif model_name == "Bert":
model = BertForSequenceClassification.from_pretrained(
"bert-base-uncased", num_labels=3)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
collate_fn=bert_collate)
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=batch_size,
collate_fn=bert_collate)
#device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
#LOSS : weighted cross entropy loss, by class counts of other classess
if weighted_loss:
weights = torch.tensor([0.9414, 0.2242, 0.8344], device=device)
else:
weights = torch.ones(3, device=device)
#weights = torch.tensor([1.0, 1.0, 1.0], device = device) #get_loss_weights(train_data).to(device) # not to run again
criterion = nn.CrossEntropyLoss(weight=weights)
if soft_labels:
criterion = weighted_soft_cross_entropy
#latent model
if unsupervised:
vocab_size = len(dataset.vocab)
criterion = nn.CrossEntropyLoss(weight=weights, reduction='none')
model = Rationalisation_model(vocab_size,
embedding_dim=embedding_dim,
model=model_name,
batch_size=batch_size,
combine=combine,
criterion=criterion)
if not model_name == "Bert":
model.embedding.weight.data.copy_(dataset.vocab.vectors)
if combine:
model.embedding_glove.weight.data.copy_(dataset.glove.vectors)
#model to device
model.to(device)
#optimiser
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
if model_name == "Bert":
optimizer = AdamW(model.parameters(),
lr=learning_rate,
correct_bias=False)
# Linear scheduler for adaptive lr
scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=num_warmup_steps,
t_total=num_total_steps)
else:
scheduler = None
plot_log = defaultdict(list)
for epoch in range(num_epochs):
#train and validate
epoch_loss, epoch_acc = train_epoch(model,
train_loader,
optimizer,
criterion,
device,
soft_labels=soft_labels,
weights=weights,
scheduler=scheduler,
unsupervised=unsupervised)
val_loss, val_acc = evaluate_epoch(model,
val_loader,
criterion,
device,
soft_labels=soft_labels,
weights=weights,
unsupervised=unsupervised)
#save for plotting
for name, point in zip(
["train_loss", "train_accuracy", "val_loss", "val_accuracy"],
[epoch_loss, epoch_acc, val_loss, val_acc]):
plot_log[f'{name}'] = point
#realtime feel
print(f'Epoch: {epoch+1}')
print(
f'\tTrain Loss: {epoch_loss:.5f} | Train Acc: {epoch_acc*100:.2f}%'
)
print(f'\t Val. Loss: {val_loss:.5f} | Val. Acc: {val_acc*100:.2f}%')
sample_sentences_and_z(model, train_loader, device, dataset.vocab)
#save plot
results_directory = f'plots/{experiment_number}'
os.makedirs(results_directory, exist_ok=True)
for name, data in plot_log.items():
save_plot(data, name, results_directory)
#save model
torch.save(model, os.path.join(results_directory, 'model_cnn.pth'))
#confusion matrix and all that fun
loss, acc, predictions, ground_truth = evaluate_epoch(
model,
val_loader,
criterion,
device,
is_final=True,
soft_labels=soft_labels,
weights=weights,
unsupervised=unsupervised)
conf_matrix = confusion_matrix(ground_truth, predictions)
class_report = classification_report(ground_truth, predictions)
print('\nFinal Loss and Accuracy\n----------------\n')
print(f'\t Val. Loss: {loss:.5f} | Val. Acc: {acc*100:.2f}%')
print('\nCONFUSION MATRIX\n----------------\n')
print(conf_matrix)
print('\nCLASSSIFICATION REPORT\n----------------------\n')
print(class_report)
plot_confusion_matrix(ground_truth,
predictions,
classes=["Hate speech", "Offensive", "Neither"],
normalize=False,
title='Confusion matrix')
plt.show()
for audio, lbl in train_ds:
train_ct += 1
val_ct = 0
for audio, lbl in val_ds:
val_ct += 1
test_ct = 0
for audio, lbl in test_ds:
test_ct += 1
train_ds.repeat(None)
val_ds.repeat(None)
test_ds.repeat(None)
model = LSTM()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
cp_path = "training_2/cp-{epoch:04d}.h5py"
cp_dir = os.path.dirname(cp_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=cp_path,
verbose=1,
save_weights_only=True,
period=1)
log_dir = "./runs"
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
outputs=critic_model.get_layer('q_outputs').output)
ones = np.ones(shape=(len(test_seg), num_actions))
pred = predict(actor_q_model, (input_validation, ones))
print('On validate data: ')
metrics(y_test, pred)
ones_tr = np.ones(shape=(len(train_seg), num_actions))
pred_tr = predict(actor_q_model, (input_train, ones_tr))
print('On train data: ')
metrics(y_train, pred_tr)
elif MODEL == 'LSTM':
class_nums = 2
lstm = LSTM(class_nums, embeddings_matrix, maxlen=maxlen)
_ = lstm(inputs=inputs)
adam = tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
rmsprop = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06)
lstm.compile(loss='categorical_crossentropy', optimizer=adam)
if MODE == 'train':
lstm.fit(input_train,
tr_one_hot,
batch_size=32,
epochs=20,
verbose=1,
shuffle=True)
def main(args):
with open(args.config, 'r') as f:
config = json.load(f)
if config['optimizer'] == 'SGD':
optimizer = SGD(lr=config['learning_rate'],
decay=config['learning_rate'] / config['epochs'],
momentum=config['momentum'])
else:
raise Exception('Unsupported optimizer: {}.'.format(
config['optimizer']))
model_name = str.lower(config['model'])
if model_name == 'lstm':
model = LSTM(config['input_length'], 2)
elif model_name == 'conv1d':
model = Conv1D(config['input_length'], 2)
elif model_name == 'conv2d':
model = Conv2D(config['input_length'], 2)
else:
raise Exception('Unsupported model: {}.'.format(config['model']))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
wav_paths = glob('{}/**'.format(args.data_dir), recursive=True)
wav_paths = [x for x in wav_paths if '.wav' in x]
classes = sorted(os.listdir(args.data_dir))
le = LabelEncoder()
le.fit(classes)
labels = [get_class(x, args.data_dir) for x in wav_paths]
labels = le.transform(labels)
print('CLASSES: ', list(le.classes_))
print(le.transform(list(le.classes_)))
wav_train, wav_val, label_train, label_val = train_test_split(
wav_paths,
labels,
test_size=config['validation_split'],
random_state=0)
tg = DataGenerator(wav_train,
label_train,
config['input_length'],
len(set(label_train)),
batch_size=config['batch_size'])
vg = DataGenerator(wav_val,
label_val,
config['input_length'],
len(set(label_val)),
batch_size=config['batch_size'])
output_sub_dir = os.path.join(args.output_dir, model_name,
datetime.now().strftime('%Y%m%d_%H%M%S'))
os.makedirs(output_sub_dir)
callbacks = [
EarlyStopping(monitor='val_loss',
patience=config['patience'],
restore_best_weights=True,
verbose=1),
ModelCheckpoint(filepath=os.path.join(
output_sub_dir, 'model.{epoch:02d}-{val_loss:.4f}.h5'),
monitor='val_loss',
save_best_only=True,
verbose=1),
CSVLogger(os.path.join(output_sub_dir, 'epochs.csv'))
]
model.fit(tg,
validation_data=vg,
epochs=config['epochs'],
verbose=1,
callbacks=callbacks)
def main():
#print the config args
print(config.transfer_learning)
print(config.mode)
print(config.input_size)
# Fix Seed for Reproducibility #
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(config.seed)
# Samples, Weights, and Plots Path #
paths = [config.weights_path, config.plots_path, config.numpy_path]
for path in paths:
make_dirs(path)
# Prepare Data #
data = load_data(config.combined_path, config.which_data, config.preprocess, config.resample)
# id = config.which_data.split('_')[0]
id = 12 #BOON added
print("Data of {} is successfully Loaded!".format(config.which_data))
print(type(data))
print(data.shape)
# Plot Time-series Data #
if config.plot:
plot_full(config.plots_path, data, id, config.feature)
plot_split(config.plots_path, data, id, config.valid_start, config.test_start, config.feature)
# Min-Max Scaler #
scaler = MinMaxScaler()
data.iloc[:,:] = scaler.fit_transform(data)
print(type(data))
# Split the Dataset #
train_X, train_Y, val_X, val_Y, test_X, test_Y, test_shifted = \
get_time_series_data_(data, config.valid_start, config.test_start, config.feature, config.label, config.window)
print(train_X.shape)
print(train_Y.shape)
# Get Data Loader #
train_loader, val_loader, test_loader = \
get_data_loader(train_X, train_Y, val_X, val_Y, test_X, test_Y, config.batch_size)
# Constants #
best_val_loss = 100
best_val_improv = 0
# Lists #
train_losses, val_losses = list(), list()
val_maes, val_mses, val_rmses, val_mapes, val_mpes, val_r2s = list(), list(), list(), list(), list(), list()
# Prepare Network #
if config.network == 'dnn':
model = DNN(config.window, config.hidden_size, config.output_size).to(device)
elif config.network == 'cnn':
model = CNN(config.window, config.hidden_size, config.output_size).to(device)
elif config.network == 'rnn':
model = RNN(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'lstm':
model = LSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
elif config.network == 'gru':
model = GRU(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'recursive':
model = RecursiveLSTM(config.input_size, config.hidden_size, config.num_layers, config.output_size).to(device)
elif config.network == 'attentional':
model = AttentionalLSTM(config.input_size, config.key, config.query, config.value, config.hidden_size, config.num_layers, config.output_size, config.bidirectional).to(device)
else:
raise NotImplementedError
if config.mode == 'train':
# If fine-tuning #
print('config.TL = {}'.format(config.transfer_learning))
if config.transfer_learning:
print('config.TL = {}'.format(config.transfer_learning))
print('TL: True')
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_12.pkl'.format(config.network))))
for param in model.parameters():
param.requires_grad = True
# Loss Function #
criterion = torch.nn.MSELoss()
# Optimizer #
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr, betas=(0.5, 0.999))
optimizer_scheduler = get_lr_scheduler(config.lr_scheduler, optimizer, config)
# Train and Validation #
print("Training {} started with total epoch of {} using Driver ID of {}.".format(config.network, config.num_epochs, id))
for epoch in range(config.num_epochs):
# Train #
for i, (data, label) in enumerate(train_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Calculate Loss #
train_loss = criterion(pred, label)
# Back Propagation and Update #
optimizer.zero_grad()
train_loss.backward()
optimizer.step()
# Add items to Lists #
train_losses.append(train_loss.item())
print("Epoch [{}/{}]".format(epoch+1, config.num_epochs))
print("Train")
print("Loss : {:.4f}".format(np.average(train_losses)))
optimizer_scheduler.step()
# Validation #
with torch.no_grad():
for i, (data, label) in enumerate(val_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred_val = model(data)
# Calculate Loss #
val_loss = criterion(pred_val, label)
val_mae = mean_absolute_error(label.cpu(), pred_val.cpu())
val_mse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=True)
val_rmse = mean_squared_error(label.cpu(), pred_val.cpu(), squared=False)
val_mpe = mean_percentage_error(label.cpu(), pred_val.cpu())
val_mape = mean_absolute_percentage_error(label.cpu(), pred_val.cpu())
val_r2 = r2_score(label.cpu(), pred_val.cpu())
# Add item to Lists #
val_losses.append(val_loss.item())
val_maes.append(val_mae.item())
val_mses.append(val_mse.item())
val_rmses.append(val_rmse.item())
val_mpes.append(val_mpe.item())
val_mapes.append(val_mape.item())
val_r2s.append(val_r2.item())
# Print Statistics #
print("Validation")
print("Loss : {:.4f}".format(np.average(val_losses)))
print(" MAE : {:.4f}".format(np.average(val_maes)))
print(" MSE : {:.4f}".format(np.average(val_mses)))
print("RMSE : {:.4f}".format(np.average(val_rmses)))
print(" MPE : {:.4f}".format(np.average(val_mpes)))
print("MAPE : {:.4f}".format(np.average(val_mapes)))
print(" R^2 : {:.4f}".format(np.average(val_r2s)))
# Save the model only if validation loss decreased #
curr_val_loss = np.average(val_losses)
if curr_val_loss < best_val_loss:
best_val_loss = min(curr_val_loss, best_val_loss)
# if config.transfer_learning:
# torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer.pkl'.format(config.network, id)))
# else:
# torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}.pkl'.format(config.network, id)))
if config.transfer_learning:
torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer_BOON_reshaped.pkl'.format(config.network, id)))
else:
torch.save(model.state_dict(), os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_BOON_reshaped.pkl'.format(config.network, id)))
print("Best model is saved!\n")
best_val_improv = 0
elif curr_val_loss >= best_val_loss:
best_val_improv += 1
print("Best Validation has not improved for {} epochs.\n".format(best_val_improv))
if best_val_improv == 10:
break
elif config.mode == 'test':
# Prepare Network #
if config.transfer_learning:
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_transfer_BOON_reshaped.pkl'.format(config.network, id))))
else:
model.load_state_dict(torch.load(os.path.join(config.weights_path, 'BEST_{}_Device_ID_{}_BOON_reshaped.pkl'.format(config.network, id))))
print("{} for Device ID {} is successfully loaded!".format((config.network).upper(), id))
with torch.no_grad():
pred_test, labels = list(), list()
for i, (data, label) in enumerate(test_loader):
# Data Preparation #
data = data.to(device, dtype=torch.float32)
label = label.to(device, dtype=torch.float32)
# Forward Data #
pred = model(data)
# Add items to Lists #
pred_test += pred
labels += label
# Derive Metric and Plot #
if config.transfer_learning:
pred, actual = test(config.plots_path, id, config.network, scaler, pred_test, labels, test_shifted, transfer_learning=True)
else:
pred, actual = test(config.plots_path, id, config.network, scaler, pred_test, labels, test_shifted)
def __init__(self, Data, lstm_neurons, epochs, dense_neurons=3):
self.lstm = LSTM(Data, lstm_neurons, epochs)
self.lstm.fit_model(False)
self.PredStates()
print(self.unperturbed.shape)
