def train(train_iter, dev_iter, model):
"""
Performs training on train set and evaluation on validation set
Arguments:
train_iter: Iterator for train set
dev_iter: Iterator for valid set
model: PyTorch model
args: command-line args
"""
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer, patience=1,
factor=0.1, mode='max', verbose=True)
best_auc = 0
epoch = 0
last_epoch = 0
model.train()
for epoch in range(1, EPOCHS+1):
for feature, target in train_iter:
feature = torch.autograd.Variable(feature)
target = torch.autograd.Variable(target)
feature, target = feature.cuda(), target.cuda()
optimizer.zero_grad()
logit = model(feature)
loss = F.binary_cross_entropy_with_logits(logit, target)
loss.backward()
optimizer.step()
dev_auc = eval(dev_iter, model)
scheduler.step(dev_auc)
if dev_auc > best_auc:
best_auc = dev_auc
last_epoch = epoch
save(model, DATA_PATH, 'stack', 'model')
def test(tmpdir) -> None:
data_dir = tmpdir.mkdir("data_dir")
output_dir = tmpdir.mkdir("output_dir")
model_dir = tmpdir.mkdir("model_dir")
create_multivariate_datasets(data_dir)
datasets = load_multivariate_datasets(data_dir)
prediction_length = 2
predictor = train(
datasets,
output_dir,
model_dir,
context_length=12,
prediction_length=prediction_length,
skip_size=2,
ar_window=3,
channels=6,
scaling=False,
output_activation="sigmoid",
epochs=1,
batch_size=5,
learning_rate=1e-2,
seed=42,
)
forecasts, tss, agg_metrics, item_metrics = evaluate(predictor, datasets.test, num_samples=1)
save(predictor, model_dir)
predictor = model_fn(model_dir)
request_body = {}
request_body["target"] = np.random.randn(10, prediction_length).tolist()
request_body["start"] = "2001-01-01"
request_body["source"] = []
ret, _ = transform_fn(predictor, json.dumps(request_body), None, None)
forecast_samples = np.array(ret["forecasts"]["samples"])
assert forecast_samples.shape == (1, prediction_length, 10)
agg_metrics = json.loads(ret["agg_metrics"])
for metric in ["RMSE", "ND", "MSE"]:
assert agg_metrics[metric] < 1.5, f"assertion failed for metric: {metric}"
return
def init_generation_zero(folder=''):
"""
creates the folder structure for gen-0, which contains the untrained, random model
"""
current_gen = 'gen-0'
root = os.path.join(folder, 'model')
model_folder_path = mkdir(root, current_gen)
tf_folder_path = mkdir(root, current_gen, 'tf')
weight_path = os.path.join(model_folder_path, 'weights.h5')
keras_path = os.path.join(model_folder_path, 'keras_model.h5')
frozen_path = os.path.join(model_folder_path, 'frozen_model.pb')
tf_path = os.path.join(tf_folder_path, 'connect4')
creator = network.Con4Zero(network.INPUT_SHAPE)
neural = creator()
neural.save_weights(weight_path)
train.save(weight_path, keras_path, tf_path, frozen_path)
def get_calibrated_model():
trainloader, valid_loader, testloader, log_freq = data_load()
# load saved model
net = Net_MCDO()
net = nn.DataParallel(net)
net = load('Net_MCDO', net)
# Now we're going to wrap the model with a decorator that adds temperature scaling
cali_model = ModelWithTemperature(net)
# Tune the model temperature, and save the results
cali_model.set_temperature(valid_loader)
#model = nn.DataParallel(model)
#model_filename = os.path.join('./model', 'model_with_temperature.pkl')
save('model_with_temperature', cali_model)
#torch.save(model.state_dict(), model_filename)
print('Temperature scaled model sved')
print('Done!')
""" net = net.eval()
cali_model = cali_model.eval() """
test(net, True, testloader)
test(cali_model, True, testloader)
def main():
options, args = option_parser.parse_args()
prepare()
if options.run:
trainer, params = train.get_new_model_trainer(glove_txt,
open(snli_train, "r"),
open(snli_dev, "r"))
trainer.train()
train.save(trainer, options.train, params, trainer.train_data)
results, loss, accuracy = trainer.predict(
SNLI(open(snli_test, "r"), glove_txt))
write_results_to_file("results", results, loss, accuracy)
else:
if options.train:
trainer, params = train.get_new_model_trainer(
glove_txt, open(snli_train, "r"), open(snli_dev, "r"))
trainer.train()
train.save(trainer, options.train, params, trainer.train_data)
if options.test:
trainer, params, vocab = train.load_trainer_and_args(options.test)
results, loss, accuracy = trainer.predict(
SNLI(open(snli_test, "r"), glove_txt))
write_results_to_file("results", results, loss, accuracy)
backgroundsTrainData = backgrounds[trainBackgroundsMin:trainBackgroundsMax]
backgroundsTestData = backgrounds[testBackgroundsMin:testBackgroundsMax]
responsesPersons = np.ones(len(personsTrainData), np.uint8)
responsesBackgrounds = np.zeros(len(backgroundsTrainData), np.uint8)
responses = np.append(responsesPersons, responsesBackgrounds)
trainHog = np.append(personsTrainData.flatten(), backgroundsTrainData.flatten())
trainHog = np.reshape(trainHog, (-1,5880))
print('Entrenando...')
svm = train.createSVM()
train.train(svm, trainHog, responses)
train.save(svm, 'saveData.dat')
print('Probando...')
testHog = np.append(personsTestData.flatten(), backgroundsTestData.flatten())
testHog = np.reshape(testHog, (-1, 5880))
testData = np.array(testHog, dtype=np.float32)
testResponse = svm.predict(testData)[1].ravel()
correctPersons = np.count_nonzero(testResponse[:testPersonsMax - testPersonsMin])/(testPersonsMax - testPersonsMin)*100
print('Personas correctas: {:.2f}%'.format(correctPersons))
print('Personas incorrectas: {:.2f}%'.format(100 - correctPersons))
wrongBackgrounds = np.count_nonzero(testResponse[testPersonsMax - testPersonsMin:])/(testBackgroundsMax-testBackgroundsMin)*100
print('Fondos correctos: {:.2f}%' .format(100 - wrongBackgrounds))
print('Fondos incorrectos: {:.2f}%' .format(wrongBackgrounds))
def train_confidnetnet_model(train, test, dictionary, params, options):
#####################################################################################################
# training model using 50% of positive and 50% of negative data in mini batch
#####################################################################################################
ids_train, labels_train, msg_train, code_train = train
ids_test, labels_test, msg_test, code_test = test
dict_msg, dict_code = dictionary
print('Dictionary message: %i -- Dictionary code: %i' % (len(dict_msg), len(dict_code)))
print('Training data')
info_label(labels_train)
pad_msg_train = padding_data(data=msg_train, dictionary=dict_msg, params=params, type='msg')
pad_code_train = padding_data(data=code_train, dictionary=dict_code, params=params, type='code')
print(pad_msg_train.shape, pad_code_train.shape)
print('Testing data')
info_label(labels_test)
pad_msg_test = padding_data(data=msg_test, dictionary=dict_msg, params=params, type='msg')
pad_code_test = padding_data(data=code_test, dictionary=dict_code, params=params, type='code')
print(pad_msg_test.shape, pad_code_test.shape)
# set up parameters
params.cuda = (not params.no_cuda) and torch.cuda.is_available()
del params.no_cuda
params.filter_sizes = [int(k) for k in params.filter_sizes.split(',')]
params.save_dir = os.path.join(params.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
params.vocab_msg, params.vocab_code = len(dict_msg), len(dict_code)
if len(labels_train.shape) == 1:
params.class_num = 1
else:
params.class_num = labels_train.shape[1]
params.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if options == 'clf':
# create and train the defect model
model = DefectNet(args=params)
if torch.cuda.is_available():
model = model.cuda()
model = freeze_layers(model=model, freeze_uncertainty_layers=True)
# print('Training model with options', options)
# for param in model.named_parameters():
# print(param[0], param[1].requires_grad)
optimizer = torch.optim.Adam(model.parameters(), lr=params.l2_reg_lambda)
steps = 0
batches_test = mini_batches(X_msg=pad_msg_test, X_code=pad_code_test, Y=labels_test)
write_log = list()
for epoch in range(1, params.num_epochs + 1):
# building batches for training model
batches_train = mini_batches_update(X_msg=pad_msg_train, X_code=pad_code_train, Y=labels_train)
for batch in batches_train:
pad_msg, pad_code, labels = batch
if torch.cuda.is_available():
pad_msg, pad_code, labels = torch.tensor(pad_msg).cuda(), torch.tensor(
pad_code).cuda(), torch.cuda.FloatTensor(labels)
else:
pad_msg, pad_code, labels = torch.tensor(pad_msg).long(), torch.tensor(pad_code).long(), torch.tensor(
labels).float()
optimizer.zero_grad()
predict, uncertainty = model.forward(pad_msg, pad_code)
loss = nn.BCELoss()
loss = loss(predict, labels)
loss.backward()
optimizer.step()
steps += 1
if steps % params.log_interval == 0:
print('\rEpoch: {} step: {} - loss: {:.6f}'.format(epoch, steps, loss.item()))
print('Epoch: %i ---Training data' % (epoch))
acc, prc, rc, f1, auc_ = evaluation_confidnet(data=batches_train, model=model)
print('Accuracy: %f -- Precision: %f -- Recall: %f -- F1: %f -- AUC: %f' % (acc, prc, rc, f1, auc_))
print('Epoch: %i ---Testing data' % (epoch))
acc, prc, rc, f1, auc_ = evaluation_confidnet(data=batches_test, model=model)
print('Accuracy: %f -- Precision: %f -- Recall: %f -- F1: %f -- AUC: %f' % (acc, prc, rc, f1, auc_))
write_log.append('Epoch - testing: %i --- Accuracy: %f -- Precision: %f -- Recall: %f -- F1: %f -- AUC: %f' % (epoch, acc, prc, rc, f1, auc_))
if epoch % 5 == 0:
save(model, params.save_dir, 'epoch', epoch)
write_file(params.save_dir + '/log.txt', write_log)
if options == 'confidnet':
# create and train the defect model
model = DefectNet(args=params)
if torch.cuda.is_available():
model = model.cuda()
if params.project == 'openstack':
model.load_state_dict(torch.load('./snapshot/2020-05-17_09-37-57/epoch_55.pt'), strict=True)
if params.project == 'qt':
model.load_state_dict(torch.load('./snapshot/2020-05-17_12-50-56/epoch_15.pt'), strict=True)
model = freeze_layers(model=model, freeze_uncertainty_layers=False)
print('Training model with options', options)
for param in model.named_parameters():
print(param[0], param[1].requires_grad)
optimizer = torch.optim.Adam(model.parameters(), lr=params.l2_reg_lambda)
steps = 0
batches_test = mini_batches(X_msg=pad_msg_test, X_code=pad_code_test, Y=labels_test)
write_log = list()
for epoch in range(1, params.num_epochs + 1):
# building batches for training model
batches_train = mini_batches_update(X_msg=pad_msg_train, X_code=pad_code_train, Y=labels_train)
for batch in batches_train:
pad_msg, pad_code, labels = batch
if torch.cuda.is_available():
pad_msg, pad_code, labels = torch.tensor(pad_msg).cuda(), torch.tensor(
pad_code).cuda(), torch.cuda.FloatTensor(labels)
else:
pad_msg, pad_code, labels = torch.tensor(pad_msg).long(), torch.tensor(pad_code).long(), torch.tensor(
labels).float()
optimizer.zero_grad()
predict, uncertainty = model.forward(pad_msg, pad_code)
loss = confid_mse_loss((predict, uncertainty), labels, args=params)
loss.backward()
optimizer.step()
steps += 1
if steps % params.log_interval == 0:
print('\rEpoch: {} step: {} - loss: {:.6f}'.format(epoch, steps, loss.item()))
print('Epoch: %i ---Training data' % (epoch))
auc_ = evaluation_uncertainty(data=batches_train, model=model)
print('AUC: %f' % (auc_))
print('Epoch: %i ---Testing data' % (epoch))
auc_ = evaluation_uncertainty(data=batches_test, model=model)
print('AUC: %f' % (auc_))
write_log.append('Epoch - testing: %i --- AUC: %f' % (epoch, auc_))
if epoch % 5 == 0:
save(model, params.save_dir, 'epoch', epoch)
write_file(params.save_dir + '/log.txt', write_log)
def main():
parser = argparse.ArgumentParser(description='CNN Home Eneregy Management System')
# learning
parser.add_argument('-lr', type=float, default=0.1, help='initial learning rate [default: 0.001]')
parser.add_argument('-adjust-lr', type=list,default=[0.001,0.0005,0.0002],help='if you use adjust lr')
parser.add_argument('-epochs', type=int, default=500, help='number of epochs for train [default: 256]')
parser.add_argument('-test-batch-size',type=int, default=10000, help='batch size when you eval')
parser.add_argument('-batch-size', type=int, default=128, help='batch size for training [default: 64]')
parser.add_argument('-log-interval', type=int, default=100, help='how many steps to wait before logging training status [default: 1]')
parser.add_argument('-test-interval', type=int, default=10, help='how many steps to wait before testing [default: 100]')
parser.add_argument('-save-interval', type=int, default=500, help='how many steps to wait before saving [default:500]')
parser.add_argument('-save-dir', type=str, default='model', help='where to save the snapshot')
parser.add_argument('-early-stop', type=int, default=1000, help='iteration numbers to stop without performance increasing')
parser.add_argument('-save-best', type=bool, default=False, help='whether to save when get best performance')
parser.add_argument('-save-prefix',type=str,default='checkpoint_cnn')
# data
parser.add_argument('-shuffle', action='store_true', default=False, help='shuffle the data every epoch')
parser.add_argument('-normalize-type', type=str, default='standard',help='[normalized,standard]')
# model
parser.add_argument('-dropout', type=float, default=0, help='the probability for dropout [default: 0.5]')
parser.add_argument('-max-norm', type=float, default=3.0, help='l2 constraint of parameters [default: 3.0]')
parser.add_argument('-input-dim', type=int, default=31, help='number of embedding dimension [default: 128]')
parser.add_argument('-input-num',type=int, default=15, help='input size')
parser.add_argument('-output-num',type=int, default=3, help='output size')
parser.add_argument('-kernel-num', type=int, default=300, help='number of each kind of kernel')
parser.add_argument('-kernel-sizes', type=str, default='3', help='comma-separated kernel size to use for convolution')
parser.add_argument('-static', action='store_true', default=True, help='fix the embedding')
parser.add_argument('-conv-depth',type=int, default=2, help='The depth of conv layer[1,2,3]')
parser.add_argument('-fc-depth',type=int,default=1,help='The depth of fully connected layer')
parser.add_argument('-fc-size',type=str, default='-1',help='The number of unit in fully connected layer[default: -1]')
parser.add_argument('-batch-normalization', default=True,help='using bath normalization')
# device
parser.add_argument('-device', type=int, default=-1, help='device to use for iterate data, -1 mean cpu [default: -1]')
parser.add_argument('-no-cuda', action='store_true', default=False, help='disable the gpu')
# option
parser.add_argument('-snapshot', type=str, default=None, help='filename of model snapshot [default: None]')
parser.add_argument('-predict', type=str, default=None, help='predict the sentence given')
parser.add_argument('-test', action='store_true', default=False, help='train or test')
parser.add_argument('--start-epoch', default=1, type=int,
help='If you load a model, program write log from loaded epoch')
parser.add_argument('--t',type=int,default=15)
# parser.add_argument('-resume',type=str,default='./model/std_41/lr_0.001/checkpoint_cnn_epochs_70.pth.tar')
parser.add_argument('-resume',type=str,default='')
parser.add_argument('--seed', type=int, default=0, help='random seed')
parser.add_argument('--name',type=str,default='cnn_21')
args = parser.parse_args()
torch.manual_seed(args.seed)
use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
cudnn.benchmark = True
cudnn.deterministic = True
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
writer = SummaryWriter('./logs/'+args.name+'/') #0.01,0.001,0.0001
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
args.fc_size = [int(k) for k in args.fc_size.split(',')]
args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime(args.name+'/lr_'+str(args.lr)))
#call the dataset
train_dataset = mydataset.CustomDataset(args,"../data/dataset_seq/"+args.normalize_type+"/dataset_train.csv")
test_dataset = mydataset.CustomDataset(args,"../data/dataset_seq/"+args.normalize_type+"/dataset_test_"+str(args.t)+".csv")
train_loss_dataset = mydataset.CustomDataset(args,"../data/dataset_seq/"+args.normalize_type+"/dataset_train_loss.csv")
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=args.shuffle, **kwargs)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs)
train_loss_loader = torch.utils.data.DataLoader(train_loss_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs)
cnn = model.CNN_HEMS(args).to(device)
if args.snapshot is not None:
print('\nLoading model from {}...'.format(args.snapshot))
cnn.load_state_dict(torch.load(args.snapshot))
criterion = nn.MSELoss().to(device)
optimizer = optim.Adam(cnn.parameters(), lr=args.lr)
if args.resume:
if os.path.isfile(args.resume):
checkpoint=load_chechpoint(args.resume)
cnn.load_state_dict(checkpoint['cnn'])
optimizer.load_state_dict(checkpoint['optimizer'])
args.start_epoch = checkpoint['epoch']+1
print("change_learning rate : {}".format(args.lr))
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr
else:
print("=> no checkpoint found at '{}'".format(args.resume))
for epoch in range(args.start_epoch,args.epochs):
print('Epoch : ', epoch)
# adjust_learning_rate(optimizer,epoch,args.adjust_lr)
train.train(args,train_loader,cnn,optimizer,criterion,test_loader,train_loss_loader,writer,epoch)
# for name, param in cnn.named_parameters():
# try:
# writer.add_histogram(name,param.clone().cpu().data.numpy(),epoch)
# writer.add_histogram(name+'grad',param.grad.clone().cpu().data.numpy(),epoch)
# except:
# print('param err')
# continue
test_loss,test_mae_loss, prediction_mae,prediction_rmse =train.eval(args,test_loader,cnn,criterion)
train_loss,train_mae_loss,_,_ =train.eval(args,train_loss_loader,cnn,criterion)
write_energy2tensorboard(train_loss,test_loss,prediction_rmse,epoch,writer,name='mse')
write_energy2tensorboard(train_mae_loss,test_mae_loss,prediction_mae,epoch,writer,name='_mae')
try:
is_best =test_mae_loss < best_loss
except:
best_loss =test_mae_loss
is_best =test_mae_loss<best_loss
best_loss =min(test_mae_loss,best_loss)
if is_best:
train.save({'epoch':epoch,
'cnn':cnn.state_dict(),
'optimizer':optimizer.state_dict(),
'best_loss':best_loss},args.save_dir,args.save_prefix,epoch)