def main():
print("Testing...")
args = parse_args()
model, config = load_from_path(args.save_path, return_config=True)
if args.dataset_path is not None:
config.defrost()
config.DATASET.ROOT = args.dataset_path
config.freeze()
evaluate(model, config)
def attention_test(args):
chosen_params = dict(params)
results = []
for attention in [False, True]:
chosen_params['attention'] = attention
runs = 5
acc_d = {}
f1_d = {}
for i in tqdm(range(runs)):
train_dataset, valid_dataset, test_dataset = data.load_dataset(
args.train_batch_size,
args.test_batch_size,
min_freq=chosen_params['min_freq'])
embedding = data.generate_embedding_matrix(
train_dataset.dataset.text_vocab,
rand=chosen_params['rand_emb'],
freeze=chosen_params['freeze'])
model = RNN(embedding, chosen_params)
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=chosen_params['lr'])
for epoch in range(args.epochs):
print(f'******* epoch: {epoch+1} *******')
train(model, train_dataset, optimizer, criterion, args)
evaluate(model, valid_dataset, criterion, 'Validation')
acc, f1 = evaluate(model, test_dataset, criterion, 'Test')
acc_d['acc_' + 'run' + str(i)] = acc
f1_d['f1_' + 'run' + str(i)] = f1
mean = np.mean(list(acc_d.values()))
std = np.std(list(acc_d.values()))
acc_d['mean'] = mean
acc_d['std'] = std
mean = np.mean(list(f1_d.values()))
std = np.std(list(f1_d.values()))
f1_d['mean'] = mean
f1_d['std'] = std
results.append((acc_d, f1_d))
with open(os.path.join(SAVE_DIR, 'attention.txt'), 'a') as f:
print(f'', file=f)
for idx, (acc, f1) in enumerate(results):
print('[no attention]' if idx == 0 else '[attention]', file=f)
print(acc, file=f)
print(f1, file=f)
def validate(self,
lr,
batch_size,
epochs,
input_classes,
embedding_dimensions=[4, 20, 10],
hidden_nodes=64,
save_labels_predictions=False,
weighted='unweighted'):
""" Performs k-fold cross-validation on the objects data set with a given model on given levels of the data
set with given training parameters.
Args:
n_classes = the number of classes is the dataset
hidden_nodes = the number of hidden nodes in the hidden layers of the LSTM model
number_of_layers= the number of layers of the LSTM model
lr = learning rate, default is 5e-3
batch_size = the batch size
epochs = the number of epochs
Returns:
- A tuple containing the average (precision, recall, F1-measure).
"""
scores = np.empty(self.k)
total_labels_predictions = pd.DataFrame()
for i in range(self.k):
# Initialises model.
model = LSTM(input_dimensions=[2, 13, 4],
embedding_dimensions=embedding_dimensions,
hidden_nodes=hidden_nodes,
n_layers=1,
n_classes=13,
input_classes=input_classes).to(device)
# Trains the model on the training set belonging to the iteration of the k-fold.
self.data.set_dialogue_ids(self.train_ids[i])
train(model, self.data, lr, batch_size, epochs, weighted)
# Tests the model on the test set belonging to the iteration of the k-fold.
self.data.set_dialogue_ids(self.test_ids[i])
if save_labels_predictions:
labels_predictions_fold, scores[i] = evaluate(
model, self.data, save_labels_predictions)
labels_predictions_fold = pd.DataFrame(
labels_predictions_fold.reshape(-1, 3))
total_labels_predictions = pd.concat(
[total_labels_predictions, labels_predictions_fold])
else:
scores[i] = evaluate(model, self.data, save_labels_predictions)
if save_labels_predictions:
return total_labels_predictions, scores
return scores
def translate():
data = request.get_json(force=True)
message = data['src']
message_normalized = unicode_to_ascii(message)
message_normalized = normalize_string(message)
if data['direction'] == 0: # en-fi
translation = evaluate(encoder_en_fi, attn_decoder_en_fi,
message_normalized, en_lang, fi_lang)
else:
translation = evaluate(encoder_fi_en, attn_decoder_fi_en,
message_normalized, fi_lang, en_lang)
return jsonify({"src": message, 'tgt': ' '.join(translation[0][:-1])})
def test_model_score(loader, output_lang, encoder, decoder, targetlang):
encoder.eval()
decoder.eval()
total = 0
total_loss = 0
predict_file = 'predict_temp'
predict_lines = open(predict_file, 'w')
for i, (input_batches, input_lengths, target_batches,
target_lengths) in enumerate(loader):
input_batches, input_lengths, target_batches, target_lengths = input_batches.to(
device), input_lengths.to(device), target_batches.to(
device), target_lengths.to(device)
decoded_words, all_decoder_outputs = evaluate(
input_batches, input_lengths, output_lang, target_batches,
target_lengths, encoder, decoder)
loss = masked_cross_entropy(all_decoder_outputs.contiguous(),
target_batches.contiguous(),
target_lengths)
total_loss += loss
total += 1
predict_lines.write(''.join(decoded_words) + '\n')
predict_lines.close()
if targetlang == 'zh':
target_file = '../iwslt-zh-en-processed/dev.tok.en'
else:
target_file = '../iwslt-vi-en-processed/dev.tok.en'
result = subprocess.run('cat {} | sacrebleu {}'.format(
predict_file, target_file),
shell=True,
stdout=subprocess.PIPE)
score = get_blue_score(str(result))
return (total_loss / total), score
def main(args):
workdir = Path(args.workdir)
logger = get_logger()
logger.info(f'config:\n{args}')
state = Saver.load_best_from_folder(workdir, map_location='cpu')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
logger.info(f'use device: {device}')
num_points = len(args.data.symmetry)
model = models.__dict__[args.model.name](num_points)
model.load_state_dict(state['model'])
model.to(device)
# datasets
valtransform = Transform(args.dsize, args.padding, args.data.meanshape,
args.data.meanbbox)
valdata = datasets.__dict__[args.data.name](**args.data.val)
valdata.transform = valtransform
valloader = DataLoader(valdata,
args.batch_size,
False,
num_workers=args.num_workers,
pin_memory=False)
score_fn = IbugScore(args.left_eye, args.right_eye)
evalmeter = evaluate(model, valloader, score_fn, device)
logger.info(evalmeter)
def eval_MH(x_dev, mask_dev, y_dev):
if FLAGS.use_pre_vec:
glove_vec = load_glove(FLAGS.vocab_file, FLAGS.vec_file)
else:
glove_vec = None
model = Text_attention2(glove_vec, FLAGS.seq_length, FLAGS.num_classes,
FLAGS.embedding_size, FLAGS.vocab_size,
FLAGS.batch_size, FLAGS.hidden_size,
FLAGS.num_attention_heads, FLAGS.num_hidden_layers,
FLAGS.sent_attention_size, FLAGS.initializer_range,
FLAGS.base_learning_rate, FLAGS.decay_rate)
print("Loading test data...")
start_time = time.time()
# x_test, y_test = load_polarity_data_and_labels(FLAGS.positive_data_file, FLAGS.negative_data_file)
# x_test, mask, y = process_file(x_test, y_test, FLAGS.vocab_file, FLAGS.seq_length)
session = tf.Session()
session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess=session, save_path=save_path) # 读取保存的模型
print('Testing...')
loss_test, acc_test = evaluate(session, model, x_dev, mask_dev, y_dev,
FLAGS.batch_size)
msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}'
print(msg.format(loss_test, acc_test))
"""
def test_inference(args, model, testloader):
""" Returns the test accuracy and loss.
"""
device = 'cuda' if torch.cuda.is_available(
) and not args.cpu_only else 'cpu'
confmat = evaluate(model, testloader, device, args.num_classes)
return confmat.acc_global, confmat.iou_mean, str(confmat)
def main(opts):
train_dataset = ColonCancerDataset('dataset/colon_cancer', train=True)
train_loader = DataLoader(train_dataset, batch_size=opts.batch_size, shuffle=True, num_workers=opts.num_workers, drop_last=False)
attention_model = AttentionModelColonCancer(squeeze_channels=True, softmax_smoothing=0)
feature_model = FeatureModelColonCancer(in_channels=3, out_channels=500)
classification_head = ClassificationHead(in_channels=500, num_classes=len(train_dataset.CLASSES))
ats_model = ATSModel(attention_model, feature_model, classification_head, n_patches=opts.n_patches,
patch_size=opts.patch_size)
ats_model = ats_model.to(opts.device)
optimizer = optim.Adam(ats_model.parameters(), lr=opts.lr)
logger = AttentionSaverTrafficSigns(opts.output_dir, ats_model, train_dataset, opts)
criterion = nn.CrossEntropyLoss()
entropy_loss_func = MultinomialEntropy(opts.regularizer_strength)
for epoch in range(opts.epochs):
train_loss, train_metrics = train(ats_model, optimizer, train_loader,
criterion, entropy_loss_func, opts)
with torch.no_grad():
test_loss, test_metrics = evaluate(ats_model, train_loader, criterion,
entropy_loss_func, opts)
logger(epoch, (train_loss, test_loss), (train_metrics, test_metrics))
def test():
data = np.load(config.test_dir, allow_pickle=True)
word_test = data["words"]
label_test = data["labels"]
test_dataset = NERDataset(word_test, label_test, config)
logging.info("--------Dataset Build!--------")
# build data_loader
test_loader = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False,
collate_fn=test_dataset.collate_fn)
logging.info("--------Get Data-loader!--------")
# Prepare model
if config.model_dir is not None:
model = BertNER.from_pretrained(config.model_dir)
model.to(config.device)
logging.info("--------Load model from {}--------".format(
config.model_dir))
else:
logging.info("--------No model to test !--------")
return
val_metrics = evaluate(test_loader, model, mode='test')
val_f1 = val_metrics['f1']
logging.info("test loss: {}, f1 score: {}".format(val_metrics['loss'],
val_f1))
val_f1_labels = val_metrics['f1_labels']
for label in config.labels:
logging.info("f1 score of {}: {}".format(label, val_f1_labels[label]))
def extract_sentiment_words():
# create vocabulary using wikitext2
train_txt, _, _ = torchtext.datasets.WikiText2.splits(TEXT)
TEXT.build_vocab(train_txt)
start = time.time()
x_train, y_train, x_val, y_val, rtrain, rtest = preprocess()
end = time.time()
print("PREPROCESSING TIME: {}".format(end - start))
ntokens = len(TEXT.vocab.stoi) # the size of vocabulary
# FIXME set up batched examples for better generality
# batch_size = 20
# eval_batch_size = 10
# configs
emsize = 200 # embedding dimension
nhid = 200 # feedforward dimension
nlayers = 2 # n encoders
nhead = 2 # multiattention heads
dropout = 0.2 # the dropout value
# initialize main torch vars
model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)
criterion = nn.CrossEntropyLoss().to(device)
lr = 0.05 # learning rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)
best_val_loss = float("inf")
epochs = 50
best_model = None
for epoch in range(1, epochs + 1):
epoch_start_time = time.time()
train_model(x_train, y_train, model, criterion, optimizer, scheduler, epoch)
val_loss = evaluate(x_val, y_val,rtest, model,criterion)
print('-' * 89)
print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '
'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),
val_loss, math.exp(val_loss)))
print('-' * 89)
if val_loss < best_val_loss:
best_val_loss = val_loss
best_model = model
scheduler.step()
# test_loss = evaluate(best_model, criterion, test_data)
# print('=' * 89)
# print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
# test_loss, math.exp(test_loss)))
# print('=' * 89)
return best_model
def main():
args = parse_config()
cfg = load_config(args.config)
# create folder
save_path = cfg.base.save_path
log_path = cfg.base.log_path
if os.path.exists(save_path):
warning = 'Save path {} exists.\nDo you want to overwrite it? (y/n)\n'.format(
save_path)
if not (args.overwrite or input(warning) == 'y'):
sys.exit(0)
else:
os.makedirs(save_path)
logger = SummaryWriter(log_path)
copy_config(args.config, save_path)
# print configuration
if args.print_config:
print_config({
'BASE CONFIG': cfg.base,
'DATA CONFIG': cfg.data,
'TRAIN CONFIG': cfg.train
})
else:
print_msg('LOADING CONFIG FILE: {}'.format(args.config))
# train
set_random_seed(cfg.base.random_seed)
model = generate_model(cfg)
train_dataset, test_dataset, val_dataset = generate_dataset(cfg)
estimator = Estimator(cfg.train.criterion, cfg.data.num_classes)
train(cfg=cfg,
model=model,
train_dataset=train_dataset,
val_dataset=val_dataset,
estimator=estimator,
logger=logger)
# test
print('This is the performance of the best validation model:')
checkpoint = os.path.join(save_path, 'best_validation_weights.pt')
evaluate(cfg, model, checkpoint, test_dataset, estimator)
print('This is the performance of the final model:')
checkpoint = os.path.join(save_path, 'final_weights.pt')
evaluate(cfg, model, checkpoint, test_dataset, estimator)
def hyperparam_optim_test(args):
var_params = {
'cell_name': ['lstm'],
'hidden_size': [50, 150, 300],
'num_layers': [2, 4, 5],
'min_freq': [0, 100, 500],
'lr': [1e-3, 1e-4, 1e-7],
'dropout': [0, 0.4, 0.7],
'freeze': [False, True],
'rand_emb': [False, True],
'attention': [False]
}
results = []
for i in tqdm(range(10)):
chosen_params = {k: random.choice(v) for (k, v) in var_params.items()}
train_dataset, valid_dataset, test_dataset = data.load_dataset(
args.train_batch_size,
args.test_batch_size,
min_freq=chosen_params['min_freq'])
embedding = data.generate_embedding_matrix(
train_dataset.dataset.text_vocab,
rand=chosen_params['rand_emb'],
freeze=chosen_params['freeze'])
model = RNN(embedding, chosen_params)
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=chosen_params['lr'])
for epoch in range(args.epochs):
print(f'******* epoch: {epoch+1} *******')
train(model, train_dataset, optimizer, criterion, args)
evaluate(model, valid_dataset, criterion, 'Validation')
acc, f1 = evaluate(model, test_dataset, criterion, 'Test')
result = dict(chosen_params)
result['acc'] = acc
result['f1'] = f1
results.append(result)
with open(os.path.join(SAVE_DIR, 'params_search.txt'), 'a') as f:
for result in results:
print(result, file=f)
def cell_comparison_test(args):
train_dataset, valid_dataset, test_dataset = data.load_dataset(
args.train_batch_size, args.test_batch_size)
embedding = data.generate_embedding_matrix(
train_dataset.dataset.text_vocab)
var_params = {
'hidden_size': [50, 150, 300],
'num_layers': [1, 2, 4],
'dropout': [0.1, 0.4, 0.7],
'bidirectional': [True, False],
'attention': [False]
}
for idx, (key, values) in enumerate(var_params.items()):
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.set_title('Variable ' + key)
for cell_name in tqdm(['rnn', 'lstm', 'gru']):
results = []
for i in range(len(values)):
current_params = {
k: v[i] if k == key else v[1]
for (k, v) in var_params.items()
}
current_params['cell_name'] = cell_name
model = RNN(embedding, current_params)
model
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
for epoch in range(args.epochs):
print(f'******* epoch: {epoch+1} *******')
train(model, train_dataset, optimizer, criterion, args)
evaluate(model, valid_dataset, criterion, 'Validation')
result, _ = evaluate(model, test_dataset, criterion, 'Test')
results.append(result)
ax.plot(values, results, marker='o', label=cell_name)
plt.legend(loc='best')
plt.xlabel(key)
plt.ylabel('accuracy')
fig.savefig(os.path.join(SAVE_DIR, key + '.png'))
plt.close(fig)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--data_path",
default="./data/binary_memory_50/binary_data_memory_test.pkl",
type=str,
help="test data path",
)
parser.add_argument("--model_path",
required=True,
type=str,
help="model path")
parser.add_argument("--args_path",
required=True,
type=str,
help="args path")
args_ = parser.parse_args()
args = torch.load(args_.args_path)
args.eval_data_file = args_.data_path
args.eval_output_dir = os.path.join(args.output_dir, "test_result/")
os.makedirs(args.eval_output_dir, exist_ok=True)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.DEBUG if args.debug else logging.INFO,
)
n_pitcher = [*[2] * args.n_pitcher_disc, *[5] * args.n_pitcher_cont]
n_batter = [*[2] * args.n_batter_disc, *[5] * args.n_batter_cont]
n_state = [4, 4, 5, 8, 9, 10, 16, 22, 23]
model = BaseballTransformer(
n_pitcher,
n_batter,
n_state,
args.n_memory_layer,
args.n_encoder_layer,
memory_len=args.memory_len,
d_model=args.d_model,
n_head=args.n_head,
dropout=args.dropout,
attention_type=args.attention_type,
).to(args.device)
if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
model = torch.nn.DataParallel(model)
model.load_state_dict(torch.load(args_.model_path), strict=False)
model.eval()
result, _, f1_log, cm = evaluate(args, args.eval_data_file, model)
result_dir = os.path.join(args.eval_output_dir, "test_results.txt")
utils.print_result(result_dir, result, f1_log, cm)
logger.info(" Results are saved to, %s", result_dir)
draw_cm(cm, args.eval_output_dir)
def run():
args = parse_opts()
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "9"
# GLOBAL VARS #
MODE = args.mode
CLASS_WEIGHT = False
N_EP = 20
FLATTEN = args.flatten
RNN = args.rnn
BATCH_SIZE = args.batch_size
####
datasets, dataloaders = init_dataset(
BATCH_SIZE, single_channel=args.single_channel)
print('[Train] class counts', np.unique(
datasets['train'].target_vals, return_counts=True))
print('[Test] class counts', np.unique(
datasets['test'].target_vals, return_counts=True))
n_ch = 1 if args.single_channel else 3
if MODE == 'min':
in_channels = datasets['train'].min_depth*n_ch
elif MODE == 'max':
in_channels = datasets['train'].max_depth*n_ch
torch.manual_seed(0)
# init net
net = init_net(opt=args.model_idx, in_channels=in_channels)
class_weight = None
if CLASS_WEIGHT:
cnts = Counter(datasets['train'].target_vals)
n = len(datasets['train'])
class_weight = [max(cnts.values())/cnts['0'],
max(cnts.values())/cnts['1']]
class_weight = torch.FloatTensor(class_weight)
cross_entrp_loss = nn.CrossEntropyLoss(weight=class_weight).cuda()
focal_loss = FocalLoss().cuda()
optimizer = optim.Adam(net.parameters(), lr=0.000027)
criterion = cross_entrp_loss
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(
# optimizer, 'min', verbose=True, patience=7)
for ep in range(N_EP):
train_epoch(net, dataloaders['train'], optimizer,
criterion, ep, scheduler=None, flatten=FLATTEN, MODE=MODE, rnn=RNN)
valid_loss = evaluate(net, dataloaders['test'], criterion,
ep, flatten=FLATTEN, MODE=MODE, rnn=RNN)
def autoencoder(self, sentence):
encoder = torch.load('test_encoder_imdb10000_0.0037651115254803288',
map_location={'cuda:0': 'cpu'})
decoder = torch.load('test_decoder_imdb10000_0.0037651115254803288',
map_location={'cuda:0': 'cpu'})
output_words, _, loss = evaluate(encoder, decoder, sentence,
self.input_lang, self.output_lang)
return output_words, loss
def main(args):
chosen_params = dict(params)
chosen_params['attention'] = True
train_dataset, valid_dataset, test_dataset = data.load_dataset(
args.train_batch_size, args.test_batch_size)
embedding = data.generate_embedding_matrix(
train_dataset.dataset.text_vocab, freeze=False)
model = RNN(embedding, chosen_params)
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
for epoch in range(args.epochs):
print(f'******* epoch: {epoch} *******')
train(model, train_dataset, optimizer, criterion, args)
evaluate(model, valid_dataset, criterion, 'Validation')
evaluate(model, test_dataset, criterion, 'Test')
def test(config, model, test_data):
"""
test function
"""
test_acc, test_loss, test_report, test_confusion = evaluate(config, model, test_data, test=True)
msg = '\nTest Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
def build_sensitivity_profile(model, criterion, data_loader_test):
quant_layer_names = []
for name, module in model.named_modules():
if name.endswith("_quantizer"):
module.disable()
layer_name = name.replace("._input_quantizer",
"").replace("._weight_quantizer", "")
if layer_name not in quant_layer_names:
quant_layer_names.append(layer_name)
for i, quant_layer in enumerate(quant_layer_names):
print("Enable", quant_layer)
for name, module in model.named_modules():
if name.endswith("_quantizer") and quant_layer in name:
module.enable()
print(F"{name:40}: {module}")
with torch.no_grad():
evaluate(model, criterion, data_loader_test, device="cuda")
for name, module in model.named_modules():
if name.endswith("_quantizer") and quant_layer in name:
module.disable()
print(F"{name:40}: {module}")
def embedding_baseline_test(args):
chosen_params = dict(params)
results = []
for rand_emb in [True, False]:
chosen_params['rand_emb'] = rand_emb
train_dataset, valid_dataset, test_dataset = data.load_dataset(
args.train_batch_size,
args.test_batch_size,
min_freq=chosen_params['min_freq'])
embedding = data.generate_embedding_matrix(
train_dataset.dataset.text_vocab,
rand=chosen_params['rand_emb'],
freeze=chosen_params['freeze'])
result = {}
for m in ['baseline', 'rnn']:
if m == 'rnn':
model = RNN(embedding, chosen_params)
else:
model = Baseline(embedding)
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
for epoch in range(args.epochs):
print(f'******* epoch: {epoch} *******')
train(model, train_dataset, optimizer, criterion, args)
evaluate(model, valid_dataset, criterion, 'Validation')
acc, f1 = evaluate(model, test_dataset, criterion, 'Test')
result[m + '_acc_rand_emb' + str(rand_emb)] = acc
result[m + '_f1_rand_emb' + str(rand_emb)] = f1
results.append(result)
with open(os.path.join(SAVE_DIR, 'embedding_baseline.txt'), 'a') as f:
for res in results:
print(res, file=f)
def test_on_fer_test_set(fer_path, model_type="CustomVGG"):
start_time = time()
fer = pd.read_csv(fer_path)
if "attribution" not in fer:
raise Exception(
"Fer not split between train/val/test. Please run split_fer script."
)
fer_test = fer[fer["attribution"] == "test"].reset_index()
model = load_model(model_type=model_type)
print("Loaded fer test set and model in {}s".format(
round(time() - start_time, 2)))
start_time = time()
def preprocess_batch(pixelstring_batch, emotions_batch, DEVICE):
if model_type == "CustomVGG":
return preprocess_batch_custom_vgg(pixelstring_batch,
emotions_batch, DEVICE, False,
config["loss_mode"])
elif model_type == "DenseSIFTHybrid":
return preprocess_batch_dense_sift_hybrid(pixelstring_batch,
emotions_batch, DEVICE,
False,
config["loss_mode"])
elif model_type == "SIFTHybrid":
return preprocess_batch_sift_hybrid(pixelstring_batch,
emotions_batch, DEVICE, False,
config["loss_mode"])
use_descriptors = (model_type == "DenseSIFTHybrid"
or model_type == "SIFTHybrid")
dummy_weights = torch.FloatTensor([1] * len(config["catslist"])).to(
DEVICE) # we don't care about the test loss value here.
proba, _, acc, cm1, cm2, acc_fact = evaluate(
model,
fer_test,
preprocess_batch,
dummy_weights,
DEVICE,
compute_cm=True,
use_descriptors=use_descriptors)
print("FINAL ACCURACY: {}".format(acc))
print("Average predicted proba for right class: {}".format(proba))
print("Duration on {} test faces: {}s".format(
len(fer_test), round(time() - start_time, 2)))
print("Accuracy with grouped classes : {}".format(acc_fact))
print("Close the confusion matrices to end the script.")
plot_confusion_matrix(cm1, config["catslist"])
plot_confusion_matrix(cm2, ["bad", "good", "surprise", "neutral"])
def get_scores(self,
dev_or_test,
data_dir=DATA_DIR,
models_dir=MODELS_BASE_DIR,
device=DEVICE):
if dev_or_test not in ("dev", "test"):
raise ValueError
# BioBERT_en
id2scores_m1 = bert_predict(os.path.join(models_dir, "biobert-en"),
test_or_dev=dev_or_test,
use_data="en",
max_seq_length=256,
batch_size=16,
data_dir=data_dir,
device=device,
return_logits=True)
test_loader, dev_loader, V, Tv, C, T = get_test_data("clstm",
"en",
max_seq_len=256,
batch_size=64,
data_dir=data_dir)
vocab_size = V
titles_vocab_size = Tv
num_classes = C
T = T.to(device)
model = models.ICDCodeAttentionLSTM(vocab_size,
400,
num_classes,
T,
Tv=titles_vocab_size,
h=300,
bidirectional=True)
model.load_state_dict(
torch.load(
os.path.join(MODELS_BASE_DIR, "clstm-pubmed-en", "model.pt")))
model.to(device)
model.eval()
if dev_or_test == "dev":
data_loader = dev_loader
else:
data_loader = test_loader
_, (scores_m2, _, _, scores_m2_ids, _) = evaluate(data_loader,
model,
device,
no_labels=True)
id2scores_m2 = {
val: scores_m2[i]
for i, val in enumerate(scores_m2_ids)
}
return id2scores_m1, id2scores_m2, num_classes
def main(args):
weights_file = (args.weightsFile)
positiveImagePath = (args.positiveTestImages)
negativeImagePath = (args.negativeTestImages)
os.system("python3 createTrainingData.py {} {} {}".format(
positiveImagePath, negativeImagePath, 1))
X_LL, X_LH, X_HL, X_HH, X_index, Y, imageCount = readWaveletData(
positiveImagePath, negativeImagePath, positiveTestImagePath,
positiveTestImagePath)
X_LL = np.array(X_LL)
X_LL = X_LL.reshape((imageCount, height, width, depth))
X_LH = np.array(X_LH)
X_LH = X_LH.reshape((imageCount, height, width, depth))
X_HL = np.array(X_HL)
X_HL = X_HL.reshape((imageCount, height, width, depth))
X_HH = np.array(X_HH)
X_HH = X_HH.reshape((imageCount, height, width, depth))
CNN_model = createModel(height, width, depth, num_classes)
CNN_model.load_weights(weights_file)
evaluate(CNN_model, X_LL, X_LH, X_HL, X_HH, Y)
def main():
holdout_test = True
fine_tune_task = config["task"]
logger.info("init model...")
pretrained_name = get_pretrained_name(config["model_name"])
if fine_tune_task == "paws":
num_labels = 2
train_loader, dev_loader = load_paws(config["PAWS-QQP"],
balanced=config["balanced"])
elif fine_tune_task == "standsent":
num_labels = 5
train_loader, dev_loader, test_loader = load_stanford(
config["STANFORD_LOC"],
phrase_len=config["phrase_len"],
reserve_test=holdout_test)
else:
logger.error("unsupport fine tune task: {}".format(fine_tune_task))
model, tokenizer = load_model(pretrained_name=pretrained_name,
load_tuned=False,
num_labels=num_labels)
optimizer = AdamW(model.parameters(), lr=1e-5)
logger.info("training...")
model.train()
init_prec, init_loss, best_prec, best_loss, best_model = train_iter(
model,
tokenizer,
optimizer,
train_loader,
dev_loader,
task=fine_tune_task,
early_stopping=True,
max_epochs=config["n_epochs"],
print_every=config["print_every"],
evaluate_every=config["evaluate_every"])
logger.info("done training.")
training_info_str = \
""" training summary:
training loss {} -> {}
test precision {} -> {}
""".format(init_loss, best_loss, init_prec, best_prec)
logger.info(training_info_str)
if holdout_test and (fine_tune_task == "standsent"):
# evaluate on holdout test set (with phrases of various lengths)
test_prec = evaluate(best_model, tokenizer, test_loader,
fine_tune_task)
logger.info("precision on holdout test: {}".format(test_prec))
def train(self, args, model_class: nn.Module, train) -> nn.Module:
"""Trains a model locally using Hogwild!
Args:
model_class (nn.Module): The class of a PyTorch model.
train: A function that trains the model instance locally.
It must also encapsulate the dataloading.
"""
torch.manual_seed(args.seed)
model = model_class()
model.share_memory(
) # gradients are allocated lazily, so they are not shared here
processes = []
if args.capture_results:
start = timer()
for rank in range(args.num_processes):
p = mp.Process(target=train, args=(rank, args, model))
p.start()
processes.append(p)
for p in processes:
p.join()
acc, loss = evaluate(args, model)
if args.capture_results:
end = timer()
acc, loss = evaluate(args, model)
t = end - start
print('{} {} {}'.format(acc, loss, t))
return model
def main():
# reproducibility
seed = CONFIG['RANDOM_SEED']
set_random_seed(seed)
# load dataset
train_dataset, test_dataset, val_dataset = generate_data(
CONFIG['TRAIN_PATH'], CONFIG['TEST_PATH'], CONFIG['TEST_PATH'],
CONFIG['INPUT_SIZE'])
save_dir = os.path.split(CONFIG['SAVE_PATH'])[0]
if not os.path.exists(save_dir):
os.makedirs(save_dir)
model = MyModel(bbr_net.resnet_bbr,
CONFIG['BOTTLENECK_SIZE'],
CONFIG['NUM_CLASS'],
pretrained=False).cuda()
# train
model, record_epochs, val_ious, losses = train(
model=model,
train_dataset=train_dataset,
val_dataset=val_dataset,
epochs=CONFIG['EPOCHS'],
learning_rate=CONFIG['LEARNING_RATE'],
batch_size=CONFIG['BATCH_SIZE'],
save_path=CONFIG['SAVE_PATH'])
pickle.dump((record_epochs, val_ious, losses),
open(CONFIG['RECORD_PATH'], 'wb'))
# test
visualized_images_path = CONFIG['VISUALIZED_IMAGES_PATH']
if not os.path.exists(visualized_images_path):
os.makedirs(visualized_images_path)
evaluate(CONFIG['SAVE_PATH'], test_dataset, visualized_images_path)
def evaluate_hw2(cfg: DictConfig) -> float:
main_utils.init(cfg)
load_v2()
# Load dataset
path_image_train = '/datashare/train2014/COCO_train2014_'
path_question_train = '/datashare/v2_OpenEnded_mscoco_train2014_questions.json'
train_dataset = VQADataset(path_answers=cfg['main']['paths']['train'],
path_image=path_image_train,
path_questions=path_question_train)
path_image_val = '/datashare/val2014/COCO_val2014_'
path_question_train = '/datashare/v2_OpenEnded_mscoco_val2014_questions.json'
val_dataset = VQADataset(path_answers=cfg['main']['paths']['validation'],
path_image=path_image_val,
path_questions=path_question_train,
word_dict=train_dataset.word_dict)
eval_loader = DataLoader(val_dataset,
cfg['train']['batch_size'],
shuffle=True,
num_workers=cfg['main']['num_workers'])
image_dim = train_dataset.pic_size
output_dim = 2410
model = VQAModel(batch_size=cfg['train']['batch_size'],
word_vocab_size=train_dataset.vocab_size,
lstm_hidden=cfg['train']['num_hid'],
output_dim=output_dim,
dropout=cfg['train']['dropout'],
word_embedding_dim=cfg['train']['word_embedding_dim'],
question_output_dim=cfg['train']['question_output_dim'],
image_dim=image_dim,
last_hidden_fc_dim=cfg['train']['last_hidden_fc_dim'])
if torch.cuda.is_available():
model = model.cuda()
model.load_state_dict(
torch.load('model.pkl',
map_location=lambda storage, loc: storage)['model_state'])
model.train(False)
eval_score, eval_loss = evaluate(model, eval_loader)
print(f"The evaluation score is {eval_score}")
return eval_score
def main(opts):
train_dataset = MNIST('dataset/mega_mnist', train=True)
train_loader = DataLoader(train_dataset,
batch_size=opts.batch_size,
shuffle=True,
num_workers=1)
test_dataset = MNIST('dataset/mega_mnist', train=False)
test_loader = DataLoader(test_dataset,
shuffle=False,
batch_size=opts.batch_size,
num_workers=1)
attention_model = AttentionModelMNIST(squeeze_channels=True,
softmax_smoothing=1e-4)
feature_model = FeatureModelMNIST(in_channels=1)
classification_head = ClassificationHead(in_channels=32, num_classes=10)
ats_model = ATSModel(attention_model,
feature_model,
classification_head,
n_patches=opts.n_patches,
patch_size=opts.patch_size)
ats_model = ats_model.to(opts.device)
optimizer = optim.Adam(ats_model.parameters(), lr=opts.lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=opts.decrease_lr_at,
gamma=0.1)
logger = AttentionSaverMNIST(opts.output_dir, ats_model, test_dataset,
opts)
criterion = nn.CrossEntropyLoss()
entropy_loss_func = MultinomialEntropy(opts.regularizer_strength)
for epoch in range(opts.epochs):
train_loss, train_metrics = train(ats_model, optimizer, train_loader,
criterion, entropy_loss_func, opts)
with torch.no_grad():
test_loss, test_metrics = evaluate(ats_model, test_loader,
criterion, entropy_loss_func,
opts)
logger(epoch, (train_loss, test_loss), (train_metrics, test_metrics))
scheduler.step()
def test_model_loss(loader, output_lang, encoder, decoder):
encoder.eval()
decoder.eval()
total = 0
total_loss = 0
for i, (input_batches, input_lengths, target_batches, target_lengths) in enumerate(loader):
input_batches, input_lengths, target_batches, target_lengths = input_batches.to(device), input_lengths.to(device), target_batches.to(device), target_lengths.to(device)
decoded_words, all_decoder_outputs = evaluate(input_batches, input_lengths, output_lang, target_batches, target_lengths, encoder, decoder)
#print(all_decoder_outputs.shape)
loss = masked_cross_entropy(
all_decoder_outputs.contiguous(),
target_batches.contiguous(),
target_lengths)
total_loss += loss.item()
total += 1
return (total_loss / total)
q_net.cuda()
agent = dqn.DQNAgent(q_net, args.double_dqn, train_env.num_actions)
if args.noisy_net:
train_policy = GreedyEpsilonPolicy(0, agent)
else:
train_policy = LinearDecayGreedyEpsilonPolicy(
args.train_start_eps,
args.train_final_eps,
args.train_eps_num_steps,
agent)
eval_policy = GreedyEpsilonPolicy(args.eval_eps, agent)
replay_memory = ReplayMemory(args.replay_buffer_size)
replay_memory.burn_in(train_env, agent, args.burn_in_frames)
evaluator = lambda logger: evaluate(eval_env, eval_policy, 10, logger)
train(agent,
train_env,
train_policy,
replay_memory,
args.gamma,
args.batch_size,
args.num_iters,
args.frames_per_update,
args.frames_per_sync,
args.frames_per_eval,
evaluator,
args.output)
