def main(args):
dataset = pd.read_csv(args.dataset, index_col=0, keep_default_na=False)
lookup = {
"pooled": bert_models.PooledModel,
"entity": bert_models.EntityModel
}
bert_class = lookup[args.bert_model_class]
bert = bert_class.from_model_checkpoint(args.bert_config_file,
args.bert_weights_file)
print(bert)
if len(args.classes) > 0:
unique_classes = sorted(args.classes)
dataset = dataset[dataset["PREDICATE"].isin(unique_classes)]
else:
unique_classes = sorted(set(dataset["PREDICATE"].values))
y = utils.get_onehot_from_labels(dataset["PREDICATE_LABEL"])
assert dataset.shape[0] == y.shape[0]
labels2preds = {i: lab for (i, lab) in enumerate(unique_classes)}
print(labels2preds)
docs = dataset["SENTENCE"].values
scores = bert.predict(docs, predict_classes=False).numpy()
utils.save_predictions(dataset, scores, args.outfile, labels2preds)
def evaluate(model_type, ckpt_weights_file, eval_res_folder, x_data_path,
y_data_path, batch_size, max_seq_length, vocab_size, n_classes,
embed_dim, emb_trainable, model_name, rnn_unit_type, loss_type,
hidden_dim, hidden_activation, out_activation, bidirectional,
learning_rate, verbose):
x_data, y_data = utils.create_model_input_data(x_data_path, y_data_path,
max_seq_length)
if model_type == "rnn":
if y_data is not None:
loss = evaluate_rnn_model(
ckpt_weights_file, x_data, y_data, batch_size, max_seq_length,
vocab_size, n_classes, embed_dim, emb_trainable, model_name,
rnn_unit_type, loss_type, hidden_dim, hidden_activation,
out_activation, bidirectional, learning_rate, verbose)
predictions = predict_rnn_model(ckpt_weights_file, x_data, batch_size,
max_seq_length, vocab_size, n_classes,
embed_dim, emb_trainable, model_name,
rnn_unit_type, loss_type, hidden_dim,
hidden_activation, out_activation,
bidirectional, learning_rate, verbose)
utils.save_predictions(eval_res_folder, predictions)
elif model_type == "fnn":
if y_data is not None:
loss = evaluate_ffn_model()
predictions = predict_ffn_model()
utils.save(predictions)
else:
raise ValueError(
"Unknown model type. The supported types are: rnn|ffn")
print("Loss: %.3f" % loss)
def main(data_path, model):
if model == 'char-gram':
mod = chargram_cnn.CharGramCNN()
elif model == 'nb-svm':
mod = nblr.NaiveBayes()
elif model == 'random-forest':
mod = random_forest.ToxicRandomForest()
else:
raise Exception('Unknown model: \'%s\'' % model)
# Load data
logger.info('Loading data...')
mod.load_data(data_path)
# Fit model
logger.info('Fitting model...')
mod.fit()
# Generate predictions df
logger.info('Generating training predictions...')
train_preds = mod.train_predictions()
logger.info('Generating validation predictions...')
val_preds = mod.validation_predictions()
logger.info('Generating test predictions...')
test_preds = mod.test_predictions()
# Save predictions to files
logger.info('Saving to files...')
utils.save_predictions(data_path, train_preds, val_preds, test_preds,
model)
def main(
source_dirs,
target_dir,
averaging,
):
source_dir_paths = [
os.path.join(config.PREDICTIONS_PATH, dir_) for dir_ in source_dirs
]
target_dir_path = os.path.join(config.PREDICTIONS_PATH, target_dir)
os.makedirs(target_dir_path, exist_ok=True)
print('source directories:\n' + '\n'.join(source_dir_paths) + '\n')
print('target directory:\n' + target_dir_path)
mean = np.mean if averaging == 'mean' else gmean
dirs_filenames = [os.listdir(dir_) for dir_ in source_dir_paths]
assert dirs_filenames
filenames = dirs_filenames[0]
for (other_filenames, dir_) in zip(dirs_filenames, source_dir_paths):
assert set(filenames) == set(other_filenames)
for filename in tqdm(filenames):
predictions = []
for prediction_row in zip(
*(
npz_file_iterator(os.path.join(dir_path, filename))
for dir_path in source_dir_paths
),
):
prediction = mean(np.vstack(prediction_row), axis=0)
predictions.append(prediction)
save_predictions(predictions, os.path.join(target_dir_path, filename))
def evaluate_ffn(params):
x_data, y_data = utils.load_data(params["eval_x_data"],
params["eval_y_data"])
fnn_model = model.FFNModel(input_size=x_data.shape[1],
output_size=params["output_size"],
model_name=params["model_name"],
hidden_activation=params["hidden_activation"],
out_activation=params["out_activation"],
hidden_dims=params["hidden_dims"],
layers=params["layers"],
kernel_initializer=params["kernel_initializer"],
kernel_regularizer=params["kernel_regularizer"],
dropouts=[0.0])
utils.load_model(params["eval_weights_ckpt"], fnn_model,
params["learning_rate"])
if y_data is not None:
print("Model from checkpoint %s was loaded." %
params["eval_weights_ckpt"])
metrics_names, scores = fnn_model.evaluate(
x_data,
y_data,
batch_size=params["batch_size"],
verbose=params["verbose"])
loss = scores[0]
print("Evaluation loss: %.3f" % loss)
predictions = fnn_model.predict(x_data,
batch_size=params["batch_size"],
verbose=params["verbose"])
utils.save_predictions(predictions, params["eval_res_folder"],
fnn_model.model_name + "_predictions.txt")
def test_end(self, step_outputs):
total_loss, total_predictions, total_labels, total_unique_ids, total_prediction_probs = list(
), list(), list(), list(), list()
for x in step_outputs:
total_loss.append(x["loss"])
total_predictions.append(x["predictions"])
total_labels.append(x["labels"])
total_unique_ids.append(x["unique_ids"])
total_prediction_probs.append(x["prediction_probs"])
avg_loss = torch.stack(total_loss).double().mean()
unique_ids = torch.cat(total_unique_ids).long()
predictions = torch.cat(total_predictions).long()
prediction_probs = torch.cat(total_prediction_probs, dim=0).double()
labels = torch.cat(total_labels).long()
correct = predictions.eq(labels.view_as(predictions)).long()
accuracy = correct.double().mean()
save_predictions(self.hparams.output_path,
unique_ids.data.cpu().numpy(),
predictions.data.cpu().numpy(),
labels.data.cpu().numpy(),
correct.cpu().numpy(), [
prediction_probs[:, i].data.cpu().numpy()
for i in range(self.bert_classifier.label_size)
], f"{self.bert_classifier.name}-test")
return {
"loss": avg_loss,
"progress_bar": {
"test_loss": avg_loss,
"test_accuracy": accuracy
},
"log": {
"test_loss": avg_loss,
"test_accuracy": accuracy
}
}
def test_model(
model: nn.Module,
dataloaders: Dict[str, torch.utils.data.dataloader.DataLoader],
device: torch.device,
save_pred_path: Union[None, str] = None,
is_inception: bool = False
) -> None:
# Set model to evaluate mode
model.eval()
# Initialize predictions list
predictions = []
# class_to_idx dict (class: idx) and reverse
class_to_idx = dataloaders['train'].dataset.class_to_idx
idx_to_class = {idx: cls for cls, idx in class_to_idx.items()}
# Iterate over data.
progress_bar = tqdm(dataloaders['test'], desc=f'Test: ')
for i, (inputs, paths) in enumerate(progress_bar):
inputs = inputs.to(device)
with torch.no_grad():
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
# accumulate the preductions after each batch
assert len(preds.shape) == 1
predictions.extend(list(zip(paths, preds.tolist())))
# save predictions
if save_pred_path is not None:
save_predictions(save_pred_path, predictions, idx_to_class)
def evaluate_rnn(params):
x_data, y_data = utils.load_data(params["eval_x_data"],
params["eval_y_data"])
rnn_model = model.RNNModel(
max_seq_length=x_data.shape[1],
input_size=params["input_size"],
output_size=params["output_size"],
embed_dim=params["embed_dim"],
emb_trainable=params["emb_trainable"],
model_name=params["model_name"],
hidden_activation=params["hidden_activation"],
out_activation=params["out_activation"],
hidden_dim=params["hidden_dims"][0],
kernel_initializer=params["kernel_initializer"],
kernel_regularizer=params["kernel_regularizer"],
recurrent_regularizer=params["recurrent_regularizer"],
input_dropout=0.0,
recurrent_dropout=0.0,
rnn_unit_type=params["rnn_unit_type"],
bidirectional=params["bidirectional"],
attention=params["attention"],
embs_matrix=params["embs_matrix"])
utils.load_model(params["eval_weights_ckpt"], rnn_model,
params["learning_rate"])
print("Model from checkpoint %s was loaded." % params["eval_weights_ckpt"])
# if y_data is not None:
# metrics_names, scores = rnn_model.evaluate(x_data, y_data, batch_size=params["batch_size"], verbose=params["verbose"])
# loss = scores[0]
# print("Evaluation loss: %.3f"%loss)
sample_idxs = np.random.randint(x_data.shape[0],
size=params["attn_sample_size"])
x_data_sample = x_data[sample_idxs, :]
cPickle.dump(
sample_idxs,
open(os.path.join(params["eval_res_folder"], "sample_idxs.pickle"),
"wb"))
if params["attention"]:
attention_weights = get_attention_weights(rnn_model, x_data_sample)
print("Attention weights shape: ", attention_weights.shape)
cPickle.dump(
attention_weights,
open(
os.path.join(params["eval_res_folder"], "attn_weights.pickle"),
"wb"))
vocab = cPickle.load(open(params["vocab_file"]))
if params["plot_attn"]:
inverse_vocab = {value: key for key, value in vocab.items()}
utils.plot_attention_weights(x_data_sample,
attention_weights,
inverse_vocab,
params["eval_res_folder"],
ids=sample_idxs)
predictions = rnn_model.predict(x_data,
batch_size=params["batch_size"],
verbose=params["verbose"])
utils.save_predictions(predictions, params["eval_res_folder"],
rnn_model.model_name + "_predictions.txt")
def predict_test(x_test):
if softmax_activation:
y_test_raw = model.predict(x_test, verbose=1)
y_test = np.argmax(y_test_raw, axis=1)
else:
y_test_raw = model.predict(x_test, verbose=1)
y_test = (y_test_raw > 0.5).astype(int).sum(axis=1) - 1
save_predictions(y_test, files, save_name=output_csv)
def test_end(self, step_outputs):
total_loss, total_predictions, total_labels, total_unique_ids, total_prediction_probs = list(
), list(), list(), list(), list()
for x in step_outputs:
total_loss.append(x["loss"])
total_predictions.append(x["predictions"])
total_labels.append(x["labels"])
total_unique_ids.append(x["unique_ids"])
total_prediction_probs.append(x["prediction_probs"])
avg_loss = torch.stack(total_loss).double().mean()
unique_ids = torch.cat(total_unique_ids).long()
predictions = torch.cat(total_predictions).long()
prediction_probs = torch.cat(total_prediction_probs, dim=0).double()
labels = torch.cat(total_labels).long()
correct = predictions.eq(labels.view_as(predictions)).long()
real_token_indices = (
labels.flatten() !=
BertTokenClassificationDataset.POS_IGNORE_LABEL_IDX
).nonzero().flatten()
accuracy = predictions.flatten().index_select(
0, real_token_indices).eq(labels.flatten().index_select(
0, real_token_indices)).double().mean()
predictions_data, labels_data, correct_data = list(), list(), list()
prediction_probs_data = [[] for i in range(self.num_labels)]
for pred, l, c, prob in zip(predictions.cpu(), labels.cpu(),
correct.cpu(), prediction_probs.cpu()):
real_token_indices = (
l.flatten() !=
BertTokenClassificationDataset.POS_IGNORE_LABEL_IDX
).nonzero().flatten()
predictions_data.append(pred.flatten().index_select(
0, real_token_indices).tolist())
labels_data.append(l.flatten().index_select(
0, real_token_indices).tolist())
correct_data.append(c.flatten().index_select(
0, real_token_indices).tolist())
for i in range(self.num_labels):
prediction_probs_data[i].append(
prob[:, i].flatten().index_select(
0, real_token_indices).tolist())
save_predictions(self.hparams.output_path,
unique_ids.data.cpu().numpy(), predictions_data,
labels_data, correct_data, prediction_probs_data,
f"{self.hparams.name}-test")
return {
"loss": avg_loss,
"progress_bar": {
"test_loss": avg_loss,
"test_accuracy": accuracy
},
"log": {
"test_loss": avg_loss,
"test_accuracy": accuracy
}
}
def evaluate_single_model(pattern_id,
pattern_iter_output_dir,
eval_data,
dev_data,
eval_config,
results_dict,
dev_result_all,
eval_result_all,
save_logits=False,
save_predictions=False):
# if not wrapper:
wrapper = TransformerModelWrapper.from_pretrained(
pattern_iter_output_dir)
eval_result = wrapper.eval(
eval_data, eval_config.per_gpu_eval_batch_size, eval_config.n_gpu, eval_config.metrics)
dev_result = wrapper.eval(
dev_data, eval_config.per_gpu_eval_batch_size, eval_config.n_gpu, eval_config.metrics)
logger.info(
"--- RESULT (pattern_id={}) ---".format(pattern_id))
logger.info("eval results:")
logger.info(eval_result['scores'])
logger.info("dev results:")
logger.info(dev_result['scores'])
results_dict['eval_set'] = eval_result['scores']
results_dict['dev_set'] = dev_result['scores']
for metric, value in eval_result['scores'].items():
eval_result_all[metric][pattern_id].append(value)
for metric, value in dev_result['scores'].items():
dev_result_all[metric][pattern_id].append(value)
if save_logits:
save_logits(os.path.join(pattern_iter_output_dir,
'eval_logits.txt'), eval_result['logits'])
save_logits(os.path.join(pattern_iter_output_dir,
'dev_logits.txt'), dev_result['logits'])
if save_predictions:
save_predictions(os.path.join(
pattern_iter_output_dir, 'eval_predictions.jsonl'), wrapper, eval_result)
save_predictions(os.path.join(
pattern_iter_output_dir, 'dev_predictions.jsonl'), wrapper, dev_result)
def predict_and_save(X, y, z, test, clf, features):
"""
----------------
fit and predict
----------------
"""
clf.fit(X,y)
model_name = str(clf.__class__).split(".")[-1].split("'")[0]
#! make predictions on test data and store the results as csv
preds = [x for x in clf.predict(test)]
header = [['id', 'votes']]
result = [[z[index], prediction] for index, prediction in enumerate(preds)]
save_predictions(header + result, model_name + "_Submission_%s_%s-%s.csv"% ("-".join(features),
str(date.today()), str(int(time.time()))))
def save_files(self, method):
if method == 'category_weighted_average':
val_preds = {m.fpath: m.predictions for m in self.models}
val = self.category_weighted_average(val_preds)
file_paths = [self.files_dir + f for f in self.files_names]
test_preds = {
fp: read_file(fp.replace('-val.out', '-test.out'))
for fp in file_paths
}
test = self.category_weighted_average(test_preds)
blend = pd.read_csv(blend_file, index_col='id')
test = 0.6 * test + 0.4 * blend
else:
raise NotImplementedError('Not implemented for \'%s\'' % method)
utils.save_predictions(os.path.dirname(self.trues_file) + '/',
train_preds=None,
val_preds=val,
test_preds=test,
model='ensemble')
def on_epoch_end(self, epoch, logs={}):
global best_kappa, best_kappa_epoch, val_kappas
if modelClass.last_activation == "softmax":
y_pred = self.model.predict(X_val)
y_pred = np.argmax(y_pred, axis=1)
else:
y_pred = self.model.predict(X_val) > 0.5
y_pred = y_pred.astype(int).sum(axis=1) - 1
_val_kappa = cohen_kappa_score(y_val, y_pred, weights='quadratic')
val_kappas.append(_val_kappa)
print(f"val_kappa: {_val_kappa:.4f}")
if _val_kappa > best_kappa:
best_kappa = _val_kappa
best_kappa_epoch = epoch
print(
"Validation Kappa has improved. Predicting and saving model.")
if modelClass.last_activation == "softmax":
y_test = self.model.predict(x_test, verbose=(verbose - 2) * -1)
y_test = np.argmax(y_test, axis=1)
else:
y_test = self.model.predict(x_test,
verbose=(verbose - 2) * -1) > 0.5
y_test = y_test.astype(int).sum(axis=1) - 1
save_predictions(y_test,
file_list,
save_name='predictions/{}.csv'.format(model_name))
self.model.save(os.path.join(model_path, model_name) + '_best.h5')
pd.DataFrame(val_kappas).to_csv(
os.path.join(model_path, 'history', model_name) + '-kappa.csv',
header=False,
index=False)
return
def update_accuracies(images_dir, curr_max_class, num_classes, classifier,
accuracies, save_dir, batch_size, shuffle, dataset):
seen_classes_test_loader = get_data_loader(images_dir,
False,
0,
curr_max_class,
batch_size=batch_size,
shuffle=shuffle,
dataset=dataset)
seen_probas, seen_top1, seen_top5 = compute_accuracies(
seen_classes_test_loader, classifier, num_classes)
print('\nSeen Classes (%d-%d): top1=%0.2f%% -- top5=%0.2f%%' %
(0, curr_max_class - 1, seen_top1, seen_top5))
accuracies['seen_classes_top1'].append(float(seen_top1))
accuracies['seen_classes_top5'].append(float(seen_top5))
# save accuracies and predictions out
utils.save_accuracies(accuracies,
min_class_trained=0,
max_class_trained=curr_max_class,
save_path=save_dir)
utils.save_predictions(seen_probas, 0, curr_max_class - 1, save_dir)
def main(
experiment_name,
dropout_p,
batch_size,
num_workers,
augmentation,
folds,
):
transforms = make_augmentation_transforms(augmentation, mode='test')
test_dataset = dataset.TestDataset(transform=transforms)
model = make_nasnet_model(
num_classes=config.NUM_CLASSES,
dropout_p=dropout_p,
)
test_data_loader = DataLoader(
dataset=test_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=False,
pin_memory=True,
)
test_predictions = np.zeros((folds, len(test_dataset), config.NUM_CLASSES))
for fold_num in range(folds):
checkpoint = load_checkpoint(
f'{experiment_name}_{fold_num}_{folds}_best.pth')
model.load_state_dict(checkpoint['state_dict'])
model = model.cuda().eval()
with torch.no_grad():
for batch_index, (waves, _) in enumerate(tqdm(test_data_loader)):
waves = Variable(waves).cuda()
logits = model(waves)
probs = F.softmax(logits, dim=1)
numpy_probs = probs.cpu().data.numpy()
start_index = batch_index * batch_size
end_index = start_index + numpy_probs.shape[0]
test_predictions[fold_num, start_index:end_index] = numpy_probs
save_predictions(test_predictions, f'{experiment_name}.pkl')
def store_prediction(self, sess, batch_x, batch_y, masks, name):
prediction = sess.run(self.net.predictor,
feed_dict={
self.net.x: batch_x,
self.net.y: batch_y,
self.net.keep_prob: 1.
})
loss = sess.run(self.net.cost,
feed_dict={
self.net.x: batch_x,
self.net.y: batch_y,
self.net.keep_prob: 1.
})
logging.info("Validaiton loss = {:.4f}".format(loss))
prediction_folder = os.path.join(self.prediction_path, name)
if os.path.exists(prediction_folder):
shutil.rmtree(prediction_folder, ignore_errors=True)
os.mkdir(prediction_folder)
utils.save_predictions(batch_x, batch_y, prediction, masks,
prediction_folder)
def evaluate_query_set(self, query_set_dict, has_masks, distance_eq):
gt = query_set_dict["gt"]
predictions = []
for file, im in query_set_dict["images"].items():
mask = None
if has_masks:
mask_filename = os.path.join(
self.output_folder,
file.split("/")[-1].split(".")[0] + ".png",
)
im, mask = mask_background(im)
gt_mask = query_set_dict["masks"][file.replace("jpg", "png")]
self.calc_mask_metrics(gt_mask[..., 0] / 255, mask / 255)
if self.opt.save:
save_mask(mask_filename, mask)
fv = self.calc_FV_query(im, mask)
distances = calculate_distances(self.feature_vector_protoypes, fv,
distance_eq)
predictions.append(list(distances.argsort()[:10]))
if self.opt.save:
save_predictions(
os.path.join(
self.output_folder,
"result_{}.pkl".format(int(has_masks) + 1),
),
predictions,
)
map_k = mapk(gt, predictions)
return map_k
def doc_classification(
task_config,
model_name_or_path,
cache_dir,
data_dir,
save_dir,
model_dir,
run_name="0",
lr=1e-05,
warmup_steps=5000,
balance_classes=True,
embeds_dropout=0.1,
epochs=200, # large because we use early stopping by default
batch_size=20,
grad_acc_steps=1,
early_stopping_metric="roc_auc",
early_stopping_mode="max",
early_stopping_patience=10,
model_class="Bert",
tokenizer_class="BertTokenizer",
do_lower_case=False,
do_train=True,
do_eval=True,
do_hpo=False,
print_preds=False,
print_dev_preds=False,
max_seq_len=512,
seed=11,
eval_every=500,
use_amp=False,
use_cuda=True,
):
# Load task config
task_config = yaml.safe_load(open(task_config))
data_dir = data_dir
save_dir = save_dir
model_dir = model_dir
# Create label list from args list or (for large label lists) create from file by splitting by space
if isinstance(task_config["data"]["label_list"], list):
label_list = task_config["data"]["label_list"]
else:
with open(data_dir / 'labels' /
task_config["data"]["label_list"]) as code_file:
label_list = code_file.read().split(" ")
# Register Outcome Metrics
register_task_metrics(label_list)
# General Settings
set_all_seeds(seed=seed)
device, n_gpu = initialize_device_settings(use_cuda=use_cuda,
use_amp=use_amp)
# 1.Create a tokenizer
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=model_name_or_path,
tokenizer_class=tokenizer_class,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=data_dir,
label_list=label_list,
metric=task_config["metric"],
multilabel=task_config["multilabel"],
train_filename=task_config["data"]["train_filename"],
dev_filename=task_config["data"]["dev_filename"],
dev_split=task_config["data"]["dev_split"]
if "dev_split" in task_config["data"] else None,
test_filename=task_config["data"]["test_filename"],
delimiter=task_config["data"]["parsing"]["delimiter"],
quote_char=task_config["data"]["parsing"]["quote_char"],
label_column_name=task_config["data"]["parsing"]["label_column"])
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a
# few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor,
caching=True,
cache_path=Path(cache_dir),
batch_size=batch_size)
if do_train:
# Setup MLFlow logger
ml_logger = MLFlowLogger(tracking_uri=task_config["log_dir"])
ml_logger.init_experiment(
experiment_name=task_config["experiment_name"],
run_name=f'{task_config["experiment_name"]}_{run_name}')
# 4. Create an AdaptiveModel
# a) which consists of a pretrained language model as a basis
language_model = LanguageModel.load(model_name_or_path,
language_model_class=model_class)
# b) and a prediction head on top that is suited for our task
# Define class weights
if balance_classes:
class_weights = data_silo.calculate_class_weights(
task_name=task_config["task_type"])
else:
class_weights = None
# Create Multi- or Single-Label Classification Heads
if task_config["multilabel"]:
prediction_head = MultiLabelTextClassificationHead(
class_weights=class_weights, num_labels=len(label_list))
else:
prediction_head = ExtendedTextClassificationHead(
class_weights=class_weights, num_labels=len(label_list))
model = ExtendedAdaptiveModel(
language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=embeds_dropout,
lm_output_types=[task_config["output_type"]],
device=device)
# 5. Create an optimizer
schedule_opts = {
"name": "LinearWarmup",
"num_warmup_steps": warmup_steps
}
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=lr,
device=device,
n_batches=len(data_silo.loaders["train"]),
n_epochs=epochs,
use_amp=use_amp,
grad_acc_steps=grad_acc_steps,
schedule_opts=schedule_opts)
# 6. Create an early stopping instance
early_stopping = None
if early_stopping_mode != "none":
early_stopping = EarlyStopping(mode=early_stopping_mode,
min_delta=0.0001,
save_dir=model_dir,
metric=early_stopping_metric,
patience=early_stopping_patience)
# 7. Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it
# from time to time
trainer = ExtendedTrainer(model=model,
optimizer=optimizer,
data_silo=data_silo,
epochs=epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=eval_every,
early_stopping=early_stopping,
device=device,
grad_acc_steps=grad_acc_steps,
evaluator_test=do_eval)
def score_callback(eval_score, train_loss):
tune.report(roc_auc_dev=eval_score, train_loss=train_loss)
# 8. Train the model
trainer.train(score_callback=score_callback if do_hpo else None)
# 9. Save model if not saved in early stopping
model.save(model_dir + "/final_model")
processor.save(model_dir + "/final_model")
if do_eval:
# Load newly trained model or existing model
if do_train:
model_dir = model_dir
else:
model_dir = Path(model_name_or_path)
logger.info("###### Eval on TEST SET #####")
evaluator_test = ExtendedEvaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# Load trained model for evaluation
model = ExtendedAdaptiveModel.load(model_dir, device)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
# Evaluate
results = evaluator_test.eval(model, return_preds_and_labels=True)
# Log results
utils.log_results(results,
dataset_name="test",
steps=len(evaluator_test.data_loader),
save_path=model_dir + "/eval_results.txt")
if print_preds:
# Print model test predictions
utils.save_predictions(results,
save_dir=model_dir,
multilabel=task_config["multilabel"])
if print_dev_preds:
# Evaluate on dev set, e.g. for threshold tuning
evaluator_dev = Evaluator(
data_loader=data_silo.get_data_loader("dev"),
tasks=data_silo.processor.tasks,
device=device)
dev_results = evaluator_dev.eval(model,
return_preds_and_labels=True)
utils.log_results(dev_results,
dataset_name="dev",
steps=len(evaluator_dev.data_loader),
save_path=model_dir + "/eval_dev_results.txt")
# Print model dev predictions
utils.save_predictions(dev_results,
save_dir=model_dir,
multilabel=task_config["multilabel"],
dataset_name="dev")
sampled = sample_training_data(sampling_percentage,
possible_train_instances,
possible_train_labels)
train_instances = sampled[0]
train_gold_labels = sampled[1]
predictions[dataset_filename][fold][str(
sampling_percentage)] = {}
subpredictions = predictions[dataset_filename][fold][str(
sampling_percentage)]
for strategy_name, strategy in config["generation_strategies"]:
subpredictions[strategy_name] = {}
for clf_name, base_clf in config["base_classifiers"]:
clf_pool = strategy(base_clf(), config["pool_size"])
clf_pool.fit(train_instances, train_gold_labels)
hard_voting_clf = get_voting_clf(clf_pool)
cur_predictions = hard_voting_clf.predict(
test_instances)
subpredictions[strategy_name][
clf_name] = cur_predictions.astype(int).tolist()
print "Experiment " + str(exp)
exp += 1
print "Finished experiment"
save_predictions(predictions)
print "Stored predictions"
from models import K_SVM, KernelLogReg, KernelRidgeReg
from utils import get_saved_data, save_predictions, reformat_data
if __name__ == '__main__':
# Kernels considered
methods = ['WD_d10', 'SP_k6']
# Import data
data, kernels, ID = get_saved_data(methods)
print('\n\n')
print('Getting the data.........')
X_train, y_train, X_val, y_val, X_test, kernels, ID = reformat_data(
data, kernels, ID)
print('Fitting the model.....')
model = KernelLogReg(kernels[0], ID, lambda_=0.1, solver='BFGS')
model.fit(X_train, y_train)
print('Making predictions .......')
print('\n\nAccuracy on validation set 1: {:0.4f}'.format(
model.score(model.predict(X_val), y_val)))
# Compute predictions
y_pred = save_predictions(model, X_test)
print('\n\nPredictions ok')
print('------')
print('Check prediction file Yte.csv in folder')
def classify_and_save():
pass_id, test_data = prepare_test_data()
pred = classify(test_data)
format_pred = (pred > 0.5).astype(int)
save_predictions(pass_id, format_pred.reshape(-1))
filename = filename or 'hexbug-training_video-centroid_data'
input_data = convert_data_to_matrix(filename)
boundaries = get_wall_boundaries(input_data)
# Debugging vs Final code
if args.debug:
#3000 will lead to a a corner collision
#5000 will lead to an almost perpendicular collision with a flat wall, then steering to the robot's left
#8000 will lead to an almost perfect 45 degree ricochet off the bottom wall
#11000 will lead to a bunching up in the bottom left corner
#start_index = 11000 #this will be the frame we start at for our test
start_index = 2000 # 10000 #this will be the frame we start at for our test # CLW
len_data = 3600
test_data = input_data[start_index:start_index + len_data]
actual = input_data[start_index + len_data:start_index + len_data +
60] #take the next 60 frames
past_predictions, OTHER = robot_EKF(test_data)
predicted = predict_next_frames(OTHER, test_data[-1], boundaries, 60)
print predicted
else:
test_data = input_data[:]
past_predictions, OTHER = robot_EKF(test_data)
predicted = predict_next_frames(OTHER, test_data[-1], boundaries, 60)
save_predictions(predicted)
def main():
# parse command line argument and generate config dictionary
config = parse_args()
logger.info(json.dumps(config, indent=2))
run_config = config['run_config']
optim_config = config['optim_config']
# TensorBoard SummaryWriter
if run_config['tensorboard']:
writer = SummaryWriter(run_config['outdir'])
else:
writer = None
# set random seed
seed = run_config['seed']
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
epoch_seeds = np.random.randint(np.iinfo(np.int32).max // 2,
size=optim_config['epochs'])
# create output directory
outdir = pathlib.Path(run_config['outdir'])
outdir.mkdir(exist_ok=True, parents=True)
# save config as json file in output directory
outpath = outdir / 'config.json'
with open(outpath, 'w') as fout:
json.dump(config, fout, indent=2)
# load data loaders
train_loader, test_loader = get_loader(config['data_config'])
# load model
logger.info('Loading model...')
model = utils.load_model(config['model_config'])
n_params = sum([param.view(-1).size()[0] for param in model.parameters()])
logger.info('n_params: {}'.format(n_params))
if run_config['fp16'] and not run_config['use_amp']:
model.half()
for layer in model.modules():
if isinstance(layer, nn.BatchNorm2d):
layer.float()
device = torch.device(run_config['device'])
if device.type == 'cuda' and torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model.to(device)
logger.info('Done')
train_criterion, test_criterion = utils.get_criterion(
config['data_config'])
# create optimizer
if optim_config['no_weight_decay_on_bn']:
params = [
{
'params': [
param for name, param in model.named_parameters()
if 'bn' not in name
]
},
{
'params': [
param for name, param in model.named_parameters()
if 'bn' in name
],
'weight_decay':
0
},
]
else:
params = model.parameters()
optim_config['steps_per_epoch'] = len(train_loader)
optimizer, scheduler = utils.create_optimizer(params, optim_config)
# for mixed-precision
amp_handle = apex.amp.init(
enabled=run_config['use_amp']) if is_apex_available else None
# run test before start training
if run_config['test_first']:
_, predictions = test(0, model, test_criterion, test_loader,
run_config, writer)
utils.save_predictions(predictions, 0, outdir)
state = {
'config': config,
'state_dict': None,
'optimizer': None,
'epoch': 0,
'accuracy': 0,
'best_accuracy': 0,
'best_epoch': 0,
}
epoch_logs = []
for epoch, seed in zip(range(1, optim_config['epochs'] + 1), epoch_seeds):
np.random.seed(seed)
# train
train_log = train(epoch, model, optimizer, scheduler, train_criterion,
train_loader, config, writer, amp_handle)
# test
test_log, predictions = test(epoch, model, test_criterion, test_loader,
run_config, writer)
epoch_log = train_log.copy()
epoch_log.update(test_log)
epoch_logs.append(epoch_log)
utils.save_epoch_logs(epoch_logs, outdir)
# update state dictionary
state = update_state(state, epoch, epoch_log['test']['accuracy'],
model, optimizer)
# save model
utils.save_checkpoint(state, outdir)
# save predictions
utils.save_predictions(predictions, epoch, outdir)
print("Train shape", x_train.shape)
print("Validation shape", x_val.shape)
train(x_train,
y_train,
x_val,
y_val,
n_aug=N_AUG,
save_path=MODEL_PATH)
elif MODE == "predict":
x_test, y_test, slides_test, _, _, _ = load_data(TEST_FEATURES,
flt=FLT,
fold=None)
print("Predict shape", x_test.shape)
print("Loading model from", MODEL_PATH)
booster = lgb.Booster(model_file=str(MODEL_PATH))
pred = booster.predict(x_test)
y_test_reshaped, pred_reshaped, slides_test_reshaped = reshape(
y_test, pred, N_AUG, slides=slides_test)
if y_test_reshaped is not None:
evaluate(y_test_reshaped, pred_reshaped.mean(axis=1))
# save
if PRED_PATH:
dd = dict(zip(slides_test_reshaped, pred_reshaped))
save_predictions(dd, PRED_PATH, std=True, do_rescale=False)
meta_train = None
meta_test = None
for name, meta in classifiers.items():
if name == 'mnb':
continue
p = utils.cross_val_proba(meta, train_preds, ym, 5, hash(name + str(meta)))
print 'Meta', name, f1_score(ym, np.array(p >= 0.5, dtype='l'), average='samples')
meta_train = p if meta_train is None else np.hstack((meta_train, p))
# Test set
if use_test:
meta.fit(train_preds, ym)
pt = meta.predict_proba(test_preds)
pt = np.transpose(np.array(pt)[:,:,1]) if type(pt) == type([]) else pt
meta_test = pt if meta_test is None else np.hstack((meta_test, pt))
#meta = classifiers['svc']
#print 'Meta', f1_score(ym, cross_val_predict(meta, meta_train, ym, 5, n_jobs=5), average='samples')
#if use_test:
# meta.fit(meta_train, ym)
# pt = meta.predict(meta_test)
# utils.save_predictions(idt, pt)
meta = classifiers['svc6']
print 'Meta', f1_score(ym, cross_val_predict(meta, train_preds[:, 9:], ym, 5, n_jobs=5), average='samples')
if use_test:
meta.fit(train_preds[:, 9:], ym)
pt = meta.predict(test_preds[:, 9:])
utils.save_predictions(idt, pt)
def main(prefix="",
url_feature="",
url_pred="",
url_len="",
url_weight="",
batch_size=126,
max_input_len=30,
max_sent_length=24,
embed_size=13,
acc_range=10,
sight=1,
is_classify=0,
decoder=1,
decoder_size=4,
loss='mae',
context_meaning=1,
rnn_layer=1):
# init model
model = Model(max_input_len=max_input_len,
max_sent_len=max_sent_length,
embed_size=embed_size,
using_bidirection=False,
fw_cell="basic",
bw_cell="basic",
batch_size=batch_size,
is_classify=is_classify,
use_tanh_prediction=True,
target=5 if is_classify else 1,
loss=loss,
acc_range=acc_range,
input_rnn=False,
sight=sight,
use_decoder=decoder,
dvs=decoder_size,
rnn_layer=rnn_layer)
# model.init_data_node()
tf.reset_default_graph()
with tf.device('/%s' % p.device):
model.init_ops()
saver = tf.train.Saver()
utils.assert_url(url_feature)
if url_pred:
utils.assert_url(url_pred)
dataset = utils.load_file(url_feature)
pred = utils.load_file(url_pred, False)
if is_classify:
pred = [
utils.get_pm25_class(round(float(x.replace("\n", ""))))
for x in pred
]
else:
pred = [round(float(x.replace("\n", ""))) for x in pred]
if max_input_len > 1:
utils.assert_url(url_len)
data_len = utils.load_file(url_len)
else:
data_len = None
_, test = utils.process_data(dataset,
data_len,
pred,
batch_size,
max_input_len,
max_sent_length,
True,
sight,
context_meaning=context_meaning)
else:
test = utils.load_file(url_feature)
tconfig = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=tconfig) as session:
init = tf.global_variables_initializer()
session.run(init)
# saver = tf.train.import_meta_graph(url_weight + ".meta")
saver.restore(session, url_weight)
print('==> running model')
_, _, preds, lb = model.run_epoch(session, test, shuffle=False)
preds = [x if x <= 45 else (x + 10) for x in preds]
lb = lb[0:len(preds)]
# print('Validation loss: {}'.format(valid_loss))
# print('Validation accuracy: {}'.format(valid_accuracy))
# tmp = 'Test validation accuracy: %.4f' % valid_accuracy
# utils.save_file("test_acc/%s_test_acc.txt" % prefix, tmp, False)
evaluate(preds, lb, acc_range, is_classify)
utils.save_predictions(preds, lb, p.test_preds % prefix)
def main(
model_name,
dropout_p,
experiment_name,
batch_size,
save_npz_every_n_batches,
flip_lr,
flip_ud,
crop,
crop_size,
crop_location,
resize_after_crop,
num_workers,
):
base_predictions_dir = os.path.join(
config.PREDICTIONS_PATH,
experiment_name,
)
os.makedirs(base_predictions_dir, exist_ok=True)
predictions_dir_name = 'npz'
if flip_lr:
predictions_dir_name += '_fliplr'
if flip_ud:
predictions_dir_name += '_flipud'
if crop:
crop_size_width, crop_size_height = crop_size
predictions_dir_name += (
f'_crop_{crop_size_width}x{crop_size_height}_{crop_location}')
if resize_after_crop:
predictions_dir_name += '_resize_after_crop'
print(predictions_dir_name)
predictions_dir = os.path.join(base_predictions_dir, predictions_dir_name)
os.makedirs(predictions_dir, exist_ok=True)
model = get_model(
model_name,
num_classes=config.NUM_CLASSES,
dropout_p=dropout_p,
)
checkpoint = load_checkpoint(f'{experiment_name}_best.pth')
model.load_state_dict(checkpoint['state_dict'])
model = model.cuda().eval()
if not crop:
crop_params = None
else:
crop_params = crop_size, crop_location, resize_after_crop
transform = make_test_augmentation_transforms(
crop_params,
flip_lr,
flip_ud,
)
test_dataset = dataset.CdiscountTestDataset(
bson_filepath=config.TEST_BSON_FILE,
transform=transform,
)
test_data_loader = DataLoader(
dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
)
saved_batch_index = 0
predictions = []
for batch_index, (image_groups, product_ids, img_nums) in tqdm(
enumerate(test_data_loader),
total=len(test_data_loader),
):
images = Variable(image_groups, volatile=True).cuda()
logits = model(images)
probs = F.softmax(logits)
numpy_probs = probs.cpu().data.numpy()
predictions.append(numpy_probs)
if batch_index != 0 and batch_index % save_npz_every_n_batches == 0:
save_predictions(
predictions,
os.path.join(predictions_dir, f'{batch_index}.npz'),
)
saved_batch_index += 1
predictions = []
save_predictions(
predictions,
os.path.join(predictions_dir, f'{batch_index}.npz'),
)
# Load and predict data
normalize = transforms.Normalize(mean=[0.000],
std=[1.000])
test_dataset = HW_Dataset(labels_dir=LABELS_DIR,
imgs_dir=IMGS_DIR,
data_list=TEST_LIST,
transform=transforms.Compose([
transforms.ToTensor(),
normalize
]))
test_loader = DataLoader(dataset=test_dataset, batch_size=TEST_BATCH_SIZE,
shuffle=False, num_workers=0, pin_memory=True)
# switch to evaluate mode (deactivate dropout)
model.eval()
test_loss = 0
num_incorrect = 0
total_str_dist = 0
preds = []
targets = []
with torch.no_grad():
for batch_idx, (x, y_target) in enumerate(test_loader):
if batch_idx % LOG_INTERVAL == 0:
print("performed inference on batch: ", batch_idx)
x = x.to(device)
y = model(x)
y_target = utils.encode_words(y_target, CLASS_NAMES)
preds.extend(utils.decode_output(y.cpu(), CLASS_NAMES))
targets.extend(utils.decode_label_words(y_target, CLASS_NAMES))
# save submission
utils.save_predictions(SUBMISSION_PATH, TEST_LIST, preds)
verbose=1,
min_delta=1e-4)
callbacks_list = [logger, reduce_lr, checkpoint]
STEP_SIZE_TRAIN = x_train.shape[0] // BATCH_SIZE
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_data=(x_val, y_val),
callbacks=callbacks_list,
class_weight=None,
verbose=verbose)
# model.save(os.path.join(model_path, model_name) + '.h5')
model = load_model(os.path.join(model_path, model_name) + '_best.h5')
#####
y_test = model.predict(x_test, verbose=(verbose - 2) * -1)
y_test = np.round(np.clip(y_test, 0, 4)).astype(int).ravel()
file_list = pd.read_csv(os.path.join(data_folder, 'Test/test_files.csv'),
header=None,
squeeze=True).values
os.makedirs('predictions', exist_ok=True)
save_predictions(y_test,
file_list,
save_name='predictions/{}.csv'.format(model_name))
def eval_set(loader, gt_correspondences, bbdd_fvs, opt):
masks_metrics = {"precision": [], "recall": [], "f1": []}
ious = []
predictions = []
set_bboxes = []
for name, query_image, gt_mask in loader:
if opt.apply_denoise:
query_image, Noise_level_before, Noise_level_after, blur_type_last = detect_denoise(
query_image, opt.blur_type)
# transform to another color space
multiple_painting, split_point, bg_mask = detect_paintings(query_image)
bboxes, bbox_mask = detect_bboxes(query_image)
res_mask = bg_mask.astype(bool) ^ bbox_mask.astype(
bool) if loader.detect_bboxes else bg_mask
if loader.compute_masks:
if loader.evaluate:
calc_mask_metrics(masks_metrics, gt_mask / 255, bg_mask)
if opt.save:
mask_name = name.split("/")[-1].replace(".jpg", ".png")
save_mask(
os.path.join(opt.output,
loader.root.split("/")[-1], mask_name),
res_mask * 255)
# cropped sets, no need to mask image for retrieval
if gt_mask is None:
res_mask = None
if loader.detect_bboxes:
set_bboxes.append(bboxes)
# change colorspace before computing feature vector
query_image = transform_color(
query_image, opt.color) if opt.color is not None else query_image
if multiple_painting and gt_mask is not None:
im_preds = []
left_paint = np.zeros_like(res_mask)
right_paint = np.zeros_like(res_mask)
left_paint[:, split_point:] = res_mask[:, split_point:]
right_paint[:, :split_point] = res_mask[:, :split_point]
res_masks = [left_paint, right_paint]
for submasks in res_masks:
query_fv = calc_FV(query_image, opt, submasks).ravel()
distances = calculate_distances(bbdd_fvs,
query_fv,
mode=opt.dist)
im_preds.append((distances.argsort()[:10]).tolist())
predictions.append(im_preds)
else:
query_fv = calc_FV(query_image, opt, res_mask).ravel()
distances = calculate_distances(bbdd_fvs, query_fv, mode=opt.dist)
predictions.append((distances.argsort()[:10]).tolist())
if opt.save:
save_predictions(
"{}/{}/result.pkl".format(opt.output,
loader.root.split("/")[-1]), predictions)
save_predictions(
"{}/{}/text_boxes.pkl".format(opt.output,
loader.root.split("/")[-1]),
set_bboxes)
map_k = {
i: mapk(gt_correspondences, predictions, k=i)
for i in [10, 3, 1]
} if loader.evaluate else None
avg_mask_metrics = averge_masks_metrics(
masks_metrics) if loader.evaluate else None
return map_k, avg_mask_metrics
