def test_softmax_multi_axes():
assert_allclose(softmax([[1000, 0], [1000, 0]], axis=0),
np.array([[.5, .5], [.5, .5]]), rtol=1e-13)
assert_allclose(softmax([[1000, 0], [1000, 0]], axis=1),
np.array([[1, 0], [1, 0]]), rtol=1e-13)
# Expected value computed using mpmath (with mpmath.mp.dps = 200) and then
# converted to float.
x = np.array([[-25, 0, 25, 50],
[1, 325, 749, 750]])
expected = np.array([[2.678636961770877e-33,
1.9287498479371314e-22,
1.3887943864771144e-11,
0.999999999986112],
[0.0,
1.9444526359919372e-185,
0.2689414213699951,
0.7310585786300048]])
assert_allclose(softmax(x, axis=1), expected, rtol=1e-13)
assert_allclose(softmax(x.T, axis=0), expected.T, rtol=1e-13)
# 3-d input, with a tuple for the axis.
x3d = x.reshape(2, 2, 2)
assert_allclose(softmax(x3d, axis=(1, 2)), expected.reshape(2, 2, 2),
rtol=1e-13)
def test_softmax_fixtures():
assert_allclose(softmax([1000, 0, 0, 0]), np.array([1, 0, 0, 0]),
rtol=1e-13)
assert_allclose(softmax([1, 1]), np.array([.5, .5]), rtol=1e-13)
assert_allclose(softmax([0, 1]), np.array([1, np.e])/(1 + np.e),
rtol=1e-13)
# Expected value computed using mpmath (with mpmath.mp.dps = 200) and then
# converted to float.
x = np.arange(4)
expected = np.array([0.03205860328008499,
0.08714431874203256,
0.23688281808991013,
0.6439142598879722])
assert_allclose(softmax(x), expected, rtol=1e-13)
# Translation property. If all the values are changed by the same amount,
# the softmax result does not change.
assert_allclose(softmax(x + 100), expected, rtol=1e-13)
# When axis=None, softmax operates on the entire array, and preserves
# the shape.
assert_allclose(softmax(x.reshape(2, 2)), expected.reshape(2, 2),
rtol=1e-13)
def l1(weights, rng):
weights -= np.min(weights)
return weights / np.sum(weights)
def dirichlet(weights, rng):
# weights -= np.min(weights)
return rng.dirichlet(weights)
pvals_from_weights = {
'l1': l1,
'dirichlet': dirichlet,
'softmax': lambda weights, rng: softmax(weights),
}
def pvals_from_weights_2(weights, rng):
return rng.dirichlet(weights)
def pvals_from_weights_3(weights):
return softmax(weights)
AllEqualPartition = namedtuple('AllEqualPartition',
['parts', 'apply_to_array'])
def f(x):
return random.choices(df_slice.columns, weights=softmax(x))[0]
def main():
# Get a line counter
lcounter = 0
with codecs.open(opt.data, 'r', "utf-8") as sfile:
for ix, l in enumerate(sfile):
lcounter += 1
resfile = codecs.open(opt.data, 'r')
if opt.output:
outfile = codecs.open(opt.output, 'w')
outfile2 = codecs.open(opt.output + ".track", "w")
if opt.tgt:
y = read_tgt_file(opt.tgt)
yhat = []
yprobs = []
for ix, line in tqdm(enumerate(resfile), total=lcounter):
cline = json.loads(line)
words = cline['words']
# print("len words", len(words))
key = 'class_probabilities'
if key in cline:
probs = [p[1] for p in cline['class_probabilities'][:len(words)]]
else:
# get prob from logits
logits = [p for p in cline['logits'][:len(words)]]
probs = []
for log in logits:
probability = softmax(log)
probs.append(probability[1])
tags = [1 if p > opt.threshold else 0 for p in probs]
if opt.output:
if opt.style == "phrases":
pred = get_phrases(words, tags)
elif opt.style == "sentences":
pred = get_sents(words, tags, probs)
elif opt.style == "threesent":
pred = get_three(words, outfile2, probs)
outfile.write(pred + "\n")
if opt.tgt:
yhat.append(tags)
yprobs.append(probs)
# if ix > 150:
# break
if opt.tgt:
print("Evaluating Model...")
y_flat = []
yhat_flat = []
probs_flat = []
for tgt, pred, pr in zip(y, yhat, yprobs):
if len(pred) != len(tgt):
pass
# print("woa", len(pred), len(tgt))
else:
for t, p, cpr in zip(tgt, pred, pr):
y_flat.append(t)
yhat_flat.append(p)
probs_flat.append(cpr)
y_flat = np.array(y_flat)
yhat_flat = np.array(yhat_flat)
probs_flat = np.array(probs_flat)
fpr, tpr, thresholds = roc_curve(y_flat, probs_flat)
print("AUC: {:.1f}".format(auc(fpr, tpr) * 100))
print("F1-Score (binary): {:.2f}".format(
f1_score(y_flat, yhat_flat) * 100))
print("Confusion Matrix:")
print(confusion_matrix(y_flat, yhat_flat))
tgt_names = ["O", "I"]
print("Classification Report:")
print(classification_report(y_flat, yhat_flat, target_names=tgt_names))
resfile.close()
if opt.output:
outfile.close()
outfile2.close()
def main(_):
bert_config = modeling.BertConfig.from_json_file(config_dict[FLAGS.model_size])
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
tpu_cluster_resolver = None
if use_tpu:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
keep_checkpoint_max=1,
model_dir=FLAGS.output_path,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=iterations_per_loop,
num_shards=num_tpu_cores,
per_host_input_for_training=is_per_host))
model_fn = model_fn_builder(
bert_config=bert_config,
num_labels=2,
init_checkpoint=init_checkpoint,
use_tpu=use_tpu,
use_one_hot_embeddings=use_tpu)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.batch_size,
eval_batch_size=FLAGS.batch_size,
predict_batch_size=FLAGS.batch_size,
params={"qc_scores": "qc_scores"})
tf.logging.info("***** Running evaluation *****")
tf.logging.info(" Batch size = %d", FLAGS.batch_size)
for split in ["valid", "test"]:
maxp_run = load_run(os.path.join(FLAGS.first_model_path, "{}_{}_result.trec".format(FLAGS.dataset, split)))
query_docids_map = []
data_path = os.path.join(FLAGS.output_path, "rerank-{0}_kc-{1}".format(FLAGS.rerank_num, FLAGS.kc), "data")
result_path = os.path.join(FLAGS.output_path, "rerank-{0}_kc-{1}".format(FLAGS.rerank_num, FLAGS.kc), "result")
if not tf.gfile.Exists(result_path):
tf.gfile.MakeDirs(result_path)
with tf.gfile.Open(os.path.join(data_path, "chunk_passage_ids_{0}.txt".format(split))) as ref_file:
for line in ref_file:
query_docids_map.append(line.strip().split("\t"))
predict_input_fn = input_fn_builder(
dataset_path=os.path.join(data_path, "chunk_passage_{0}.tf".format(split)),
is_training=False,
seq_length=FLAGS.max_seq_length,
drop_remainder=False)
total_count = 0
result_file = tf.gfile.Open(os.path.join(result_path, "{0}_{1}_result.trec".format(FLAGS.dataset, split)), 'w')
ckpt = tf.train.latest_checkpoint(checkpoint_dir=FLAGS.third_model_path)
print("use latest ckpt: {0}".format(ckpt))
result = estimator.predict(input_fn=predict_input_fn,
yield_single_examples=True,
checkpoint_path=ckpt)
start_time = time.time()
results = []
result_dict = collections.OrderedDict()
for item in result:
results.append((item["qc_scores"], item["probs"]))
total_count += 1
if total_count == len(query_docids_map) or query_docids_map[total_count][0] != \
query_docids_map[total_count - 1][0]:
chunk_num = len(results) // FLAGS.rerank_num
assert chunk_num <= FLAGS.kc
qc_scores, probs = list(zip(*results))
qc_scores = np.stack(qc_scores)
cp_scores = np.stack(probs)[:, 1]
qc_scores = np.reshape(qc_scores, [FLAGS.rerank_num, chunk_num])
cp_scores = np.reshape(cp_scores, [FLAGS.rerank_num, chunk_num])
# softmax normalization
qc_scores = softmax(qc_scores, axis=-1)
scores = np.sum(np.multiply(qc_scores, cp_scores), axis=-1, keepdims=False)
start_idx = total_count - FLAGS.rerank_num * chunk_num
end_idx = total_count
query_ids, chunk_ids, passage_ids, labels, qc_scores = zip(*query_docids_map[start_idx:end_idx])
assert len(set(query_ids)) == 1, "Query ids must be all the same."
query_id = query_ids[0]
candidate_docs = list()
for pid in passage_ids:
doc_id = pid.split("_")[0]
if doc_id not in candidate_docs:
candidate_docs.append(doc_id)
result_dict[query_id] = dict()
for i, doc in enumerate(candidate_docs):
result_dict[query_id][doc] = scores[i]
rerank_list = sorted(result_dict[query_id].items(), key=lambda x: x[1], reverse=True)
last_score = rerank_list[-1][1]
for doc in maxp_run[query_id][FLAGS.rerank_num:]:
current_score = last_score - 0.01
result_dict[query_id][doc] = current_score
last_score = current_score
ranking_list = sorted(result_dict[query_id].items(), key=lambda x: x[1], reverse=True)
for rank, (doc_id, score) in enumerate(ranking_list):
result_file.write(
"\t".join([query_id, "Q0", doc_id, str(rank + 1), str(score), "chunk_passage_PRF"]) + "\n")
results = []
if total_count % 1000 == 0:
tf.logging.warn("Read {} examples in {} secs".format(
total_count, int(time.time() - start_time)))
result_file.close()
tf.logging.info("Done Evaluating!")
def softmax(x):
return softmax(x,axis=1)
def gradF(self, state, q):
eta = self.eta
return softmax(q / eta)
def check_probs(self):
_, logits = self.model.predict([self._text])
probs = softmax(logits[0])
probs_3dp = ["{:.5f}".format(float(i)) for i in probs]
return probs_3dp, len(
self.tokenizer.encode(self._text, add_special_tokens=False))
def normalize_advice(self, advice):
return softmax(advice)
def softMax(x):
"""Compute softmax values for each sets of scores in x.
:param x: vector of features
"""
return softmax(x, axis=0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='data')
parser.add_argument('--teachers_dir', type=str, default='models/Teachers')
parser.add_argument('--max_seq_length', type=int, default=128)
parser.add_argument('--model_name_or_path', type=str, default=None)
parser.add_argument('--output_dir', type=str, default=None)
parser.add_argument('--logging_dir', type=str, default=None)
parser.add_argument('--num_train_epochs', type=int, default=3)
parser.add_argument('--per_device_train_batch_size', type=int, default=32)
parser.add_argument('--logging_steps', type=int, default=100)
parser.add_argument('--do_train', type=bool, default=True)
parser.add_argument('--do_eval', type=bool, default=True)
parser.add_argument('--do_predict', type=bool, default=True)
parser.add_argument('--evaluation_strategy', type=str, default='epoch')
parser.add_argument('--save_steps', type=int, default=None)
parser.add_argument('--seed', type=int, default=1)
args = parser.parse_args()
print(args)
outputs = []
dataclass_types = [
ModelArguments, DataTrainingArguments, TrainingArguments
]
for dtype in dataclass_types:
keys = {f.name for f in dataclasses.fields(dtype)}
inputs = {k: v for k, v in vars(args).items() if k in keys}
obj = dtype(**inputs)
outputs.append(obj)
model_args, data_args, training_args = outputs
logger = logging.getLogger(__name__)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name
if model_args.tokenizer_name else model_args.model_name_or_path,
cache_dir=model_args.cache_dir,
use_fast=model_args.use_fast,
)
predictions = {} # it will be filled with teachers' predictions
teachers_folders = os.listdir(args.teachers_dir)
for teacher in teachers_folders:
print("Obtaining predictions of teacher: ", teacher)
global_data_handler = NERDataHandler(tokenizer)
labels = global_data_handler.get_labels(data_args.labels)
global label_map
label_map = {i: label for i, label in enumerate(labels)}
num_labels = len(labels)
config = AutoConfig.from_pretrained(
os.path.join(args.teachers_dir, teacher),
num_labels=num_labels,
id2label=label_map,
label2id={label: i
for i, label in enumerate(labels)},
cache_dir=model_args.cache_dir,
)
model = AutoModelForTokenClassification.from_pretrained(
os.path.join(args.teachers_dir, teacher),
from_tf=bool(".ckpt" in os.path.join(args.teachers_dir, teacher)),
config=config,
cache_dir=model_args.cache_dir,
)
trainer = Trainer(model=model, )
# setting test dataset
global_data_handler.set_dataset(
data_dir=os.path.join(data_args.data_dir, 'GLOBAL', teacher),
labels=labels,
model_type=config.model_type,
max_seq_length=data_args.max_seq_length,
overwrite_cache=data_args.overwrite_cache,
mode="train_dev",
save_cache=False)
predictions[teacher], _, _ = trainer.predict(
global_data_handler.datasets['train_dev'])
predictions[teacher] = softmax(predictions[teacher], axis=2)
print("Aggregating distributions...")
teachers_predictions = np.concatenate(
[predictions[teacher] for teacher in teachers_folders], axis=-1)
teachers_predictions = aggregate_proba(teachers_predictions)
np.save(
os.path.join(args.data_dir, 'GLOBAL', 'Student',
'teachers_predictions.npy'), teachers_predictions)
def evaluate(args, device, model, tokenizer, dump_prediction_files=False, prefix=""):
logger = logging.getLogger(__name__)
eval_output_dir = args.reports_dir
results_txt = {} # this will just store the precision, recall, f1 of a single run of evaluate (for txt model)
results_img = {}
if not os.path.exists(eval_output_dir):
print('The output directory %s does not exist!'%eval_output_dir)
exit(0)
# os.makedirs(eval_output_dir)
eval_dataset, _ = model_utils.load_and_cache_examples(args, tokenizer)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size,
num_workers=args.num_cpu_workers, pin_memory=True)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix)) # prefix refers to epoch
logger.info(" Num examples = %d", len(eval_dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
nb_eval_steps = 0
preds_txt = None
preds_img = None
out_labels = None
report_ids = None
img_embeddings = None
txt_embeddings = None
model.eval()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
batch = tuple(t.to(device=device, non_blocking=True) for t in batch)
image, label_raw, txt_ids, txt_mask, txt_segment_ids, label_onehot_or_ordinal, report_id = batch
for j in range(len(label_raw)):
if args.output_channel_encoding == 'multiclass':
converted_label = torch.tensor(model_utils.convert_to_onehot(label_raw[j][0]), dtype=torch.long)
if args.output_channel_encoding == 'multilabel':
converted_label = torch.tensor(model_utils.convert_to_ordinal(label_raw[j][0]), dtype=torch.long)
label_onehot_or_ordinal[j] = converted_label
# we need the above conversion because label_raw is the one read from the csv file
label_onehot_or_ordinal.to(device=device)
#image, label_raw, input_ids, input_mask, segment_ids, label_onehot_or_ordinal = batch
with torch.no_grad():
inputs = { 'input_img': image,
'input_ids': txt_ids,
'attention_mask': txt_mask,
'token_type_ids': txt_segment_ids,
'labels': None,
'bert_pool_last_hidden': args.bert_pool_last_hidden,
'bert_pool_use_img': args.bert_pool_use_img,
'bert_pool_img_lowerlevel': args.bert_pool_img_lowerlevel}
#inputs = { 'input_img': image,
# 'input_ids': input_ids,
# 'attention_mask': input_mask,
# 'token_type_ids': segment_ids,
# 'labels': None,
# 'same_classifier': args.share_img_txt_classifier} # we purposefully make it none so that logits returned not loss
outputs = model(**inputs)
img_embedding, img_logits, txt_embedding, txt_logits = outputs[:4]
#img_label = txt_labels_ordinal #Replace the image label with the ordinally encoded label
#txt_labels = txt_labels_ordinal
out_label = label_onehot_or_ordinal
out_label_raw = label_raw
nb_eval_steps += 1
if (preds_txt is None and preds_img is not None) or (preds_img is None and preds_txt is not None):
raise Exception('No image or text prediction - this is not possible because they should be None at the same time')
img_embedding_batch = img_embedding.detach().cpu().numpy()
txt_embedding_batch = txt_embedding.detach().cpu().numpy()
pred_txt_batch = txt_logits.detach().cpu().numpy() # shape: 10,3
pred_img_batch = img_logits.detach().cpu().numpy()
out_label_batch = out_label.detach().cpu().numpy()
out_label_raw_batch = out_label_raw.detach().cpu().numpy()
report_id_batch = report_id.detach().cpu()
if preds_txt is None and preds_img is None:
preds_txt = pred_txt_batch
preds_img = pred_img_batch
out_labels = out_label_batch # gold label of image and text is the same at the moment
out_labels_raw = out_label_raw_batch
report_ids = report_id_batch
img_embeddings = img_embedding_batch
txt_embeddings = txt_embedding_batch
else:
preds_txt = np.append(preds_txt, pred_txt_batch, axis=0)
preds_img = np.append(preds_img, pred_img_batch, axis=0)
out_labels = np.append(out_labels, out_label_batch, axis=0)
out_labels_raw = np.append(out_labels_raw, out_label_raw_batch, axis=0)
report_ids = np.append(report_ids, report_id_batch, axis=0)
img_embeddings = np.append(img_embeddings, img_embedding_batch, axis=0)
txt_embeddings = np.append(txt_embeddings, txt_embedding_batch, axis=0)
if args.print_predictions:
output_preds_file = os.path.join(eval_output_dir, "eval_results_images_preds.txt")
with open(output_preds_file, "w") as writer:
for i in range(len(report_ids)):
writer.write('%s, '%report_ids[i])
writer.write('%s\n'%preds_img[i])
if args.print_embeddings:
out_labels_raw_path = os.path.join(eval_output_dir, "eval_results_labels")
np.save(out_labels_raw_path, out_labels_raw)
img_embeddings_path = os.path.join(eval_output_dir, "eval_results_image_embeddings")
np.save(img_embeddings_path, img_embeddings)
txt_embeddings_path = os.path.join(eval_output_dir, "eval_results_text_embeddings")
np.save(txt_embeddings_path, txt_embeddings)
# with open(output_preds_file, "w") as writer:
# for i in range(len(report_ids)):
# writer.write('%s\n'%img_embeddings[i])
# writer.write('%s\n'%txt_embeddings[i])
if args.compute_accuracy_f1:
fed_labels = out_labels if args.output_channel_encoding == 'multilabel' else out_labels_raw
acc_f1_txt, _, _ = eval_metrics.compute_acc_f1_metrics(fed_labels, preds_txt,
args.output_channel_encoding) # based on thresholding
acc_f1_img, _, _ = eval_metrics.compute_acc_f1_metrics(fed_labels, preds_img,
args.output_channel_encoding)
results_txt.update(acc_f1_txt)
results_img.update(acc_f1_img)
if args.compute_auc:
if args.output_channel_encoding == 'multilabel':
preds_img_squashed = logistic.cdf(preds_img)
preds_txt_squashed = logistic.cdf(preds_txt)
else:
preds_img_squashed = softmax(preds_img, axis=1) #TODO geeticka: check if the dimensions are right
preds_txt_squashed = softmax(preds_txt, axis=1)
auc_images, pairwise_auc_images = eval_metrics.compute_auc(out_labels, preds_img_squashed,
args.output_channel_encoding)
auc_txt, pairwise_auc_txt = eval_metrics.compute_auc(out_labels, preds_txt_squashed,
args.output_channel_encoding)
if args.output_channel_encoding == 'multiclass': # let's compute the 3 channel auc value
ord_auc_images = eval_metrics.compute_ordinal_auc_from_multiclass(out_labels_raw, preds_img_squashed)
ord_auc_txt = eval_metrics.compute_ordinal_auc_from_multiclass(out_labels_raw, preds_txt_squashed)
results_img['ordinal_auc'] = ord_auc_images
results_txt['ordinal_auc'] = ord_auc_txt
results_img['auc'] = auc_images
results_txt['auc'] = auc_txt
results_img['pairwise_auc'] = pairwise_auc_images
results_txt['pairwise_auc'] = pairwise_auc_txt
if args.compute_mse:
results_txt['mse'] = eval_metrics.compute_mse(preds_txt, out_labels, args.output_channel_encoding)
results_img['mse'] = eval_metrics.compute_mse(preds_img, out_labels, args.output_channel_encoding)
#print("Result img all egs", result_img_all_egs)
#result_img = np.mean(result_img_all_egs)
#print('Result img', result_img)
if dump_prediction_files == True:
#TODO: only do below for the main checkpoint
for vals, variables in zip(['txt', 'img'],[[results_txt, out_labels, preds_txt],
[results_img, out_labels, preds_img]]):
output_eval_file = os.path.join(eval_output_dir, "eval_results_{}.txt".format(vals))
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results {} {} *****".format(vals, prefix))
print('**** Eval results {} {} *****'.format(vals, prefix))
for key in sorted(variables[0].keys()): # result file
logger.info(" %s = %s"%(key, str(variables[0][key])))
print(' %s = %s'%(key, str(variables[0][key])))
writer.write("%s = %s\n" % (key, str(variables[0][key])))
with open(os.path.join(eval_output_dir, 'gold_labels_{}.txt'.format(vals)), 'w') as gold_file:
for item in variables[1]: # gold labels
gold_file.write('%s\n'%item)
with open(os.path.join(eval_output_dir, 'predicted_labels_{}.txt'.format(vals)), 'w') as pred_file:
for item in variables[2]:
pred_file.write('%s\n'%item)
return results_txt, results_img
# for key, value in tqdm(data_dict1.items()):
# if key in data_dict1.keys():
# array1 = np.array(value)
# array2 = np.array(data_dict2[key])
# # array3 = np.array(data_dict3[key])
# # print(array1.shape)
# # m_array = (array1 + array2 + array3) / 3
# m_array = (array1 + array2) / 2
# data_json = {"image_path":key, "image_logits":m_array.tolist()}
# file.write(json.dumps(data_json) + '\n')
# ensemble method
ensemble_method = "mean"
# # merge
with open(os.path.join(PATH, "r200_r101_repeat_200_clean.csv"),
"w") as csvfile:
writer = csv.writer(csvfile)
writer.writerow(["image_name", "class"])
for key, value in tqdm(data_dict1.items()):
if key in data_dict1.keys():
array1 = np.array(value)
array2 = np.array(data_dict2[key])
array3 = np.array(data_dict3[key])
# print(array1.shape)
if ensemble_method == "mean":
# m_array = (array1 + array2) / 2
m_array = (array1 + array2 + array3) / 3
# TODO: vote method
predlabel = np.argmax(softmax(m_array))
writer.writerow([key, str(predlabel)])
def soft_identity(self):
data = self.data[~np.isnan(self.data).any(axis=1)]
unq, idx, cnt = np.unique(data[:, 3], return_inverse=True, return_counts=True)
avg = np.bincount(idx, weights=data[:, 2]) / cnt
soft = softmax(avg)
return dict(zip(unq.astype(int), soft))
def eval_label(self, train_dataloader):
torch.cuda.empty_cache()
self.bert_lm_sentence.eval()
self.metric_module.eval()
preds = []
all_label_ids = []
for step, batch in enumerate(tqdm(train_dataloader, desc="eval")):
if self.args.use_cuda:
batch = tuple(t.cuda() for t in batch)
else:
batch = tuple(t for t in batch)
with torch.no_grad():
label_desc1, label_len1, label_mask1, label_desc2, label_len2, label_mask2, label_ids = batch
label_desc1.data = label_desc1.data[:, 0:int(max(
label_len1))] # trim down input to max len of the batch
label_mask1.data = label_mask1.data[:, 0:int(max(
label_len1))] # trim down input to max len of the batch
label_emb1 = self.encode_label_desc(label_desc1, label_len1,
label_mask1)
label_desc2.data = label_desc2.data[:, 0:int(max(label_len2))]
label_mask2.data = label_mask2.data[:, 0:int(max(label_len2))]
label_emb2 = self.encode_label_desc(label_desc2, label_len2,
label_mask2)
loss, prob = self.metric_module.forward(label_emb1,
label_emb2,
true_label=label_ids)
if len(preds) == 0:
preds.append(prob.detach().cpu().numpy())
all_label_ids.append(label_ids.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
prob.detach().cpu().numpy(),
axis=0)
all_label_ids[0] = np.append(all_label_ids[0],
label_ids.detach().cpu().numpy(),
axis=0) # row array
# end eval
all_label_ids = all_label_ids[0]
preds = preds[0]
if self.metric_option == 'entailment':
preds = softmax(
preds,
axis=1) ## softmax, return both prob of 0 and 1 for each label
print(preds)
print(all_label_ids)
result = 0
if self.args.test_file is None: ## save some time
result = acc_and_f1(
preds, all_label_ids, self.metric_option
) ## interally, we will take care of the case of @entailment vs @cosine
for key in sorted(result.keys()):
print("%s=%s" % (key, str(result[key])))
return result, preds
def Fwd(self, x, h):
a = self.RNN_W @ h + self.RNN_U @ x + self.RNN_b # mx1
h = np.tanh(a) # mx1
o = self.RNN_V @ h + self.RNN_c # Cx1
p = softmax(o) # Cx1
return a, h, o, p
if iteration % 1 == 0:
print("Iteration ", iteration)
print(" Current state:", s)
print(" Best action per state:", np.argmax(Q, axis=1))
print(" Q-table:")
for s_ in range(Q.shape[0]):
for a_ in range(Q.shape[1]):
print(" ["+str(s_)+","+str(a_)+"] -> N: ",ET[s_,a_],", R: ",Q[s_,a_])
#Decide whether to do greedy or random
rnd = np.random.random_sample()
e_param = e_param0 / (1 + iteration * e_param_decay)
print(" E-param:",e_param)
if rnd<e_param: # be random
print("ET:",ET[s], ", softmax:",(1.0-softmax(ET[s])))
a = np.random.choice(possible_actions[s], p=(1.0-softmax(ET[s]))) # choose an action (randomly)
print(" Random action:",a)
else: # be greedy
a = np.argmax(Q, axis=1)[s]
print(" Greedy action:",a)
print(" ->Distr of transitions [",s,",",a,"]: ", T[s,a])
sp = np.random.choice(range(T.shape[0]), p=T[s, a]) # pick next state using T[s, a]
print(" Transitioned to state:",sp)
transition_history.append([s,a,sp])
# Approach 1 - Just reward in end state with value depending on the actions before
#non-terminal state
if (sp < T.shape[0]-1):
def main():
args = build_argparser().parse_args()
cap = open_images_capture(args.input, args.loop)
# Plugin initialization for specified device and load extensions library if specified.
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if args.cpu_extension and 'CPU' in args.device:
core.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Reading Mask-RCNN model {}'.format(args.mask_rcnn_model))
mask_rcnn_model = core.read_model(args.mask_rcnn_model)
input_tensor_name = 'image'
try:
n, c, h, w = mask_rcnn_model.input(input_tensor_name).shape
if n != 1:
raise RuntimeError(
'Only batch 1 is supported by the demo application')
except RuntimeError:
raise RuntimeError(
'Demo supports only topologies with the following input tensor name: {}'
.format(input_tensor_name))
required_output_names = {'boxes', 'labels', 'masks', 'text_features.0'}
for output_tensor_name in required_output_names:
try:
mask_rcnn_model.output(output_tensor_name)
except RuntimeError:
raise RuntimeError(
'Demo supports only topologies with the following output tensor names: {}'
.format(', '.join(required_output_names)))
log.info('Reading Text Recognition Encoder model {}'.format(
args.text_enc_model))
text_enc_model = core.read_model(args.text_enc_model)
log.info('Reading Text Recognition Decoder model {}'.format(
args.text_dec_model))
text_dec_model = core.read_model(args.text_dec_model)
mask_rcnn_compiled_model = core.compile_model(mask_rcnn_model,
device_name=args.device)
mask_rcnn_infer_request = mask_rcnn_compiled_model.create_infer_request()
log.info('The Mask-RCNN model {} is loaded to {}'.format(
args.mask_rcnn_model, args.device))
text_enc_compiled_model = core.compile_model(text_enc_model, args.device)
text_enc_infer_request = text_enc_compiled_model.create_infer_request()
log.info('The Text Recognition Encoder model {} is loaded to {}'.format(
args.text_enc_model, args.device))
text_dec_compiled_model = core.compile_model(text_dec_model, args.device)
text_dec_infer_request = text_dec_compiled_model.create_infer_request()
log.info('The Text Recognition Decoder model {} is loaded to {}'.format(
args.text_dec_model, args.device))
hidden_shape = text_dec_model.input(args.trd_input_prev_hidden).shape
text_dec_output_names = {
args.trd_output_symbols_distr, args.trd_output_cur_hidden
}
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
if args.delay:
delay = args.delay
else:
delay = int(cap.get_type() in ('VIDEO', 'CAMERA'))
visualizer = Visualizer(['__background__', 'text'],
show_boxes=args.show_boxes,
show_scores=args.show_scores)
frames_processed = 0
metrics = PerformanceMetrics()
video_writer = cv2.VideoWriter()
start_time = perf_counter()
frame = cap.read()
if frame is None:
raise RuntimeError("Can't read an image from the input")
presenter = monitors.Presenter(args.utilization_monitors, 45,
(frame.shape[1] // 4, frame.shape[0] // 8))
if args.output and not video_writer.open(
args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(),
(frame.shape[1], frame.shape[0])):
raise RuntimeError("Can't open video writer")
while frame is not None:
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
# Run the MaskRCNN model.
mask_rcnn_infer_request.infer({input_tensor_name: input_image})
outputs = {
name: mask_rcnn_infer_request.get_tensor(name).data[:]
for name in required_output_names
}
# Parse detection results of the current request
boxes = outputs['boxes'][:, :4]
scores = outputs['boxes'][:, 4]
classes = outputs['labels'].astype(np.uint32)
raw_masks = outputs['masks']
text_features = outputs['text_features.0']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
mask = segm_postprocess(box, raw_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
feature = next(
iter(
text_enc_infer_request.infer({
'input':
np.expand_dims(feature, axis=0)
}).values()))
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
text_confidence = 1.0
for i in range(MAX_SEQ_LEN):
text_dec_infer_request.infer({
args.trd_input_prev_symbol:
np.reshape(prev_symbol_index, (1, )),
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
decoder_output = {
name: text_dec_infer_request.get_tensor(name).data[:]
for name in text_dec_output_names
}
symbols_distr = decoder_output[args.trd_output_symbols_distr]
symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
text_confidence *= symbols_distr_softmaxed[prev_symbol_index]
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text if text_confidence >= args.tr_threshold else '')
if len(boxes) and args.raw_output_message:
log.debug(
' -------------------------- Frame # {} -------------------------- '
.format(frames_processed))
log.debug(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.debug(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
presenter.drawGraphs(frame)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks, texts,
masks_tracks_ids)
metrics.update(start_time, frame)
frames_processed += 1
if video_writer.isOpened() and (args.output_limit <= 0 or
frames_processed <= args.output_limit):
video_writer.write(frame)
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
if not args.no_show:
key = cv2.waitKey(delay)
esc_code = 27
if key == esc_code:
break
presenter.handleKey(key)
start_time = perf_counter()
frame = cap.read()
metrics.log_total()
for rep in presenter.reportMeans():
log.info(rep)
def evaluate(model,
val_dataloader,
ood_dataloader,
criterion,
device,
writer,
T=None,
eps=None) -> Tuple:
"""
Evaluate model. This function prints validation loss and accuracy.
:param model: model to evaluate
:param dataloader: dataloader representing the validation data
:param callable criterion: the loss function
:return: None
"""
# Validation
model = model.to(device).eval()
all_predictions = []
all_losses = []
all_logits = []
all_labels = []
print("Temerature = ", T)
print("Epsilon = ", eps)
for images, labels in tqdm.tqdm(val_dataloader):
images, labels = images.to(device), labels.to(device).long()
for p in model.parameters():
p.grad.zero_()
if T is not None:
# ODIN case: this is simple
# Just make a slight adversarial attack on the images
# source: https://arxiv.org/pdf/1706.02690.pdf
images.requires_grad = True
logits = model.forward(images)
logits = logits / T
pred_labels = logits.argmax(-1)
pred_loss = criterion(logits, pred_labels)
pred_loss.backward()
corrupted_images = images - eps * torch.sign(-(images.grad))
logits = model.forward(corrupted_images)
else:
logits = model.forward(images)
labels = labels.cpu().detach().numpy()
logits = logits.cpu().detach().numpy()
all_logits.append(logits)
all_labels.append(labels)
# just not to write additional code for ood and vanilla task
# valid mask corresponds only for known labels
valid_logits_mask = (labels < logits.shape[1])
valid_logits = logits[valid_logits_mask]
valid_labels = labels[valid_logits_mask]
valid_predictions = valid_logits.argmax(1)
all_predictions.append((valid_predictions == valid_labels))
all_losses.append(
criterion(torch.from_numpy(valid_logits),
torch.from_numpy(valid_labels)).item())
torch.cuda.empty_cache()
# OOD
all_ood_probs = []
for images, labels in tqdm.tqdm(ood_dataloader):
images, labels = images.to(device), labels.to(device).long()
for p in model.parameters():
p.grad.zero_()
if T is not None:
# ODIN case: this is simple
# Just make a slight adversarial attack on the images
# source: https://arxiv.org/pdf/1706.02690.pdf
images.requires_grad = True
logits = model.forward(images)
logits = logits / T
pred_labels = logits.argmax(-1)
pred_loss = criterion(logits, pred_labels)
pred_loss.backward()
corrupted_images = images - eps * torch.sign(-(images.grad))
logits = model.forward(corrupted_images)
images.grad.zero_()
else:
logits = model.forward(images)
all_ood_probs.append(F.softmax(logits, dim=1).cpu().detach().numpy())
accuracy = np.concatenate(all_predictions).mean()
all_probs = softmax(np.concatenate(all_logits), axis=1).max(1)
all_labels = np.concatenate(all_labels)
all_ood_probs = np.concatenate(all_ood_probs).max(1)
plt.figure(figsize=(10, 8))
plt.title("Known classes vs OOD max logit distribution")
plt.hist(all_probs,
bins=20,
color="blue",
alpha=0.5,
label="max_softmax_probability_true",
density=True)
plt.hist(all_ood_probs,
bins=20,
color="red",
alpha=0.5,
label="max_softmax_probability_ood",
density=True)
plt.savefig("hist.png")
#log_hist_as_picture(all_labels, all_logits, ood_label=logits.shape[1])
loss = np.mean(all_losses)
print(" Evaluation results: \n Accuracy: {:.4f}\n Loss: {:.4f}".format(
accuracy, loss))
return loss, accuracy
from tvm.contrib.download import download_testdata
# Download a list of labels
labels_url = "https://s3.amazonaws.com/onnx-model-zoo/synset.txt"
labels_path = download_testdata(labels_url, "synset.txt", module="data")
with open(labels_path, "r") as f:
labels = [l.rstrip() for l in f]
output_file = "predictions.npz"
# Open the output and read the output tensor
if os.path.exists(output_file):
with np.load(output_file) as data:
scores = softmax(data["output_0"])
scores = np.squeeze(scores)
scores = np.argsort(scores)[::-1]
for i in scores[0:5]:
print("class='%s' with probability=%f" % (labels[i], scores[i]))
########################################################################
# When running the script, a list of predictions should be printed similar
# the the example below.
#
# .. code-block:: bash
#
# $ python post_processing.py
# class=n02123045 tabby, tabby cat ; probability=446.000000
# class=n02123159 tiger cat ; probability=675.000000
def _compute_auc(self):
prob = softmax(self.logits, axis=-1)
real_prob = prob[:,1]
self.auc = roc_auc_score(y_true=self.labels, y_score=real_prob) \
if self.auc is None else self.auc
return self.auc
def predict_proba(self, X):
z = self.predict_z(X)
dists = -(z.reshape(-1, 1) - self.z_means)**2
yp = softmax(dists, axis=1)
return yp
# Probabilities plot
dataPerRoom = dict()
dataPerRoom['room1'] = (np.array(data[rpiIds[0]]['btPower']) + np.array(
data[rpiIds[1]]['btPower']) + np.array(data[rpiIds[2]]['btPower'])) / 3
dataPerRoom['room2'] = (np.array(data[rpiIds[3]]['btPower']) + np.array(
data[rpiIds[4]]['btPower']) + np.array(data[rpiIds[3]]['btPower'])) / 3
dataPerRoom['room3'] = (np.array(data[rpiIds[6]]['btPower']) + np.array(
data[rpiIds[7]]['btPower']) + np.array(data[rpiIds[8]]['btPower'])) / 3
probabilities = np.zeros((len(times), 3))
for idx, time in enumerate(times):
currentBtPowers = np.array([
dataPerRoom['room1'][idx], dataPerRoom['room2'][idx],
dataPerRoom['room3'][idx]
])
probabilities[idx, :] = softmax(currentBtPowers)
fig, ax = plt.subplots(figsize=(12, 8))
roomLines = [0] * 3
colors = ['b', 'lime', 'red']
markers = ['*', 'o', '+']
for idx in range(3):
room = 'room' + str(idx + 1)
x = data[rpiIds[idx]]['dates']
y = probabilities[:, idx]
x = range(len(y))
plt.plot(x,
y,
color=colors[idx],
linestyle="None",
marker=markers[idx],
def do_eval(self, train_dataloader, labeldesc_loader, edge_index):
torch.cuda.empty_cache()
self.eval()
tr_loss = 0
preds = []
all_label_ids = []
with torch.no_grad(): ## don't need to update labels anymore
label_emb = self.gcn_2layer(labeldesc_loader, edge_index)
print('sample gcn label_emb')
print(label_emb)
## for each batch
for step, batch in enumerate(tqdm(train_dataloader, desc="eval")):
batch = tuple(t for t in batch)
label_id_number_left, label_id_number_right, label_ids = batch
label_id_number_left = label_id_number_left.squeeze(1).data.numpy(
) ## using as indexing, so have to be array int, not tensor
label_id_number_right = label_id_number_right.squeeze(
1).data.numpy()
## need to backprop somehow
## predict the class bio/molec/cellcompo ?
## predict if 2 labels are similar ? ... sort of doing the same thing as gcn already does
with torch.no_grad():
loss, score = self.metric_module.forward(
label_emb[label_id_number_left],
label_emb[label_id_number_right],
true_label=label_ids.cuda())
tr_loss = tr_loss + loss
if len(preds) == 0:
preds.append(score.detach().cpu().numpy())
all_label_ids.append(label_ids.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
score.detach().cpu().numpy(),
axis=0)
all_label_ids[0] = np.append(all_label_ids[0],
label_ids.detach().cpu().numpy(),
axis=0) # row array
# end eval
all_label_ids = all_label_ids[0]
preds = preds[0]
if self.metric_option == 'entailment':
preds = softmax(
preds,
axis=1) ## softmax, return both prob of 0 and 1 for each label
print(preds)
print(all_label_ids)
result = 0
if self.args.test_file is None: ## save some time
result = acc_and_f1(
preds, all_label_ids, self.metric_option
) ## interally, we will take care of the case of @entailment vs @cosine
for key in sorted(result.keys()):
print("%s=%s" % (key, str(result[key])))
return result, preds, tr_loss
def get_argmax_char(chunk, int_to_char):
return int_to_char[np.argmax(softmax(chunk))]
def clusterSM(outpth,
score,
bdpc,
clnum,
pcnum=None,
VamModel=None,
BuildModel=None,
condition=None,
setID=None,
entries=None):
realtimedate = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
start = time.time()
print('# clusterSM')
if not isinstance(condition, str):
condition = str(condition)
if BuildModel:
figdst = os.path.join(
*[outpth, entries['Model name'].get(), 'Example model figures'])
else:
figdst = os.path.join(
outpth, 'Result based on ' + os.path.splitext(
os.path.basename(entries['Model to apply'].get()))[0])
if not os.path.exists(figdst):
try:
os.makedirs(figdst)
except:
entries['Status'].delete(0, END)
entries['Status'].insert(0, 'Please choose the right folder')
NN = 10
if pcnum is None:
pcnum = 20
if BuildModel:
VamModel['clnum'] = clnum
VamModel['pcnum'] = pcnum
else:
clnum = VamModel['clnum']
pcnum = VamModel['pcnum']
cms00 = score[:, 0:pcnum]
cms = deepcopy(cms00)
if BuildModel:
mincms = np.amin(cms, axis=0)
VamModel['mincms'] = mincms
VamModel['boxcoxlambda'] = np.zeros(len(cms.T))
VamModel['testmean'] = np.zeros(len(cms.T))
VamModel['teststd'] = np.zeros(len(cms.T))
else:
mincms = VamModel['mincms']
for k in range(len(cms.T)):
test = cms.T[k]
test = test - mincms[k] + 1
if BuildModel:
test[test < 0] = 0.000000000001
test, maxlog = stats.boxcox(test)
test = np.asarray(test)
VamModel['boxcoxlambda'][k] = maxlog
VamModel['testmean'][k] = np.mean(test)
VamModel['teststd'][k] = np.std(test)
cms.T[k] = (test - np.mean(test)) / np.std(test)
else:
test[test < 0] = 0.000000000001
test = stats.boxcox(test, VamModel['boxcoxlambda'][k])
cms.T[k] = (test -
VamModel['testmean'][k]) / VamModel['teststd'][k]
cmsn = deepcopy(cms)
if BuildModel:
cmsn_Norm = preprocessing.normalize(cmsn)
if isinstance(clnum, str):
clnum = int(clnum)
kmeans = KMeans(
n_clusters=clnum, init='k-means++', n_init=3, max_iter=300).fit(
cmsn_Norm
) # init is plus,but orginally cluster, not available in sklearn
C = kmeans.cluster_centers_
VamModel['C'] = C
D = spatial.distance.cdist(cmsn, C, metric='euclidean')
IDX = np.argmin(D, axis=1)
IDX_dist = np.amin(D, axis=1)
else:
if isinstance(clnum, str):
clnum = int(clnum)
C = VamModel['C']
D = spatial.distance.cdist(cmsn, C, metric='euclidean')
# why amin? D shows list of distance to cluster centers.
IDX = np.argmin(D, axis=1)
IDX_dist = np.around(np.amin(D, axis=1), decimals=2)
goodness = special.softmax(D)
offx, offy = np.meshgrid(range(clnum), [0])
offx = np.multiply(offx, 1) + 1
offx = offx[0] * 1 - 0.5
offy = np.subtract(np.multiply(offy, 1), 1.5) + 1
offy = offy[0]
# define normalized colormap
bdst0 = np.empty(len(bdpc.T))
bdst = deepcopy(bdst0)
for kss in range(clnum):
c88 = IDX == kss
bdpcs = bdpc[c88, :]
mbd = np.mean(bdpcs, axis=0)
bdst0 = np.vstack((bdst0, mbd))
bdst0 = bdst0[1:]
# dendrogram of the difference between different shape
mpl.rcParams['lines.linewidth'] = 2
if BuildModel:
Y = spatial.distance.pdist(bdst0, 'euclidean')
Z = cluster.hierarchy.linkage(
Y, method='complete') # 4th row is not in matlab
Z[:,
2] = Z[:,
2] * 5 # multiply distance manually 10times to plot better.
VamModel['Z'] = Z
else:
Z = VamModel['Z']
cluster.hierarchy.set_link_color_palette(['k'])
fig289, ax289 = plt.subplots(figsize=(6, 2), linewidth=2.0, frameon=False)
plt.yticks([])
R = cluster.hierarchy.dendrogram(Z,
p=0,
truncate_mode='mlab',
orientation='bottom',
ax=None,
above_threshold_color='k')
leaflabel = np.array(R['ivl'])
dendidx = leaflabel
cluster.hierarchy.set_link_color_palette(None)
mpl.rcParams['lines.linewidth'] = 1
plt.axis('equal')
plt.axis('off')
IDXsort = np.zeros(len(IDX))
for kss in range(clnum):
c88 = IDX == int(dendidx[kss])
IDXsort[c88] = kss
IDX = deepcopy(IDXsort)
fig922, ax922 = plt.subplots(figsize=(17, 2))
fig291, ax291 = plt.subplots(figsize=(6, 3))
for kss in range(int(max(IDX)) + 1):
c88 = IDXsort == kss
fss = 4
bdpcs = bdpc[c88]
mbd = np.mean(bdpcs, axis=0)
bdNUM = int(round(len(mbd) / 2))
bdst = np.vstack((bdst, mbd))
xaxis = np.add(np.divide(np.append(mbd[0:bdNUM], mbd[0]), fss),
offx[kss]) * 10
yaxis = np.add(np.divide(np.append(mbd[bdNUM:], mbd[bdNUM]), fss),
offy[kss]) * 10
plt.clf()
ax289.plot(xaxis, yaxis, '-',
linewidth=2) # this is the shape of the dendrogram
plt.axis('equal')
plt.axis('off')
sid = np.argsort(np.random.rand(sum(c88), 1), axis=0)
if len(sid) < NN:
enum = len(sid)
else:
enum = NN
for knn in range(enum):
x99 = bdpcs[sid[knn], np.append(range(bdNUM), 0)]
y99 = bdpcs[sid[knn],
np.append(np.arange(bdNUM, (bdNUM * 2), 1), bdNUM)]
xax = np.add(np.divide(x99, fss), offx[kss])
yax = np.add(np.divide(y99, fss), offy[kss])
ax922.plot(xax, yax, 'r-', linewidth=1)
ax922.axis('equal')
ax922.axis('off')
if BuildModel:
ax922.set_ylim(ax922.get_ylim()[::-1])
if os.path.exists(os.path.join(figdst, "Registered objects.png")):
f1 = os.path.join(figdst,
"Registered objects " + realtimedate + ".png")
f2 = os.path.join(
figdst, "Shape mode dendrogram.png " + realtimedate + ".png")
else:
f1 = os.path.join(figdst, "Registered objects.png")
f2 = os.path.join(figdst, "Shape mode dendrogram.png")
fig922.savefig(f1, format='png', transparent=True)
fig289.savefig(f2, format='png', transparent=True)
IDX = IDX + 1
n, bins, patches = plt.hist(IDX, bins=range(clnum + 2)[1:])
fig22, ax22 = plt.subplots(figsize=(10, 5))
n = np.divide(n, np.sum(n))
n = np.multiply(n, 100)
n = np.around(n, 2)
height = n
ax22.bar(x=(np.delete(bins, 0) - 1) / 2,
height=height,
width=0.4,
align='center',
color=(0.2, 0.4, 0.6, 1),
edgecolor='black')
ax22.set_ylabel('Abundance %', fontsize=15, fontweight='bold')
ax22.set_xlabel('Shape mode', fontsize=15, fontweight='bold')
# only for paper
ax22.set_ylim([0, np.max(height) + 5])
ax22.set_title('Shape mode distribution (N=' + str(len(IDX_dist)) + ')',
fontsize=18,
fontweight='bold')
bartick = map(str, np.arange(int(np.max(IDX) + 1))[1:])
ax22.set_xticks((np.arange(np.max(IDX) + 1) / 2)[1:])
ax22.set_xticklabels(tuple(bartick), fontsize=13, fontweight='bold')
ax22.yaxis.set_tick_params(labelsize=13)
plt.setp(ax22.get_yticklabels(), fontweight="bold")
for i, v in enumerate(height):
ax22.text((i - 0.25 + 1) / 2,
v + 0.25,
str(np.around(v, decimals=1)),
color='black',
fontweight='bold',
fontsize=13)
for axis in ['top', 'bottom', 'left', 'right']:
ax22.spines[axis].set_linewidth(3)
if not BuildModel:
if os.path.exists(
os.path.join(
figdst, 'Shape mode distribution_' + setID + '_' +
condition + '.png')):
f3 = os.path.join(
figdst, 'Shape mode distribution_' + setID + '_' + condition +
'_' + realtimedate + '.png')
else:
f3 = os.path.join(
figdst,
'Shape mode distribution_' + setID + '_' + condition + '.png')
fig22.savefig(f3, format='png', transparent=True)
plt.close('all')
end = time.time()
print('For cluster, elapsed time is ' + str(end - start) + 'seconds...')
return IDX, IDX_dist, VamModel, goodness
def main():
if args.dataset == "mnist":
num_classes = 10
elif args.dataset == "fmnist":
num_classes = 10
elif args.dataset == "cifar10":
num_classes = 10
elif args.dataset == "svhn":
num_classes = 10
elif args.dataset == "cifar100":
num_classes = 100
else:
print("This dataset is not supported in current version!")
exit()
labels_train = np.load(args.outputs_path + "/label_train.npy")
labels_test = np.load(args.outputs_path + "/label_test.npy")
if len(labels_train.shape) > 1:
labels_train = labels_train.reshape((-1))
if len(labels_test.shape) > 1:
labels_test = labels_test.reshape((-1))
train_conf_all = np.load(args.outputs_path + "/" + args.output_type + "_train.npy")
test_conf_all = np.load(args.outputs_path + "/" + args.output_type + "_test.npy")
number_of_models = int(train_conf_all.shape[-1] / num_classes)
train_conf_sum = np.zeros((labels_train.shape[0], num_classes))
test_conf_sum = np.zeros((labels_test.shape[0], num_classes))
train_prediction_class_sum = np.zeros((labels_train.shape[0], num_classes))
test_prediction_class_sum = np.zeros((labels_test.shape[0], num_classes))
for model_index_counter in range(number_of_models):
train_conf_sum += train_conf_all[:, model_index_counter * num_classes:(model_index_counter + 1) * num_classes]
test_conf_sum += test_conf_all[:, model_index_counter * num_classes:(model_index_counter + 1) * num_classes]
temp1 = np.argmax(train_conf_all[:, model_index_counter * num_classes:(model_index_counter + 1) * num_classes], axis=1)
temp2 = np.argmax(test_conf_all[:, model_index_counter * num_classes:(model_index_counter + 1) * num_classes], axis=1)
train_prediction_class_sum += to_categorical(temp1, num_classes)
test_prediction_class_sum += to_categorical(temp2, num_classes)
if args.output_type == "confidence":
confidence_train_for_prediction = train_conf_sum / (model_index_counter + 1)
confidence_test_for_prediction = test_conf_sum / (model_index_counter + 1)
elif args.output_type == "logit":
confidence_train_for_prediction = softmax(train_conf_sum / (model_index_counter + 1), axis=1)
confidence_test_for_prediction = softmax(test_conf_sum / (model_index_counter + 1), axis=1)
else:
print("Output type does not exist!")
exit()
if args.attack_type == "all":
confidence_train_for_attack = train_conf_all[:, 0:(model_index_counter + 1) * num_classes]
confidence_test_for_attack = test_conf_all[:, 0:(model_index_counter + 1) * num_classes]
elif args.attack_type == "aggregated":
confidence_train_for_attack = confidence_train_for_prediction
confidence_test_for_attack = confidence_test_for_prediction
else:
print("Attack type is not valid!")
exit()
labels_train_by_model = np.argmax(confidence_train_for_prediction, axis=1)
labels_test_by_model = np.argmax(confidence_test_for_prediction, axis=1)
acc_train = np.sum(labels_train == labels_train_by_model)/labels_train.shape[0]
acc_test = np.sum(labels_test == labels_test_by_model)/labels_test.shape[0]
correctly_classified_indexes_train = labels_train_by_model == labels_train
incorrectly_classified_indexes_train = labels_train_by_model != labels_train
correctly_classified_indexes_test = labels_test_by_model == labels_test
incorrectly_classified_indexes_test = labels_test_by_model != labels_test
MI_x_train_all = []
MI_y_train_all = []
MI_x_test_all = []
MI_y_test_all = []
MI_cor_labeled_indexes_all = []
MI_incor_labeled_indexes_all = []
for j in range(num_classes):
#Prepare the data for training and testing attack models (for all data and also correctly labeled samples)
class_yes_x = confidence_train_for_attack[tuple([labels_train == j])]
class_no_x = confidence_test_for_attack[tuple([labels_test == j])]
if class_yes_x.shape[0] < 10 or class_no_x.shape[0] < 10:
print("Class " + str(j) + " doesn't have enough sample for training an attack model (SKIPPED)!")
continue
class_yes_x_correctly_labeled = correctly_classified_indexes_train[tuple([labels_train == j])]
class_no_x_correctly_labeled = correctly_classified_indexes_test[tuple([labels_test == j])]
class_yes_x_incorrectly_labeled = incorrectly_classified_indexes_train[tuple([labels_train == j])]
class_no_x_incorrectly_labeled = incorrectly_classified_indexes_test[tuple([labels_test == j])]
class_yes_size = int(class_yes_x.shape[0] * what_portion_of_sampels_attacker_knows)
class_yes_x_train = class_yes_x[:class_yes_size]
class_yes_y_train = np.ones(class_yes_x_train.shape[0])
class_yes_x_test = class_yes_x[class_yes_size:]
class_yes_y_test = np.ones(class_yes_x_test.shape[0])
class_yes_x_correctly_labeled = class_yes_x_correctly_labeled[class_yes_size:]
class_yes_x_incorrectly_labeled = class_yes_x_incorrectly_labeled[class_yes_size:]
class_no_size = int(class_no_x.shape[0] * what_portion_of_sampels_attacker_knows)
class_no_x_train = class_no_x[:class_no_size]
class_no_y_train = np.zeros(class_no_x_train.shape[0])
class_no_x_test = class_no_x[class_no_size:]
class_no_y_test = np.zeros(class_no_x_test.shape[0])
class_no_x_correctly_labeled = class_no_x_correctly_labeled[class_no_size:]
class_no_x_incorrectly_labeled = class_no_x_incorrectly_labeled[class_no_size:]
y_size = class_yes_x_train.shape[0]
n_size = class_no_x_train.shape[0]
if sampling == "undersampling":
if y_size > n_size:
class_yes_x_train = class_yes_x_train[:n_size]
class_yes_y_train = class_yes_y_train[:n_size]
else:
class_no_x_train = class_no_x_train[:y_size]
class_no_y_train = class_no_y_train[:y_size]
elif sampling == "oversampling":
if y_size > n_size:
class_no_x_train = np.tile(class_no_x_train, (int(y_size / n_size), 1))
class_no_y_train = np.zeros(class_no_x_train.shape[0])
else:
class_yes_x_train = np.tile(class_yes_x_train, (int(n_size / y_size), 1))
class_yes_y_train = np.ones(class_yes_x_train.shape[0])
MI_x_train = np.concatenate((class_yes_x_train, class_no_x_train), axis=0)
MI_y_train = np.concatenate((class_yes_y_train, class_no_y_train), axis=0)
MI_x_test = np.concatenate((class_yes_x_test, class_no_x_test), axis=0)
MI_y_test = np.concatenate((class_yes_y_test, class_no_y_test), axis=0)
MI_x_train_all.extend(MI_x_train)
MI_y_train_all.extend(MI_y_train)
MI_x_test_all.extend(MI_x_test)
MI_y_test_all.extend(MI_y_test)
MI_cor_labeled_indexes = np.concatenate((class_yes_x_correctly_labeled, class_no_x_correctly_labeled), axis=0)
MI_incor_labeled_indexes = np.concatenate((class_yes_x_incorrectly_labeled, class_no_x_incorrectly_labeled), axis=0)
MI_cor_labeled_indexes_all.extend(MI_cor_labeled_indexes)
MI_incor_labeled_indexes_all.extend(MI_incor_labeled_indexes)
MI_x_train_all = np.array(MI_x_train_all)
MI_y_train_all = np.array(MI_y_train_all)
MI_x_test_all = np.array(MI_x_test_all)
MI_y_test_all = np.array(MI_y_test_all)
#To shuffle the training data:
shuffle_index = np.random.permutation(MI_x_train_all.shape[0])
MI_x_train_all = MI_x_train_all[shuffle_index]
MI_y_train_all = MI_y_train_all[shuffle_index]
# MI attack
if args.attack_type == "all":
attack_model = nn.Sequential(nn.Linear(num_classes * (model_index_counter + 1), 128), nn.ReLU(),
nn.Linear(128, 64), nn.ReLU(), nn.Linear(64, 1), nn.Sigmoid())
elif args.attack_type == "aggregated":
attack_model = nn.Sequential(nn.Linear(num_classes, 128), nn.ReLU(), nn.Linear(128, 64), nn.ReLU(),
nn.Linear(64, 1), nn.Sigmoid())
else:
print("Attack type is not valid!")
exit()
attack_model = attack_model.cuda()
criterion = nn.BCELoss().cuda()
optimizer = optim.Adam(attack_model.parameters(), lr=0.001)
MI_x_train_cuda = torch.from_numpy(MI_x_train_all).float().cuda()
MI_y_train_cuda = torch.from_numpy(MI_y_train_all).float().cuda()
MI_x_test_cuda = torch.from_numpy(MI_x_test_all).float().cuda()
MI_y_test_cuda = torch.from_numpy(MI_y_test_all).float().cuda()
for ep in range(30):
y_pred = attack_model(MI_x_train_cuda)
y_pred = torch.squeeze(y_pred)
train_loss = criterion(y_pred, MI_y_train_cuda)
optimizer.zero_grad()
train_loss.backward()
optimizer.step()
y_pred = attack_model(MI_x_test_cuda).cpu().detach().numpy()
if y_pred.shape[0] > 0:
MI_attack_auc = roc_auc_score(MI_y_test_all, y_pred)
else:
MI_attack_auc = -1
#Gap attack
MI_predicted_y_test_blind = np.zeros((MI_x_test_all.shape[0]))
MI_predicted_y_test_blind[MI_cor_labeled_indexes_all] = 1
y_pred = np.array(MI_predicted_y_test_blind)
if y_pred.shape[0] > 0:
MI_blind_attack_auc = roc_auc_score(MI_y_test_all, y_pred)
else:
MI_blind_attack_auc = -1
print("---------------------")
print("Ensemble of", model_index_counter + 1, "models:")
print("Train/Test accuracy:", str(np.round(acc_train*100, 2)), str(np.round(acc_test*100, 2)))
print(args.attack_type + " " + args.output_type + "-based MI attack AUC:", MI_attack_auc)
print("Gap attack AUC:", MI_blind_attack_auc)
print("---------------------")
def pvals_from_weights_3(weights):
return softmax(weights)
def predict_on_batch(self, images: np.ndarray):
processed = np.asarray((images + 1) * 255.0 / 2, np.uint8)
processed = preprocess_input(processed)
yhat = self.model.predict(processed)
yhat = softmax(yhat, axis=1)
self.predictions += yhat.tolist()
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument('--kshot',
type=int,
default=5,
help="random seed for initialization")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=16,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=64,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=1e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
args = parser.parse_args()
processors = {"rte": RteProcessor}
output_modes = {"rte": "classification"}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
train_examples = processor.get_GAP_coreference(
'gap-development.tsv', args.kshot) #train_pu_half_v1.txt
dev_examples = processor.get_GAP_coreference('gap-validation.tsv', 0)
test_examples = processor.get_GAP_coreference('gap-test.tsv', 0)
label_list = ["entailment", "not_entailment"]
entity_label_list = ["A-coref", "B-coref"]
# train_examples = get_data_hulu_fewshot('train', 5)
# train_examples, dev_examples, test_examples, label_list = load_CLINC150_with_specific_domain_sequence(args.DomainName, args.kshot, augment=False)
num_labels = len(label_list)
print('num_labels:', num_labels, 'training size:', len(train_examples),
'dev size:', len(dev_examples), 'test size:', len(test_examples))
num_train_optimization_steps = None
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
model = RobertaForSequenceClassification(num_labels)
tokenizer = RobertaTokenizer.from_pretrained(
pretrain_model_dir, do_lower_case=args.do_lower_case)
model.to(device)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
max_test_acc = 0.0
max_dev_acc = 0.0
max_dev_threshold = 0.0
if args.do_train:
train_dataloader = examples_to_features(train_examples,
label_list,
entity_label_list,
args,
tokenizer,
args.train_batch_size,
"classification",
dataloader_mode='random')
dev_dataloader = examples_to_features(dev_examples,
label_list,
entity_label_list,
args,
tokenizer,
args.eval_batch_size,
"classification",
dataloader_mode='sequential')
test_dataloader = examples_to_features(test_examples,
label_list,
entity_label_list,
args,
tokenizer,
args.eval_batch_size,
"classification",
dataloader_mode='sequential')
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
iter_co = 0
final_test_performance = 0.0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
model.train()
batch = tuple(t.to(device) for t in batch)
input_example_ids, input_ids, input_mask, segment_ids, label_ids, entity_label_ids = batch
logits = model(input_ids, input_mask)
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
optimizer.step()
optimizer.zero_grad()
global_step += 1
iter_co += 1
# if iter_co %100==0:
# print('iter_co:', iter_co, ' mean loss:', tr_loss/iter_co)
if iter_co % len(train_dataloader) == 0:
model.eval()
'''
dev set after this epoch
'''
logger.info("***** Running dev *****")
logger.info(" Num examples = %d", len(dev_examples))
eval_loss = 0
nb_eval_steps = 0
preds = []
gold_label_ids = []
example_id_list = []
for _, batch in enumerate(tqdm(dev_dataloader,
desc="dev")):
input_indices, input_ids, input_mask, segment_ids, _, label_ids = batch
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
example_ids = list(input_indices.numpy())
example_id_list += example_ids
gold_label_ids += list(
label_ids.detach().cpu().numpy())
with torch.no_grad():
logits = model(input_ids, input_mask)
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
preds = preds[0]
pred_probs = softmax(preds, axis=1)
pred_label_ids_3way = list(np.argmax(pred_probs, axis=1))
pred_prob_entail = list(pred_probs[:, 0])
assert len(example_id_list) == len(pred_prob_entail)
assert len(example_id_list) == len(gold_label_ids)
assert len(example_id_list) == len(pred_label_ids_3way)
best_current_dev_acc = 0.0
best_current_threshold = -10.0
for threshold in np.arange(0.99, 0.0, -0.01):
eval_output_list = build_GAP_output_format(
example_id_list,
gold_label_ids,
pred_prob_entail,
pred_label_ids_3way,
threshold,
dev_or_test='validation')
dev_acc = run_scorer(
'/export/home/Dataset/gap_coreference/gap-validation.tsv',
eval_output_list)
if dev_acc > best_current_dev_acc:
best_current_dev_acc = dev_acc
best_current_threshold = threshold
print('best_current_dev_threshold:',
best_current_threshold, 'best_current_dev_acc:',
best_current_dev_acc)
if best_current_dev_acc > max_dev_acc:
max_dev_acc = best_current_dev_acc
max_dev_threshold = best_current_threshold
'''eval on test set'''
logger.info("***** Running test *****")
logger.info(" Num examples = %d", len(test_examples))
eval_loss = 0
nb_eval_steps = 0
preds = []
gold_label_ids = []
example_id_list = []
for _, batch in enumerate(
tqdm(test_dataloader, desc="test")):
input_indices, input_ids, input_mask, segment_ids, _, label_ids = batch
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
example_ids = list(input_indices.numpy())
example_id_list += example_ids
gold_label_ids += list(
label_ids.detach().cpu().numpy())
with torch.no_grad():
logits = model(input_ids, input_mask)
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(
preds[0],
logits.detach().cpu().numpy(),
axis=0)
preds = preds[0]
pred_probs = softmax(preds, axis=1)
pred_label_ids_3way = list(
np.argmax(pred_probs, axis=1))
pred_prob_entail = list(pred_probs[:, 0])
assert len(example_id_list) == len(pred_prob_entail)
assert len(example_id_list) == len(gold_label_ids)
assert len(example_id_list) == len(pred_label_ids_3way)
threshold = max_dev_threshold
eval_output_list = build_GAP_output_format(
example_id_list,
gold_label_ids,
pred_prob_entail,
pred_label_ids_3way,
threshold,
dev_or_test='test')
test_acc = run_scorer(
'/export/home/Dataset/gap_coreference/gap-test.tsv',
eval_output_list)
if test_acc > max_test_acc:
max_test_acc = test_acc
print('current_test_acc:', test_acc, ' max_test_acc:',
max_test_acc)
final_test_performance = test_acc
print('final_test_performance:', final_test_performance)
def _row_normalize(count_graph,
diagonal_value=None,
normalized_by_softmax=False,
skip_zero_row=False):
"""
Normalize the count graph
Parameters
----------
count_graph
diagonal_value: float or None
normalized_by_softmax
skip_zero_row
Examples
--------
>>> _count_graph = [[0, 0, 0], [2, 8, 10], [82, 8, 0]]
>>> _row_normalize(_count_graph, diagonal_value=0, skip_zero_row=True)
[[0.0, 0.0, 0.0], [0.16666666666666666, 0.0, 0.8333333333333334], [0.9111111111111111, 0.08888888888888889, 0.0]]
>>> _count_graph = [[0, 0], [2, 8]]
>>> _row_normalize(_count_graph)
[[0.5, 0.5], [0.2, 0.8]]
>>> _count_graph = [[2, 3], [2, 8]]
>>> _row_normalize(_count_graph)
[[0.4, 0.6], [0.2, 0.8]]
>>> _count_graph = [[0, 0], [2, 8]]
>>> _row_normalize(_count_graph, normalized_by_softmax=True)
[[0.5, 0.5], [0.002472623156634775, 0.9975273768433656]]
>>> _count_graph = [[2, 3], [2, 8]]
>>> _row_normalize(_count_graph, normalized_by_softmax=True)
[[0.26894142136999505, 0.7310585786300048], [0.002472623156634775, 0.9975273768433656]]
>>> _count_graph = [[0, 0, 0], [2, 8, 10], [82, 8, 0]]
>>> _row_normalize(_count_graph, diagonal_value=0)
[[0.0, 0.5, 0.5], [0.16666666666666666, 0.0, 0.8333333333333334], [0.9111111111111111, 0.08888888888888889, 0.0]]
>>> _count_graph = [[0, 0, 0], [2, 8, 10], [82, 8, 0]]
>>> import numpy as np
>>> np.asarray(_row_normalize(_count_graph, normalized_by_softmax=True))
array([[3.33333333e-01, 3.33333333e-01, 3.33333333e-01],
[2.95387223e-04, 1.19167711e-01, 8.80536902e-01],
[1.00000000e+00, 7.28129018e-33, 2.44260074e-36]])
>>> np.asarray(_row_normalize(_count_graph, normalized_by_softmax=True, diagonal_value=0.0))
array([[0.00000000e+00, 5.00000000e-01, 5.00000000e-01],
[3.35350130e-04, 0.00000000e+00, 9.99664650e-01],
[1.00000000e+00, 7.28129018e-33, 0.00000000e+00]])
"""
_graph = np.asarray(count_graph)
zero_rows_indices = None
if skip_zero_row:
zero_rows_indices = np.sum(_graph, axis=-1) == 0
if normalized_by_softmax:
if diagonal_value is None:
_graph = softmax(_graph, axis=-1)
else:
_graph = _graph.astype(float)
np.fill_diagonal(_graph, -np.inf)
_graph = softmax(_graph, axis=-1)
if diagonal_value is not None:
np.fill_diagonal(_graph, diagonal_value)
else:
_offset_graph = _graph + 1e-50
if diagonal_value is not None:
np.fill_diagonal(_offset_graph, 0.0)
_graph = (_offset_graph.T / _offset_graph.sum(axis=-1)).T
_graph[_graph <= 1e-40] = 0.0
np.fill_diagonal(_graph, diagonal_value)
else:
_graph = (_offset_graph.T / _offset_graph.sum(axis=-1)).T
_graph[_graph <= 1e-40] = 0.0
if skip_zero_row:
_graph[zero_rows_indices] = 0.0
return _graph.tolist()