def _read(self, mode: str):
self._maybe_download_corpora(self._data_path)
filename = self.get_filename_by_mode(mode)
if filename:
file_path = os.path.join(self._data_path, filename)
with open(file_path, 'r') as snli_file:
logger.info("Reading SNLI instances from jsonl dataset at: %s",
file_path)
for line in snli_file:
fields = json.loads(line)
label = fields["gold_label"]
if label == '-':
# These were cases where the annotators disagreed; we'll just skip them. It's
# like 800 out of 500k examples in the training data.
continue
example = {
"premise": fields["sentence1"],
"hypothesis": fields["sentence2"],
"label": label
}
yield self._process(example)
else:
return None
def init_from_params(cls, params, **extras):
logger.info(f"instantiating class {cls} from params {getattr(params, 'params', params)} "
f"and extras {extras}")
if params is None:
return
if isinstance(params, str):
params = Parameters({"name": params})
try:
subclasses = register.get_by_name(cls)
except LookupError as e:
logger.debug(e)
subclasses = None
if subclasses is not None:
subclass_name = params.pop_choice("name", subclasses.keys())
subclass = subclasses[subclass_name]
if not takes_arg(subclass.init_from_params, 'extras'):
extras = {k: v for k, v in extras.items() if takes_arg(subclass.init_from_params, k)}
return subclass.init_from_params(params=params, **extras)
else:
if cls.__init__ == object.__init__:
kwargs = {}
else:
kwargs = create_kwargs(cls, params, **extras)
return cls(**kwargs)
def unzip(file_name, file_dir):
with zipfile.ZipFile(file_name) as archive:
if not all(
os.path.exists(os.path.join(file_dir, name))
for name in archive.namelist()):
logger.info("Extracting: " + file_name)
archive.extractall(file_dir)
def _read_pretrained_embeddings(pretrained_file, tmp_dir, embedding_dim, vocab,
vocab_namespace):
if not os.path.exists(pretrained_file):
logger.error("Pretrained embedding file is not existing")
return None
if tmp_dir:
if not os.path.exists(tmp_dir):
tf.gfile.MakeDirs(tmp_dir)
cache_embedding_file = os.path.join(tmp_dir, "embedding.pkl.gz")
else:
cache_embedding_file = None
if tmp_dir and os.path.exists(cache_embedding_file):
logger.info("loading cache embedding from %s." % cache_embedding_file)
with gzip.open(cache_embedding_file, 'rb') as pkl_file:
embeddings = pickle.load(pkl_file)
else:
file_ext = get_file_extension(pretrained_file)
if file_ext in ['.txt']:
embeddings = _read_pretrained_embeddings_text(
pretrained_file, embedding_dim, vocab, vocab_namespace)
else:
logger.error("Do not support this embedding file type.")
return None
if cache_embedding_file:
with gzip.open(cache_embedding_file, 'wb') as pkl_file:
pickle.dump(embeddings, pkl_file)
return embeddings
def init_from_file(cls, params_filename):
logger.info("loading config file from %s" % params_filename)
with open(params_filename, 'r', encoding='utf-8') as yaml_file:
params = yaml.load(yaml_file)
#cls.replace_special_names(params)
#cls.solve_conflict(params)
logger.info(params)
return cls(params)
def init_from_instances(cls, instances, vocab_init_files=None, pretrained_files=None, only_include_pretrained_words=False):
logger.info("create vocab from instance")
namespace_counter = collections.defaultdict(lambda: collections.defaultdict(int))
try:
for i, instance in enumerate(tqdm.tqdm(instances)):
instance.count_vocab(namespace_counter)
except StopIteration as e:
logger.error("The data reader builds vocabulary error with StopIteration.")
return cls(namespace_counter, pretrained_files=pretrained_files,
vocab_init_files=vocab_init_files,
only_include_pretrained_words=only_include_pretrained_words)
def _read_pretrained_embeddings_text(pretrained_file, embedding_dim, vocab,
vocab_namespace):
vocab_tokens = vocab.get_vocab_tokens(vocab_namespace)
vocab_size = vocab.get_vocab_size(vocab_namespace)
embeddings = {}
logger.info("Reading pretrained embeddings from: %s" % pretrained_file)
with open(pretrained_file, 'r', encoding='utf-8') as embeddings_file:
for line in tqdm.tqdm(embeddings_file):
token = line.split(" ", 1)[0]
if token in vocab_tokens:
fields = line.rstrip().split(' ')
if len(fields) - 1 != embedding_dim:
logger.warning(
"Found line with wrong number of dimensions (expected: %d; actual: %d): %s",
embedding_dim,
len(fields) - 1, line)
continue
vector = np.asarray(fields[1:], dtype='float32')
embeddings[token] = vector
if not embeddings:
ConfigureError(
"The embedding_dim or vocabulary does not fit the pretrained embedding."
)
all_embeddings = np.asarray(list(embeddings.values()))
embeddings_mean = float(np.mean(all_embeddings))
embeddings_std = float(np.std(all_embeddings))
embedding_matrix = np.random.normal(embeddings_mean, embeddings_std,
(vocab_size, embedding_dim))
embedding_matrix = embedding_matrix.astype(np.float32)
num_tokens_found = 0
index_to_tokens = vocab.get_vocab_index_to_token(vocab_namespace)
for i in range(vocab_size):
token = index_to_tokens[i]
if token in embeddings:
embedding_matrix[i] = embeddings[token]
num_tokens_found += 1
else:
logger.debug(
"Token %s was not found in the embedding file. Initialising randomly.",
token)
logger.info("Pretrained embeddings were found for %d out of %d tokens",
num_tokens_found, vocab_size)
return embedding_matrix
def extract_available_modes(saved_model_loader):
"""Return list of modes found in SavedModel."""
available_modes = []
logger.info('Checking available modes.')
for mode in [
tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL,
tf.estimator.ModeKeys.PREDICT
]:
try:
get_meta_graph_def_for_mode(saved_model_loader, mode)
except RuntimeError:
logger.warning('%s mode not found in SavedModel.' % mode)
continue
if get_signature_def_for_mode(saved_model_loader, mode) is not None:
available_modes.append(mode)
logger.info('Available modes: %s' % available_modes)
return available_modes
def model_fn(features, labels, mode, params):
logger.info("****Features****")
for name in sorted(features.keys()):
tf.logging.info("name = %s, shape = %s, data_split = %s" %
(name, features[name].shape, mode))
output_dict = embeddingmapping.forward(features, labels, mode,
params)
output_dict_new = {}
for key, value in output_dict.items():
output_dict_new[key + "_embedding"] = value
output_dict_new.update(features)
output_spec = tf.estimator.EstimatorSpec(
mode, predictions=output_dict_new)
return output_spec
def _read(self, mode: str):
filename = self.get_filename_by_mode(mode)
if filename:
file_path = os.path.join(self._data_path, filename)
if file_path.lower().endswith("jsonl"):
if self._field_mapping is None:
raise ConfigureError(
"field mapping is not provided for jsonl file.")
with open(file_path, 'r') as json_file:
logger.info("Reading instances from jsonl dataset at: %s",
file_path)
for line in json_file:
fields = json.loads(line)
example = {}
for (field_tar,
field_src) in self._field_mapping.items():
example[field_tar] = fields[field_src]
yield self._process(example)
# example = {}
# example['premise'] = fields['answer']
# example['hypothesis'] = fields['question']
# example['label'] = fields['label']
# yield self._process(example)
if file_path.lower().endswith("tsv"):
if self._field_mapping is None:
raise ConfigureError(
"field mapping is not provided for tsv file.")
with open(file_path, 'r') as csv_file:
logger.info("Reading instances from tsv dataset at: %s",
file_path)
for line in csv_file:
fields = line.strip().split("\t")
example = {}
for (field_tar,
field_src) in self._field_mapping.items():
example[field_tar] = fields[int(field_src)]
yield self._process(example)
else:
return None
def _read(self, mode: str):
self._maybe_download_corpora(self._data_path)
filename = self.get_filename_by_mode(mode)
if mode in [DataSplit.TRAIN, DataSplit.EVAL]:
dev_ids_filename = "mrpc_dev_ids.tsv"
dev_ids_path = os.path.join(self._data_path, dev_ids_filename)
dev_ids = []
with open(dev_ids_path, 'r', encoding="utf-8") as ids_fh:
for row in ids_fh:
dev_ids.append(row.strip().split('\t'))
if filename:
file_path = os.path.join(self._data_path, filename)
with open(file_path, 'r', encoding='utf-8') as mrpc_file:
logger.info(
"Reading Microsoft Research Paraphrase Corpus instances from txt dataset at: %s",
file_path)
for line in mrpc_file:
fields = line.strip().split("\t")
label, id1, id2, s1, s2 = fields
if label not in ['0', '1']:
#print(fields)
continue
if (mode == DataSplit.TRAIN and [id1, id2] not in dev_ids) \
or (mode == DataSplit.EVAL and [id1, id2] in dev_ids) \
or mode == DataSplit.PREDICT or mode == DataSplit.TEST:
if mode == DataSplit.TRAIN:
inputs = [[s1, s2], [s2, s1]]
else:
inputs = [[s1, s2]]
for inp in inputs:
example = {
"premise": inp[0],
"hypothesis": inp[1],
"label": label
}
yield self._process(example)
# else:
# print(mode, id1, id2, [id1, id2] in dev_ids)
else:
return None
def _read(self, mode: str):
self._maybe_download_corpora(self._data_path)
filenames = self.get_filename_by_mode(mode)
if filenames is None:
return None
if isinstance(filenames, str):
filenames = [filenames]
for filename in filenames:
logger.info("reading QatarLiving data from %s" % filename)
filepath = os.path.join(self._data_path, filename)
threads = self.parse(filepath)
for thread in threads:
q_id = thread['RelQuestion']['RELQ_ID']
q_category = thread['RelQuestion']['RELQ_CATEGORY']
q_subject = thread['RelQuestion']['RelQSubject']
q_text = thread['RelQuestion']['RelQBody']
comments = thread['RelComments']
assert len(comments) == 10
for comment in comments:
c_id = comment["RELC_ID"]
c_rel = comment['RELC_RELEVANCE2RELQ'].lower()
c_test = comment['RelCText']
if c_rel != "?":
if c_rel != 'good':
c_rel = 'bad'
example = {
"index": c_id,
"premise": q_text,
"hypothesis": c_test,
"label": c_rel
}
else:
example = {
"index": c_id,
"premise": q_text,
"hypothesis": c_test,
}
yield self._process(example)
def maybe_download(filepath, url):
"""Download filename from url unless it's already in directory.
Copies a remote file to local if that local file does not already exist. If
the local file pre-exists this function call, it does not check that the local
file is a copy of the remote.
Args:
directory: path to the directory that will be used.
filename: name of the file to download to (do nothing if it already exists).
url: URL to copy (or download) from.
Returns:
The path to the downloaded file.
"""
if os.path.exists(filepath):
logger.info("Not downloading, file already found: %s" % filepath)
return filepath
logger.info("Downloading %s to %s" % (url, filepath))
try:
tf.gfile.Copy(url, filepath)
except tf.errors.UnimplementedError:
try:
inprogress_filepath = filepath + ".incomplete"
# r = requests.get(url)
# with open(inprogress_filepath, 'wb') as outfile:
# outfile.write(r.content)
inprogress_filepath, _ = urlretrieve(
url, inprogress_filepath, reporthook=download_report_hook)
# Print newline to clear the carriage return from the download progress
print()
os.rename(inprogress_filepath, filepath)
except HTTPError:
if url.startswith("http"):
os.system('wget --no-check-certificat ' + url + " -O " +
filepath.replace(" ", "\ "))
else:
raise ValueError("Unrecognized URI: " + filepath)
statinfo = os.stat(filepath)
logger.info("Successfully downloaded %s, %s bytes." %
(os.path.basename(filepath), statinfo.st_size))
return filepath
def _read(self, mode: str):
self._maybe_download_corpora(self._data_path)
extra_info = {}
with open(os.path.join(self._data_path, "shared.jsonl"),
'r') as jsonl_file:
for line in jsonl_file:
fields = line.split("\t")
id = fields[0]
d = json.loads(fields[1])
extra_info[id] = d
filename = self.get_filename_by_mode(mode)
if filename:
file_path = os.path.join(self._data_path, filename)
with open(file_path, 'r') as mnli_file:
logger.info(
"Reading Multinli instances from jsonl dataset at: %s",
file_path)
for line in mnli_file:
fields = json.loads(line)
id = fields['pairID']
label = fields["gold_label"]
sent1 = fields["sentence1_binary_parse"]
sent2 = fields["sentence2_binary_parse"]
sent1 = re.sub(r'\(|\)', '', sent1)
sent2 = re.sub(r'\(|\)', '', sent2)
sent1 = sent1.replace(' ', '')
sent2 = sent2.replace(' ', '')
sent1_pos = fields['sentence1_parse']
sent2_pos = fields['sentence2_parse']
ant_feat_1 = extra_info[id]['sentence1_antonym_feature']
ant_feat_2 = extra_info[id]['sentence2_antonym_feature']
if len(ant_feat_1) or len(ant_feat_2):
print(ant_feat_1, ant_feat_2)
if label == '-':
# These were cases where the annotators disagreed; we'll just skip them. It's
# like 800 out of 500k examples in the training data.
continue
if mode in [DataSplit.TRAIN, DataSplit.EVAL]:
example = {
"id": id,
"premise": sent1,
"hypothesis": sent2,
"premise_pos": sent1_pos,
"hypothesis_pos": sent2_pos,
"label": label
}
else:
example = {
"id": id,
"premise": sent1,
"hypothesis": sent2,
"premise_pos": sent1_pos,
"hypothesis_pos": sent2_pos,
}
yield self._process(example)
else:
return None
def model_fn(features, labels, mode, params):
logger.info("****Features****")
for name in sorted(features.keys()):
tf.logging.info("name = %s, shape = %s, data_split = %s" % (name, features[name].shape, mode))
output_dict = self.forward(features, labels, mode, params)
########################################################################################
if not self._bool_init_checkpoint:
initialized_variable_names = {}
tvars = tf.trainable_variables()
if self._init_checkpoint:
(assignment_map, initialized_variable_names
) = get_assignment_map_from_checkpoint(tvars, self._init_checkpoint)
tf.train.init_from_checkpoint( self._init_checkpoint, assignment_map)
tf.logging.info("**** Trainable Variables ****")
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
self._bool_init_checkpoint = True
########################################################################################
if mode == tf.estimator.ModeKeys.TRAIN:
if 'loss' not in output_dict:
raise ModelError("Please provide loss in the model outputs for %s dataset." % mode)
train_op, optimizer_hooks = self._optimizer.get_train_op(output_dict['loss'], params)
# optimizer_hooks.append(tf_debug.LocalCLIDebugHook())
##########
if 'debugs' in output_dict:
tvars = output_dict['debugs'] # tf.trainable_variables()
print_ops = []
for op in tvars:
op_name = op.name
# op = tf.debugging.is_nan(tf.reduce_mean(op))
print_ops.append(tf.print(op.name, op, output_stream=sys.stdout))
print_op = tf.group(*print_ops)
train_op = tf.group(print_op, train_op)
# ########
# tvars = tf.trainable_variables()
# tgrads = tf.gradients(output_dict['loss'], tvars)
# lambda_est = []
# for grad, var in zip(tgrads, tvars):
# tmp = tf.reduce_mean(tf.maximum(-grad*var / 2 * var ** 2, 0))
# lambda_est.append(tmp)
# lambda_est = tf.stack(lambda_est, axis=0)
# lambda_est = tf.reduce_mean(lambda_est)
# lambda_mean, lambda_update = tf.metrics.mean(lambda_est)
# print_op = tf.print("l2 weight decay lambda value", lambda_mean, output_stream=sys.stdout)
# train_op = tf.group(train_op, print_op, lambda_update)
# #########
# ########
output_spec = tf.estimator.EstimatorSpec(mode, loss=output_dict['loss'], train_op=train_op,
export_outputs=output_dict.get("export_outputs", None),
predictions=output_dict.get('predictions', None),
training_hooks=optimizer_hooks,
eval_metric_ops=output_dict.get('metrics', None))
elif mode == tf.estimator.ModeKeys.EVAL:
output_spec = tf.estimator.EstimatorSpec(mode, export_outputs=output_dict.get("export_outputs", None),
loss=output_dict.get('loss', None),
predictions=output_dict.get('predictions', None),
eval_metric_ops=output_dict.get('metrics', None))
elif mode == tf.estimator.ModeKeys.PREDICT:
output_spec = tf.estimator.EstimatorSpec(mode, export_outputs=output_dict.get("export_outputs", None),
predictions=output_dict.get('predictions', None))
else:
raise ValueError("Mode %s are not supported." % mode)
return output_spec
def __init__(self,
data_reader=None,
eval_input_fn=None,
num_classes=None,
vocab=None,
export_dir=None,
output_file=None,
hparams=HParams()):
if data_reader is not None and eval_input_fn is None:
self._eval_input_fn = data_reader.make_estimator_input_fn(
DataSplit.EVAL, force_repeat=False)
vocab = data_reader.get_vocab()
else:
self._eval_input_fn = eval_input_fn
if num_classes is None:
num_classes = vocab.get_vocab_size(namespace='labels')
task = hparams.get('task', 'classification')
task_type = hparams.get('task_type', 'multiclass')
labels = list(range(num_classes))
dataset = self._eval_input_fn()
iterator = dataset.make_initializable_iterator()
dataset.make_initializable_iterator()
next_element = iterator.get_next()
self.saved_model_loader = loader_impl.SavedModelLoader(export_dir)
mode = DataSplit.PREDICT
signature_def = get_signature_def_for_mode(self.saved_model_loader,
mode)
input_map = generate_input_map(signature_def, next_element)
output_tensor_names = [
value.name for value in signature_def.outputs.values()
]
try:
tags = model_fn.EXPORT_TAG_MAP[mode]
except AttributeError as e:
tags = ['serve']
saver, output_tensors = self.saved_model_loader.load_graph(
tf.get_default_graph(),
tags,
input_map=input_map,
return_elements=output_tensor_names)
output_map = dict(zip(output_tensor_names, output_tensors))
outputs = {
key: output_map[value.name]
for (key, value) in signature_def.outputs.items()
}
# predict_fn = tf.contrib.predictor.from_saved_model(export_dir)
#####xlsx wirte######
tsv_file = open(output_file, 'w')
# wb = Workbook(write_only=True)
# ws = wb.create_sheet('examples')
# ws.append(['question', 'answer', 'true_label', 'predict', 'score'])
y_true = []
y_pred = []
total_num = 0
# accuracy = 0
# confusion_matrix = [[0 for j in range(num_classes)] for i in range(num_classes)]
if hparams.per_process_gpu_memory_fraction is not None and 0 < hparams.per_process_gpu_memory_fraction <= 1:
session_config = tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True)
session_config.gpu_options.per_process_gpu_memory_fraction = hparams.per_process_gpu_memory_fraction
else:
session_config = tf.ConfigProto()
with tf.Session(config=session_config) as sess:
self.saved_model_loader.restore_variables(sess, saver)
self.saved_model_loader.run_init_ops(sess, tags)
sess.run(iterator.initializer)
while True:
try:
outputs['inputs'] = next_element
output_vals = sess.run(outputs)
data_batch = output_vals['inputs']
if 'premise/tokens' in data_batch.keys(
) and 'hypothesis/tokens' in data_batch.keys():
premise_tokens_val, hypothesis_tokens_val, true_label_val = \
data_batch['premise/tokens'], data_batch['hypothesis/tokens'], data_batch['label/labels']
else:
true_label_val = data_batch['label/labels']
premise_tokens_val = [
[] for i in range(len(true_label_val))
]
hypothesis_tokens_val = [
[] for i in range(len(true_label_val))
]
# probs = output_vals['output_score']
probs = output_vals['output']
num_batch = probs.shape[0]
total_num += num_batch
print("processing %s/%s" % (num_batch, total_num))
#######################
# print(probs)
if task_type == 'multiclass':
predictions_val = np.argmax(probs, axis=1)
elif task_type == 'multilabel':
threshold = hparams.get('threshold', 0.5)
predictions_val = (probs > threshold).astype(
dtype=np.int32)
elif task_type == 'topk':
predictions_val = (probs > 0).astype(dtype=np.int32)
else:
raise ConfigureError(
"Task type %s is not support for task %s. "
"Only multiclass and multilabel is support for task %s"
% (task_type, task, task))
# predictions = (probs > 0.5).astype(np.int32)
# print(predictions)
y_true.append(true_label_val)
y_pred.append(predictions_val)
# print(predictions)
# for i in range(probs.shape[0]):
# predictions = (probs > 0.5).astype(np.int32)
# predict = predictions[i]
# label = true_label_val[i]
# if predict == label:
# accuracy += 1
# confusion_matrix[label][predict] += 1
################
for i in range(num_batch):
premise_str = vocab.convert_indexes_to_tokens(
premise_tokens_val[i], 'tokens')
premise_str = " ".join(premise_str)
hypothesis_str = vocab.convert_indexes_to_tokens(
hypothesis_tokens_val[i], 'tokens')
hypothesis_str = " ".join(hypothesis_str)
if task_type == 'multilabel' or task_type == 'topk':
predictions = [[] for i in range(num_batch)]
for (row,
col) in np.argwhere(predictions_val == 1):
predictions[row].append(col)
true_labels = [[] for i in range(num_batch)]
for row, col in np.argwhere(true_label_val == 1):
true_labels[row].append(col)
else:
predictions = predictions_val
true_labels = true_label_val
true_label = true_labels[i]
predict = predictions[i]
prob = probs[i]
if task_type == 'multiclass':
tsv_str = "\t".join([
premise_str, hypothesis_str,
vocab.get_index_token(true_label,
namespace='labels'),
vocab.get_index_token(predict,
namespace='labels'),
str(prob)
])
elif task_type == 'multilabel' or task_type == 'topk':
tsv_str = "\t".join([
premise_str, hypothesis_str, " ".join([
vocab.get_index_token(l,
namespace='labels')
for l in true_label
]), " ".join([
vocab.get_index_token(p,
namespace='labels')
for p in predict
]),
str(prob)
])
else:
raise ConfigureError(
"Task type %s is not support for task %s. "
"Only multiclass and multilabel is support for task %s"
% (task_type, task, task))
# tsv_str = "\t".join([premise_str, hypothesis_str, str(true_label), str(predict), str(prob),
# json.dumps(output_vals['query_embedding'][i].tolist()), json.dumps(output_vals['title_embedding'][i].tolist()),
# json.dumps(output_vals['query_lstm_1'][i].tolist()), json.dumps(output_vals['title_lstm_1'][i].tolist()),
# json.dumps(output_vals['query_attention'][i].tolist()), json.dumps(output_vals['title_attention'][i].tolist()),
# json.dumps(output_vals['query_lstm_2'][i].tolist()), json.dumps(output_vals['title_lstm_2'][i].tolist()),
# json.dumps(output_vals['fc1'][i].tolist()), json.dumps(output_vals['fc2'][i].tolist())
# ])
tsv_file.write(tsv_str + "\n")
# print("process %s/%s correct/total instances with accuracy %s." % (accuracy, total_num, accuracy/float(total_num)))
except tf.errors.OutOfRangeError as e:
logger.info("processed all the evalutation data")
break
# logger.warning(e)
y_true = np.concatenate(y_true, axis=0)
y_pred = np.concatenate(y_pred, axis=0)
avg_param = 'micro'
if num_classes == 2:
avg_param = 'binary'
accuracy = metrics.accuracy_score(y_true,
y_pred) # accuracy/total_num
precise, recall, f1score, support = metrics.precision_recall_fscore_support(
y_true, y_pred, labels=labels, average=avg_param)
if task_type == 'multiclass':
confusion_matrix = metrics.confusion_matrix(y_true,
y_pred,
labels=labels)
print("metrics:")
confmx_str = "label \ predict "
for i in range(num_classes):
confmx_str += "| %s | " % vocab.get_index_token(
i, namespace='labels')
confmx_str += "\n"
for i in range(num_classes):
confmx_str += "| %s | " % vocab.get_index_token(
i, namespace='labels')
for j in range(num_classes):
confmx_str += "| %s | " % confusion_matrix[i][j]
confmx_str += "\n"
print(confmx_str)
elif task_type == 'multilabel' or task_type == 'topk':
confusion_matrix = metrics.multilabel_confusion_matrix(
y_true, y_pred)
print("metrics:")
for k in range(num_classes):
print("confusion matrix for label %s" %
vocab.get_index_token(k, namespace='labels'))
confmx_str = "label \ predict "
for i in range(2):
confmx_str += "| %s | " % i
confmx_str += "\n"
for i in range(2):
confmx_str += "| %s | " % i
for j in range(2):
confmx_str += "| %s | " % confusion_matrix[k][i][j]
confmx_str += "\n"
print(confmx_str)
else:
raise ConfigureError(
"Task type %s is not support for task %s. "
"Only multiclass and multilabel is support for task %s" %
(task_type, task, task))
# confusion_matrix[1][1]/(confusion_matrix[0][1]+confusion_matrix[1][1])
# recall = confusion_matrix[1][1]/(confusion_matrix[1][0]+confusion_matrix[1][1])
# f1score = (precise+recall)/2
print("micro total accuracy precise recall f1-score")
print(
"accuracy: %.2f, precise: %.2f, recall: %.2f, f1-score: %.2f" %
(accuracy, precise, recall, f1score))
precisions, recalls, fbeta_scores, supports = metrics.precision_recall_fscore_support(
y_true, y_pred, labels=labels)
print("accuracy precise recall f1-score for each class")
print(
'======================================================================================'
)
for lab_idx, (precision, recall, fbeta_score,
support) in enumerate(
zip(precisions, recalls, fbeta_scores,
supports)):
print(
"label:%s\tprecision:%.2f\trecall:%.2f\tf1-score:%.2f\tsupport:%.2f"
% (vocab.get_index_token(lab_idx, namespace='labels'),
precision, recall, fbeta_score, support))
# legend = ["label \ predict "]
# for i in range(num_classes):
# legend.append(str(i))
# ws.append(legend)
# for i in range(num_classes):
# row = [str(i)]
# for j in range(num_classes):
# row.append(str(confusion_matrix[i][j]))
# ws.append(row)
# ws.append([])
# ws.append([])
# ws.append(['accuracy', 'precise', 'recall', 'f1-score'])
# ws.append([str(accuracy), str(precise), str(recall), str(f1score)])
# if output_file:
# if not output_file.endswith(".xlsx"):
# output_file += '.xlsx'
# wb.save(output_file)
tsv_file.close()
def __init__(self,
data_reader=None,
pred_input_fn=None,
vocab=None,
export_dir=None,
output_file=None):
if data_reader is not None and pred_input_fn is None:
self._pred_input_fn = data_reader.make_estimator_input_fn(
DataSplit.PREDICT, force_repeat=False)
vocab = data_reader.get_vocab()
else:
self._pred_input_fn = pred_input_fn
dataset = self._pred_input_fn()
iterator = dataset.make_initializable_iterator()
dataset.make_initializable_iterator()
next_element = iterator.get_next()
self.saved_model_loader = loader_impl.SavedModelLoader(export_dir)
mode = DataSplit.PREDICT
signature_def = get_signature_def_for_mode(self.saved_model_loader,
mode)
input_map = self.generate_input_map(signature_def, next_element)
output_tensor_names = [
value.name for value in signature_def.outputs.values()
]
try:
tags = model_fn.EXPORT_TAG_MAP[mode]
except AttributeError as e:
tags = ['serve']
saver, output_tensors = self.saved_model_loader.load_graph(
tf.get_default_graph(),
tags,
input_map=input_map,
return_elements=output_tensor_names)
output_map = dict(zip(output_tensor_names, output_tensors))
outputs = {
key: output_map[value.name]
for (key, value) in signature_def.outputs.items()
}
#####xlsx wirte######
# wb = Workbook(write_only=True)
# ws = wb.create_sheet('examples')
# ws.append(['index', 'question', 'answer', 'predict', 'score'])
csv_file = open(output_file, 'w', encoding="utf-8")
csv_file.write(
"\t".join(["index", 'premise', 'hypothesis', "prediction"]) + "\n")
#csv_file.write("\t".join(["index", "premise", "hypothesis", "prediction", "prob"])+"\n")
total_num = 0
#accuracy = 0
#confusion_matrix = [[0 for j in range(num_classes)] for i in range(num_classes)]
with tf.Session() as sess:
self.saved_model_loader.restore_variables(sess, saver)
self.saved_model_loader.run_init_ops(sess, tags)
sess.run(iterator.initializer)
while True:
try:
outputs['inputs'] = next_element
output_vals = sess.run(outputs)
data_batch = output_vals['inputs']
if "index/index" in data_batch:
index_val, premise_tokens_val, hypothesis_tokens_val = \
data_batch['index/index'], data_batch['premise/tokens'], data_batch['hypothesis/tokens']
indexs = [str(index, 'utf-8') for index in index_val]
else:
premise_tokens_val, hypothesis_tokens_val = \
data_batch['premise/tokens'], data_batch['hypothesis/tokens']
index_val = list(
range(total_num,
total_num + hypothesis_tokens_val.shape[0]))
indexs = [str(index) for index in index_val]
probs = output_vals['output']
num_batch = probs.shape[0]
#######################
predictions = np.argmax(probs, axis=1)
#predictions = (probs <= 0.5).astype(np.int32)
total_num += num_batch
logger.info("processing %s/%s" % (num_batch, total_num))
# for i in range(probs.shape[0]):
# predictions = (probs > 0.5).astype(np.int32)
# predict = predictions[i]
# label = true_label_val[i]
# if predict == label:
# accuracy += 1
# confusion_matrix[label][predict] += 1
################
for i in range(num_batch):
premise_str = vocab.convert_indexes_to_tokens(
premise_tokens_val[i], 'tokens')
premise_str = " ".join(premise_str)
hypothesis_str = vocab.convert_indexes_to_tokens(
hypothesis_tokens_val[i], 'tokens')
hypothesis_str = " ".join(hypothesis_str)
predict = predictions[i]
prob = probs[i]
index = indexs[i]
#csv_str = "\t".join([index, premise_str, hypothesis_str, str(predict), str(prob)])+"\n"
if hypothesis_str.strip() != "" and premise_str.strip(
) != "" and hypothesis_str.strip(
) != premise_str.strip():
csv_str = "\t".join([
index, premise_str, hypothesis_str,
str(predict)
]) + "\n"
csv_file.write(csv_str)
#ws.append([index, premise_str, hypothesis_str, str(predict), str(prob)])
#print("process %s/%s correct/total instances with accuracy %s." % (accuracy, total_num, accuracy/float(total_num)))
except tf.errors.OutOfRangeError as e:
# if output_file:
# if not output_file.endswith(".xlsx"):
# output_file += '.xlsx'
# wb.save(output_file)
csv_file.close()
break
