def _read(self, mode: str): self._maybe_download_corpora(self._data_path) filename = self.get_filename_by_mode(mode) if filename: file_path = os.path.join(self._data_path, filename) with open(file_path, 'r') as snli_file: logger.info("Reading SNLI instances from jsonl dataset at: %s", file_path) for line in snli_file: fields = json.loads(line) label = fields["gold_label"] if label == '-': # These were cases where the annotators disagreed; we'll just skip them. It's # like 800 out of 500k examples in the training data. continue example = { "premise": fields["sentence1"], "hypothesis": fields["sentence2"], "label": label } yield self._process(example) else: return None
def init_from_params(cls, params, **extras): logger.info(f"instantiating class {cls} from params {getattr(params, 'params', params)} " f"and extras {extras}") if params is None: return if isinstance(params, str): params = Parameters({"name": params}) try: subclasses = register.get_by_name(cls) except LookupError as e: logger.debug(e) subclasses = None if subclasses is not None: subclass_name = params.pop_choice("name", subclasses.keys()) subclass = subclasses[subclass_name] if not takes_arg(subclass.init_from_params, 'extras'): extras = {k: v for k, v in extras.items() if takes_arg(subclass.init_from_params, k)} return subclass.init_from_params(params=params, **extras) else: if cls.__init__ == object.__init__: kwargs = {} else: kwargs = create_kwargs(cls, params, **extras) return cls(**kwargs)
def unzip(file_name, file_dir): with zipfile.ZipFile(file_name) as archive: if not all( os.path.exists(os.path.join(file_dir, name)) for name in archive.namelist()): logger.info("Extracting: " + file_name) archive.extractall(file_dir)
def _read_pretrained_embeddings(pretrained_file, tmp_dir, embedding_dim, vocab, vocab_namespace): if not os.path.exists(pretrained_file): logger.error("Pretrained embedding file is not existing") return None if tmp_dir: if not os.path.exists(tmp_dir): tf.gfile.MakeDirs(tmp_dir) cache_embedding_file = os.path.join(tmp_dir, "embedding.pkl.gz") else: cache_embedding_file = None if tmp_dir and os.path.exists(cache_embedding_file): logger.info("loading cache embedding from %s." % cache_embedding_file) with gzip.open(cache_embedding_file, 'rb') as pkl_file: embeddings = pickle.load(pkl_file) else: file_ext = get_file_extension(pretrained_file) if file_ext in ['.txt']: embeddings = _read_pretrained_embeddings_text( pretrained_file, embedding_dim, vocab, vocab_namespace) else: logger.error("Do not support this embedding file type.") return None if cache_embedding_file: with gzip.open(cache_embedding_file, 'wb') as pkl_file: pickle.dump(embeddings, pkl_file) return embeddings
def init_from_file(cls, params_filename): logger.info("loading config file from %s" % params_filename) with open(params_filename, 'r', encoding='utf-8') as yaml_file: params = yaml.load(yaml_file) #cls.replace_special_names(params) #cls.solve_conflict(params) logger.info(params) return cls(params)
def init_from_instances(cls, instances, vocab_init_files=None, pretrained_files=None, only_include_pretrained_words=False): logger.info("create vocab from instance") namespace_counter = collections.defaultdict(lambda: collections.defaultdict(int)) try: for i, instance in enumerate(tqdm.tqdm(instances)): instance.count_vocab(namespace_counter) except StopIteration as e: logger.error("The data reader builds vocabulary error with StopIteration.") return cls(namespace_counter, pretrained_files=pretrained_files, vocab_init_files=vocab_init_files, only_include_pretrained_words=only_include_pretrained_words)
def _read_pretrained_embeddings_text(pretrained_file, embedding_dim, vocab, vocab_namespace): vocab_tokens = vocab.get_vocab_tokens(vocab_namespace) vocab_size = vocab.get_vocab_size(vocab_namespace) embeddings = {} logger.info("Reading pretrained embeddings from: %s" % pretrained_file) with open(pretrained_file, 'r', encoding='utf-8') as embeddings_file: for line in tqdm.tqdm(embeddings_file): token = line.split(" ", 1)[0] if token in vocab_tokens: fields = line.rstrip().split(' ') if len(fields) - 1 != embedding_dim: logger.warning( "Found line with wrong number of dimensions (expected: %d; actual: %d): %s", embedding_dim, len(fields) - 1, line) continue vector = np.asarray(fields[1:], dtype='float32') embeddings[token] = vector if not embeddings: ConfigureError( "The embedding_dim or vocabulary does not fit the pretrained embedding." ) all_embeddings = np.asarray(list(embeddings.values())) embeddings_mean = float(np.mean(all_embeddings)) embeddings_std = float(np.std(all_embeddings)) embedding_matrix = np.random.normal(embeddings_mean, embeddings_std, (vocab_size, embedding_dim)) embedding_matrix = embedding_matrix.astype(np.float32) num_tokens_found = 0 index_to_tokens = vocab.get_vocab_index_to_token(vocab_namespace) for i in range(vocab_size): token = index_to_tokens[i] if token in embeddings: embedding_matrix[i] = embeddings[token] num_tokens_found += 1 else: logger.debug( "Token %s was not found in the embedding file. Initialising randomly.", token) logger.info("Pretrained embeddings were found for %d out of %d tokens", num_tokens_found, vocab_size) return embedding_matrix
def extract_available_modes(saved_model_loader): """Return list of modes found in SavedModel.""" available_modes = [] logger.info('Checking available modes.') for mode in [ tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, tf.estimator.ModeKeys.PREDICT ]: try: get_meta_graph_def_for_mode(saved_model_loader, mode) except RuntimeError: logger.warning('%s mode not found in SavedModel.' % mode) continue if get_signature_def_for_mode(saved_model_loader, mode) is not None: available_modes.append(mode) logger.info('Available modes: %s' % available_modes) return available_modes
def model_fn(features, labels, mode, params): logger.info("****Features****") for name in sorted(features.keys()): tf.logging.info("name = %s, shape = %s, data_split = %s" % (name, features[name].shape, mode)) output_dict = embeddingmapping.forward(features, labels, mode, params) output_dict_new = {} for key, value in output_dict.items(): output_dict_new[key + "_embedding"] = value output_dict_new.update(features) output_spec = tf.estimator.EstimatorSpec( mode, predictions=output_dict_new) return output_spec
def _read(self, mode: str): filename = self.get_filename_by_mode(mode) if filename: file_path = os.path.join(self._data_path, filename) if file_path.lower().endswith("jsonl"): if self._field_mapping is None: raise ConfigureError( "field mapping is not provided for jsonl file.") with open(file_path, 'r') as json_file: logger.info("Reading instances from jsonl dataset at: %s", file_path) for line in json_file: fields = json.loads(line) example = {} for (field_tar, field_src) in self._field_mapping.items(): example[field_tar] = fields[field_src] yield self._process(example) # example = {} # example['premise'] = fields['answer'] # example['hypothesis'] = fields['question'] # example['label'] = fields['label'] # yield self._process(example) if file_path.lower().endswith("tsv"): if self._field_mapping is None: raise ConfigureError( "field mapping is not provided for tsv file.") with open(file_path, 'r') as csv_file: logger.info("Reading instances from tsv dataset at: %s", file_path) for line in csv_file: fields = line.strip().split("\t") example = {} for (field_tar, field_src) in self._field_mapping.items(): example[field_tar] = fields[int(field_src)] yield self._process(example) else: return None
def _read(self, mode: str): self._maybe_download_corpora(self._data_path) filename = self.get_filename_by_mode(mode) if mode in [DataSplit.TRAIN, DataSplit.EVAL]: dev_ids_filename = "mrpc_dev_ids.tsv" dev_ids_path = os.path.join(self._data_path, dev_ids_filename) dev_ids = [] with open(dev_ids_path, 'r', encoding="utf-8") as ids_fh: for row in ids_fh: dev_ids.append(row.strip().split('\t')) if filename: file_path = os.path.join(self._data_path, filename) with open(file_path, 'r', encoding='utf-8') as mrpc_file: logger.info( "Reading Microsoft Research Paraphrase Corpus instances from txt dataset at: %s", file_path) for line in mrpc_file: fields = line.strip().split("\t") label, id1, id2, s1, s2 = fields if label not in ['0', '1']: #print(fields) continue if (mode == DataSplit.TRAIN and [id1, id2] not in dev_ids) \ or (mode == DataSplit.EVAL and [id1, id2] in dev_ids) \ or mode == DataSplit.PREDICT or mode == DataSplit.TEST: if mode == DataSplit.TRAIN: inputs = [[s1, s2], [s2, s1]] else: inputs = [[s1, s2]] for inp in inputs: example = { "premise": inp[0], "hypothesis": inp[1], "label": label } yield self._process(example) # else: # print(mode, id1, id2, [id1, id2] in dev_ids) else: return None
def _read(self, mode: str): self._maybe_download_corpora(self._data_path) filenames = self.get_filename_by_mode(mode) if filenames is None: return None if isinstance(filenames, str): filenames = [filenames] for filename in filenames: logger.info("reading QatarLiving data from %s" % filename) filepath = os.path.join(self._data_path, filename) threads = self.parse(filepath) for thread in threads: q_id = thread['RelQuestion']['RELQ_ID'] q_category = thread['RelQuestion']['RELQ_CATEGORY'] q_subject = thread['RelQuestion']['RelQSubject'] q_text = thread['RelQuestion']['RelQBody'] comments = thread['RelComments'] assert len(comments) == 10 for comment in comments: c_id = comment["RELC_ID"] c_rel = comment['RELC_RELEVANCE2RELQ'].lower() c_test = comment['RelCText'] if c_rel != "?": if c_rel != 'good': c_rel = 'bad' example = { "index": c_id, "premise": q_text, "hypothesis": c_test, "label": c_rel } else: example = { "index": c_id, "premise": q_text, "hypothesis": c_test, } yield self._process(example)
def maybe_download(filepath, url): """Download filename from url unless it's already in directory. Copies a remote file to local if that local file does not already exist. If the local file pre-exists this function call, it does not check that the local file is a copy of the remote. Args: directory: path to the directory that will be used. filename: name of the file to download to (do nothing if it already exists). url: URL to copy (or download) from. Returns: The path to the downloaded file. """ if os.path.exists(filepath): logger.info("Not downloading, file already found: %s" % filepath) return filepath logger.info("Downloading %s to %s" % (url, filepath)) try: tf.gfile.Copy(url, filepath) except tf.errors.UnimplementedError: try: inprogress_filepath = filepath + ".incomplete" # r = requests.get(url) # with open(inprogress_filepath, 'wb') as outfile: # outfile.write(r.content) inprogress_filepath, _ = urlretrieve( url, inprogress_filepath, reporthook=download_report_hook) # Print newline to clear the carriage return from the download progress print() os.rename(inprogress_filepath, filepath) except HTTPError: if url.startswith("http"): os.system('wget --no-check-certificat ' + url + " -O " + filepath.replace(" ", "\ ")) else: raise ValueError("Unrecognized URI: " + filepath) statinfo = os.stat(filepath) logger.info("Successfully downloaded %s, %s bytes." % (os.path.basename(filepath), statinfo.st_size)) return filepath
def _read(self, mode: str): self._maybe_download_corpora(self._data_path) extra_info = {} with open(os.path.join(self._data_path, "shared.jsonl"), 'r') as jsonl_file: for line in jsonl_file: fields = line.split("\t") id = fields[0] d = json.loads(fields[1]) extra_info[id] = d filename = self.get_filename_by_mode(mode) if filename: file_path = os.path.join(self._data_path, filename) with open(file_path, 'r') as mnli_file: logger.info( "Reading Multinli instances from jsonl dataset at: %s", file_path) for line in mnli_file: fields = json.loads(line) id = fields['pairID'] label = fields["gold_label"] sent1 = fields["sentence1_binary_parse"] sent2 = fields["sentence2_binary_parse"] sent1 = re.sub(r'\(|\)', '', sent1) sent2 = re.sub(r'\(|\)', '', sent2) sent1 = sent1.replace(' ', '') sent2 = sent2.replace(' ', '') sent1_pos = fields['sentence1_parse'] sent2_pos = fields['sentence2_parse'] ant_feat_1 = extra_info[id]['sentence1_antonym_feature'] ant_feat_2 = extra_info[id]['sentence2_antonym_feature'] if len(ant_feat_1) or len(ant_feat_2): print(ant_feat_1, ant_feat_2) if label == '-': # These were cases where the annotators disagreed; we'll just skip them. It's # like 800 out of 500k examples in the training data. continue if mode in [DataSplit.TRAIN, DataSplit.EVAL]: example = { "id": id, "premise": sent1, "hypothesis": sent2, "premise_pos": sent1_pos, "hypothesis_pos": sent2_pos, "label": label } else: example = { "id": id, "premise": sent1, "hypothesis": sent2, "premise_pos": sent1_pos, "hypothesis_pos": sent2_pos, } yield self._process(example) else: return None
def model_fn(features, labels, mode, params): logger.info("****Features****") for name in sorted(features.keys()): tf.logging.info("name = %s, shape = %s, data_split = %s" % (name, features[name].shape, mode)) output_dict = self.forward(features, labels, mode, params) ######################################################################################## if not self._bool_init_checkpoint: initialized_variable_names = {} tvars = tf.trainable_variables() if self._init_checkpoint: (assignment_map, initialized_variable_names ) = get_assignment_map_from_checkpoint(tvars, self._init_checkpoint) tf.train.init_from_checkpoint( self._init_checkpoint, assignment_map) tf.logging.info("**** Trainable Variables ****") for var in tvars: init_string = "" if var.name in initialized_variable_names: init_string = ", *INIT_FROM_CKPT*" tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, init_string) self._bool_init_checkpoint = True ######################################################################################## if mode == tf.estimator.ModeKeys.TRAIN: if 'loss' not in output_dict: raise ModelError("Please provide loss in the model outputs for %s dataset." % mode) train_op, optimizer_hooks = self._optimizer.get_train_op(output_dict['loss'], params) # optimizer_hooks.append(tf_debug.LocalCLIDebugHook()) ########## if 'debugs' in output_dict: tvars = output_dict['debugs'] # tf.trainable_variables() print_ops = [] for op in tvars: op_name = op.name # op = tf.debugging.is_nan(tf.reduce_mean(op)) print_ops.append(tf.print(op.name, op, output_stream=sys.stdout)) print_op = tf.group(*print_ops) train_op = tf.group(print_op, train_op) # ######## # tvars = tf.trainable_variables() # tgrads = tf.gradients(output_dict['loss'], tvars) # lambda_est = [] # for grad, var in zip(tgrads, tvars): # tmp = tf.reduce_mean(tf.maximum(-grad*var / 2 * var ** 2, 0)) # lambda_est.append(tmp) # lambda_est = tf.stack(lambda_est, axis=0) # lambda_est = tf.reduce_mean(lambda_est) # lambda_mean, lambda_update = tf.metrics.mean(lambda_est) # print_op = tf.print("l2 weight decay lambda value", lambda_mean, output_stream=sys.stdout) # train_op = tf.group(train_op, print_op, lambda_update) # ######### # ######## output_spec = tf.estimator.EstimatorSpec(mode, loss=output_dict['loss'], train_op=train_op, export_outputs=output_dict.get("export_outputs", None), predictions=output_dict.get('predictions', None), training_hooks=optimizer_hooks, eval_metric_ops=output_dict.get('metrics', None)) elif mode == tf.estimator.ModeKeys.EVAL: output_spec = tf.estimator.EstimatorSpec(mode, export_outputs=output_dict.get("export_outputs", None), loss=output_dict.get('loss', None), predictions=output_dict.get('predictions', None), eval_metric_ops=output_dict.get('metrics', None)) elif mode == tf.estimator.ModeKeys.PREDICT: output_spec = tf.estimator.EstimatorSpec(mode, export_outputs=output_dict.get("export_outputs", None), predictions=output_dict.get('predictions', None)) else: raise ValueError("Mode %s are not supported." % mode) return output_spec
def __init__(self, data_reader=None, eval_input_fn=None, num_classes=None, vocab=None, export_dir=None, output_file=None, hparams=HParams()): if data_reader is not None and eval_input_fn is None: self._eval_input_fn = data_reader.make_estimator_input_fn( DataSplit.EVAL, force_repeat=False) vocab = data_reader.get_vocab() else: self._eval_input_fn = eval_input_fn if num_classes is None: num_classes = vocab.get_vocab_size(namespace='labels') task = hparams.get('task', 'classification') task_type = hparams.get('task_type', 'multiclass') labels = list(range(num_classes)) dataset = self._eval_input_fn() iterator = dataset.make_initializable_iterator() dataset.make_initializable_iterator() next_element = iterator.get_next() self.saved_model_loader = loader_impl.SavedModelLoader(export_dir) mode = DataSplit.PREDICT signature_def = get_signature_def_for_mode(self.saved_model_loader, mode) input_map = generate_input_map(signature_def, next_element) output_tensor_names = [ value.name for value in signature_def.outputs.values() ] try: tags = model_fn.EXPORT_TAG_MAP[mode] except AttributeError as e: tags = ['serve'] saver, output_tensors = self.saved_model_loader.load_graph( tf.get_default_graph(), tags, input_map=input_map, return_elements=output_tensor_names) output_map = dict(zip(output_tensor_names, output_tensors)) outputs = { key: output_map[value.name] for (key, value) in signature_def.outputs.items() } # predict_fn = tf.contrib.predictor.from_saved_model(export_dir) #####xlsx wirte###### tsv_file = open(output_file, 'w') # wb = Workbook(write_only=True) # ws = wb.create_sheet('examples') # ws.append(['question', 'answer', 'true_label', 'predict', 'score']) y_true = [] y_pred = [] total_num = 0 # accuracy = 0 # confusion_matrix = [[0 for j in range(num_classes)] for i in range(num_classes)] if hparams.per_process_gpu_memory_fraction is not None and 0 < hparams.per_process_gpu_memory_fraction <= 1: session_config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True) session_config.gpu_options.per_process_gpu_memory_fraction = hparams.per_process_gpu_memory_fraction else: session_config = tf.ConfigProto() with tf.Session(config=session_config) as sess: self.saved_model_loader.restore_variables(sess, saver) self.saved_model_loader.run_init_ops(sess, tags) sess.run(iterator.initializer) while True: try: outputs['inputs'] = next_element output_vals = sess.run(outputs) data_batch = output_vals['inputs'] if 'premise/tokens' in data_batch.keys( ) and 'hypothesis/tokens' in data_batch.keys(): premise_tokens_val, hypothesis_tokens_val, true_label_val = \ data_batch['premise/tokens'], data_batch['hypothesis/tokens'], data_batch['label/labels'] else: true_label_val = data_batch['label/labels'] premise_tokens_val = [ [] for i in range(len(true_label_val)) ] hypothesis_tokens_val = [ [] for i in range(len(true_label_val)) ] # probs = output_vals['output_score'] probs = output_vals['output'] num_batch = probs.shape[0] total_num += num_batch print("processing %s/%s" % (num_batch, total_num)) ####################### # print(probs) if task_type == 'multiclass': predictions_val = np.argmax(probs, axis=1) elif task_type == 'multilabel': threshold = hparams.get('threshold', 0.5) predictions_val = (probs > threshold).astype( dtype=np.int32) elif task_type == 'topk': predictions_val = (probs > 0).astype(dtype=np.int32) else: raise ConfigureError( "Task type %s is not support for task %s. " "Only multiclass and multilabel is support for task %s" % (task_type, task, task)) # predictions = (probs > 0.5).astype(np.int32) # print(predictions) y_true.append(true_label_val) y_pred.append(predictions_val) # print(predictions) # for i in range(probs.shape[0]): # predictions = (probs > 0.5).astype(np.int32) # predict = predictions[i] # label = true_label_val[i] # if predict == label: # accuracy += 1 # confusion_matrix[label][predict] += 1 ################ for i in range(num_batch): premise_str = vocab.convert_indexes_to_tokens( premise_tokens_val[i], 'tokens') premise_str = " ".join(premise_str) hypothesis_str = vocab.convert_indexes_to_tokens( hypothesis_tokens_val[i], 'tokens') hypothesis_str = " ".join(hypothesis_str) if task_type == 'multilabel' or task_type == 'topk': predictions = [[] for i in range(num_batch)] for (row, col) in np.argwhere(predictions_val == 1): predictions[row].append(col) true_labels = [[] for i in range(num_batch)] for row, col in np.argwhere(true_label_val == 1): true_labels[row].append(col) else: predictions = predictions_val true_labels = true_label_val true_label = true_labels[i] predict = predictions[i] prob = probs[i] if task_type == 'multiclass': tsv_str = "\t".join([ premise_str, hypothesis_str, vocab.get_index_token(true_label, namespace='labels'), vocab.get_index_token(predict, namespace='labels'), str(prob) ]) elif task_type == 'multilabel' or task_type == 'topk': tsv_str = "\t".join([ premise_str, hypothesis_str, " ".join([ vocab.get_index_token(l, namespace='labels') for l in true_label ]), " ".join([ vocab.get_index_token(p, namespace='labels') for p in predict ]), str(prob) ]) else: raise ConfigureError( "Task type %s is not support for task %s. " "Only multiclass and multilabel is support for task %s" % (task_type, task, task)) # tsv_str = "\t".join([premise_str, hypothesis_str, str(true_label), str(predict), str(prob), # json.dumps(output_vals['query_embedding'][i].tolist()), json.dumps(output_vals['title_embedding'][i].tolist()), # json.dumps(output_vals['query_lstm_1'][i].tolist()), json.dumps(output_vals['title_lstm_1'][i].tolist()), # json.dumps(output_vals['query_attention'][i].tolist()), json.dumps(output_vals['title_attention'][i].tolist()), # json.dumps(output_vals['query_lstm_2'][i].tolist()), json.dumps(output_vals['title_lstm_2'][i].tolist()), # json.dumps(output_vals['fc1'][i].tolist()), json.dumps(output_vals['fc2'][i].tolist()) # ]) tsv_file.write(tsv_str + "\n") # print("process %s/%s correct/total instances with accuracy %s." % (accuracy, total_num, accuracy/float(total_num))) except tf.errors.OutOfRangeError as e: logger.info("processed all the evalutation data") break # logger.warning(e) y_true = np.concatenate(y_true, axis=0) y_pred = np.concatenate(y_pred, axis=0) avg_param = 'micro' if num_classes == 2: avg_param = 'binary' accuracy = metrics.accuracy_score(y_true, y_pred) # accuracy/total_num precise, recall, f1score, support = metrics.precision_recall_fscore_support( y_true, y_pred, labels=labels, average=avg_param) if task_type == 'multiclass': confusion_matrix = metrics.confusion_matrix(y_true, y_pred, labels=labels) print("metrics:") confmx_str = "label \ predict " for i in range(num_classes): confmx_str += "| %s | " % vocab.get_index_token( i, namespace='labels') confmx_str += "\n" for i in range(num_classes): confmx_str += "| %s | " % vocab.get_index_token( i, namespace='labels') for j in range(num_classes): confmx_str += "| %s | " % confusion_matrix[i][j] confmx_str += "\n" print(confmx_str) elif task_type == 'multilabel' or task_type == 'topk': confusion_matrix = metrics.multilabel_confusion_matrix( y_true, y_pred) print("metrics:") for k in range(num_classes): print("confusion matrix for label %s" % vocab.get_index_token(k, namespace='labels')) confmx_str = "label \ predict " for i in range(2): confmx_str += "| %s | " % i confmx_str += "\n" for i in range(2): confmx_str += "| %s | " % i for j in range(2): confmx_str += "| %s | " % confusion_matrix[k][i][j] confmx_str += "\n" print(confmx_str) else: raise ConfigureError( "Task type %s is not support for task %s. " "Only multiclass and multilabel is support for task %s" % (task_type, task, task)) # confusion_matrix[1][1]/(confusion_matrix[0][1]+confusion_matrix[1][1]) # recall = confusion_matrix[1][1]/(confusion_matrix[1][0]+confusion_matrix[1][1]) # f1score = (precise+recall)/2 print("micro total accuracy precise recall f1-score") print( "accuracy: %.2f, precise: %.2f, recall: %.2f, f1-score: %.2f" % (accuracy, precise, recall, f1score)) precisions, recalls, fbeta_scores, supports = metrics.precision_recall_fscore_support( y_true, y_pred, labels=labels) print("accuracy precise recall f1-score for each class") print( '======================================================================================' ) for lab_idx, (precision, recall, fbeta_score, support) in enumerate( zip(precisions, recalls, fbeta_scores, supports)): print( "label:%s\tprecision:%.2f\trecall:%.2f\tf1-score:%.2f\tsupport:%.2f" % (vocab.get_index_token(lab_idx, namespace='labels'), precision, recall, fbeta_score, support)) # legend = ["label \ predict "] # for i in range(num_classes): # legend.append(str(i)) # ws.append(legend) # for i in range(num_classes): # row = [str(i)] # for j in range(num_classes): # row.append(str(confusion_matrix[i][j])) # ws.append(row) # ws.append([]) # ws.append([]) # ws.append(['accuracy', 'precise', 'recall', 'f1-score']) # ws.append([str(accuracy), str(precise), str(recall), str(f1score)]) # if output_file: # if not output_file.endswith(".xlsx"): # output_file += '.xlsx' # wb.save(output_file) tsv_file.close()
def __init__(self, data_reader=None, pred_input_fn=None, vocab=None, export_dir=None, output_file=None): if data_reader is not None and pred_input_fn is None: self._pred_input_fn = data_reader.make_estimator_input_fn( DataSplit.PREDICT, force_repeat=False) vocab = data_reader.get_vocab() else: self._pred_input_fn = pred_input_fn dataset = self._pred_input_fn() iterator = dataset.make_initializable_iterator() dataset.make_initializable_iterator() next_element = iterator.get_next() self.saved_model_loader = loader_impl.SavedModelLoader(export_dir) mode = DataSplit.PREDICT signature_def = get_signature_def_for_mode(self.saved_model_loader, mode) input_map = self.generate_input_map(signature_def, next_element) output_tensor_names = [ value.name for value in signature_def.outputs.values() ] try: tags = model_fn.EXPORT_TAG_MAP[mode] except AttributeError as e: tags = ['serve'] saver, output_tensors = self.saved_model_loader.load_graph( tf.get_default_graph(), tags, input_map=input_map, return_elements=output_tensor_names) output_map = dict(zip(output_tensor_names, output_tensors)) outputs = { key: output_map[value.name] for (key, value) in signature_def.outputs.items() } #####xlsx wirte###### # wb = Workbook(write_only=True) # ws = wb.create_sheet('examples') # ws.append(['index', 'question', 'answer', 'predict', 'score']) csv_file = open(output_file, 'w', encoding="utf-8") csv_file.write( "\t".join(["index", 'premise', 'hypothesis', "prediction"]) + "\n") #csv_file.write("\t".join(["index", "premise", "hypothesis", "prediction", "prob"])+"\n") total_num = 0 #accuracy = 0 #confusion_matrix = [[0 for j in range(num_classes)] for i in range(num_classes)] with tf.Session() as sess: self.saved_model_loader.restore_variables(sess, saver) self.saved_model_loader.run_init_ops(sess, tags) sess.run(iterator.initializer) while True: try: outputs['inputs'] = next_element output_vals = sess.run(outputs) data_batch = output_vals['inputs'] if "index/index" in data_batch: index_val, premise_tokens_val, hypothesis_tokens_val = \ data_batch['index/index'], data_batch['premise/tokens'], data_batch['hypothesis/tokens'] indexs = [str(index, 'utf-8') for index in index_val] else: premise_tokens_val, hypothesis_tokens_val = \ data_batch['premise/tokens'], data_batch['hypothesis/tokens'] index_val = list( range(total_num, total_num + hypothesis_tokens_val.shape[0])) indexs = [str(index) for index in index_val] probs = output_vals['output'] num_batch = probs.shape[0] ####################### predictions = np.argmax(probs, axis=1) #predictions = (probs <= 0.5).astype(np.int32) total_num += num_batch logger.info("processing %s/%s" % (num_batch, total_num)) # for i in range(probs.shape[0]): # predictions = (probs > 0.5).astype(np.int32) # predict = predictions[i] # label = true_label_val[i] # if predict == label: # accuracy += 1 # confusion_matrix[label][predict] += 1 ################ for i in range(num_batch): premise_str = vocab.convert_indexes_to_tokens( premise_tokens_val[i], 'tokens') premise_str = " ".join(premise_str) hypothesis_str = vocab.convert_indexes_to_tokens( hypothesis_tokens_val[i], 'tokens') hypothesis_str = " ".join(hypothesis_str) predict = predictions[i] prob = probs[i] index = indexs[i] #csv_str = "\t".join([index, premise_str, hypothesis_str, str(predict), str(prob)])+"\n" if hypothesis_str.strip() != "" and premise_str.strip( ) != "" and hypothesis_str.strip( ) != premise_str.strip(): csv_str = "\t".join([ index, premise_str, hypothesis_str, str(predict) ]) + "\n" csv_file.write(csv_str) #ws.append([index, premise_str, hypothesis_str, str(predict), str(prob)]) #print("process %s/%s correct/total instances with accuracy %s." % (accuracy, total_num, accuracy/float(total_num))) except tf.errors.OutOfRangeError as e: # if output_file: # if not output_file.endswith(".xlsx"): # output_file += '.xlsx' # wb.save(output_file) csv_file.close() break