def main(): parser = get_parser() args = parser.parse_args() if args.verbose: # logger.setLevel(logging.INFO) logging.basicConfig(level=logging.INFO) # To do: argparse for config file if args.datapath is None: config = util.initialize_from_env(model_name='final') else: config = {'datapath': args.datapath} download_data(config)
def __init__(self, model_name='final', config=None, verbose=False): if verbose: logger.setLevel(logging.INFO) if config: self.config = config else: # if no configuration is provided, try to get a default config. self.config = util.initialize_from_env(model_name=model_name) # Clear tensorflow context: tf.reset_default_graph() self.session = tf.compat.v1.Session() try: self.model = cm.CorefModel(self.config) self.model.restore(self.session) except ValueError: raise Exception("Trying to reload the model while the previous " + "session hasn't been ended. Close the existing " + "session with predictor.end_session()")
def __init__(self, config, pipeline, use_gpu): # Make e2edutch follow Stanza's GPU settings: # set the environment value for GPU, so that initialize_from_env picks it up. # if use_gpu: # os.environ['GPU'] = ' '.join(tf.config.experimental.list_physical_devices('GPU')) # else: # if 'GPU' in os.environ['GPU'] : # os.environ.pop('GPU') self.e2econfig = util.initialize_from_env(model_name='final') # Override datapath and log_root: # store e2edata with the Stanza resources, ie. a 'stanza_resources/nl/coref' directory self.e2econfig['datapath'] = Path(config['model_path']).parent self.e2econfig['log_root'] = Path(config['model_path']).parent # Download data files if not present download_data(self.e2econfig) # Start and stop a session to cache all models predictor = Predictor(config=self.e2econfig) predictor.end_session()
import tensorflow.compat.v1 as tf tf.disable_v2_behavior() def get_parser(): parser = argparse.ArgumentParser() parser.add_argument('config') parser.add_argument('--cfg_file', type=str, default=None, help="config file") parser.add_argument('--model_cfg_file', type=str, default=None, help="model config file") parser.add_argument('-v', '--verbose', action='store_true') return parser if __name__ == "__main__": args = get_parser().parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) config = util.initialize_from_env( args.config, args.cfg_file, args.model_cfg_file) model = cm.CorefModel(config) with tf.Session() as session: model.restore(session) model.evaluate(session, official_stdout=True)
def main(args=None): args = get_parser().parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) config = util.initialize_from_env(args.config, args.cfg_file, args.model_cfg_file) # Overwrite train and eval file if specified if args.train is not None: config['train_path'] = args.train if args.eval is not None: config['eval_path'] = args.eval if args.eval_conll is not None: config['conll_eval_path'] = args.eval_conll report_frequency = config["report_frequency"] eval_frequency = config["eval_frequency"] model = cm.CorefModel(config) saver = tf.train.Saver() log_dir = os.path.join(config['log_root'], config['log_dir']) writer = tf.summary.FileWriter(log_dir, flush_secs=20) max_f1 = 0 with tf.Session() as session: session.run(tf.global_variables_initializer()) model.start_enqueue_thread(session) accumulated_loss = 0.0 ckpt = tf.train.get_checkpoint_state(log_dir) if ckpt and ckpt.model_checkpoint_path: print("Restoring from: {}".format(ckpt.model_checkpoint_path)) saver.restore(session, ckpt.model_checkpoint_path) initial_time = time.time() while True: tf_loss, tf_global_step, _ = session.run( [model.loss, model.global_step, model.train_op]) accumulated_loss += tf_loss if tf_global_step % report_frequency == 0: total_time = time.time() - initial_time steps_per_second = tf_global_step / total_time average_loss = accumulated_loss / report_frequency print("[{}] loss={:.2f}, steps/s={:.2f}" .format(tf_global_step, average_loss, steps_per_second)) writer.add_summary(util.make_summary( {"loss": average_loss}), tf_global_step) accumulated_loss = 0.0 if tf_global_step % eval_frequency == 0: saver.save(session, os.path.join(log_dir, "model"), global_step=tf_global_step) eval_summary, eval_f1 = model.evaluate(session) if eval_f1 > max_f1: max_f1 = eval_f1 util.copy_checkpoint(os.path.join( log_dir, "model-{}".format(tf_global_step)), os.path.join(log_dir, "model.max.ckpt")) writer.add_summary(eval_summary, tf_global_step) writer.add_summary(util.make_summary( {"max_eval_f1": max_f1}), tf_global_step) print("[{}] evaL_f1={:.2f}, max_f1={:.2f}".format( tf_global_step, eval_f1, max_f1))
def main(args=None): parser = get_parser() args = parser.parse_args() if args.verbose: logger.setLevel(logging.INFO) # Input file in .jsonlines format or .conll. input_filename = args.input_filename ext_input = os.path.splitext(input_filename)[-1] if ext_input not in ['.conll', '.jsonlines', '.txt', '.naf']: raise Exception( 'Input file should be .naf, .conll, .txt or .jsonlines, but is {}.' .format(ext_input)) if ext_input == '.conll': labels = collections.defaultdict(set) stats = collections.defaultdict(int) docs = minimize.minimize_partition(input_filename, labels, stats, args.word_col) elif ext_input == '.jsonlines': docs = read_jsonlines(input_filename) elif ext_input == '.naf': naf_obj = naf.get_naf(input_filename) jsonlines_obj, term_ids, tok_ids = naf.get_jsonlines(naf_obj) docs = [jsonlines_obj] else: text = open(input_filename).read() docs = [util.create_example(text)] output_file = args.output_file config = util.initialize_from_env(model_name=args.model, cfg_file=args.cfg_file, model_cfg_file=args.model_cfg_file) predictor = Predictor(config=config) sentences = {} predictions = {} for example_num, example in enumerate(docs): example["predicted_clusters"] = predictor.predict(example) if args.format_out == 'jsonlines': output_file.write(json.dumps(example)) output_file.write("\n") else: predictions[example['doc_key']] = example["predicted_clusters"] sentences[example['doc_key']] = example["sentences"] if example_num % 100 == 0: logger.info("Decoded {} examples.".format(example_num + 1)) if args.format_out == 'conll': conll.output_conll(output_file, sentences, predictions) elif args.format_out == 'naf': # Check number of docs - what to do if multiple? # Create naf obj if input format was not naf if ext_input != '.naf': # To do: add linguistic processing layers for terms and tokens logger.warn('Outputting NAF when input was not naf,' + 'no dependency information available') for doc_key in sentences: naf_obj, term_ids = naf.get_naf_from_sentences( sentences[doc_key]) naf_obj = naf.create_coref_layer(naf_obj, predictions[doc_key], term_ids) naf_obj = naf.add_linguistic_processors(naf_obj) buffer = io.BytesIO() naf_obj.dump(buffer) output_file.write(buffer.getvalue().decode('utf-8')) # To do, make sepearate outputs? # TO do, use dependency information from conll? else: # We only have one input doc naf_obj = naf.create_coref_layer(naf_obj, example["predicted_clusters"], term_ids) naf_obj = naf.add_linguistic_processors(naf_obj) buffer = io.BytesIO() naf_obj.dump(buffer) output_file.write(buffer.getvalue().decode('utf-8'))