def load_model(): if not os.path.exists(Hyperparameters.MODEL_PATH): model = ModelBuilder.build_model(Hyperparameters.INPUT_SHAPE, Hyperparameters.OUTPUT_SHAPE) else: model = load_model( Hyperparameters.MODEL_PATH, custom_objects=ModelBuilder.get_model_custom_objects()) return model
def main(_): tf.gfile.MakeDirs(FLAGS.output_dir) if FLAGS.is_fixed_emb: emb_matrix = utils.get_emb_matrix(FLAGS.data_dir, FLAGS.max_features) clr = CyclicLR(base_lr=FLAGS.min_lr, max_lr=FLAGS.max_lr, step_size=2740, mode='exp_range', gamma=0.99994) matcher = TextMatcher(FLAGS.model_name, FLAGS.vocab_file, FLAGS.do_lower_case, FLAGS.max_seq_len) model_builder = ModelBuilder(model_name=FLAGS.model_name, max_len=FLAGS.max_seq_len, input_dim=FLAGS.input_dim, max_features=FLAGS.max_features, units=FLAGS.units, num_filter=FLAGS.num_filter) if FLAGS.is_fixed_emb: model_builder.set_embedding_matrix(emb_matrix) model = model_builder.build_model() print(model.summary()) if FLAGS.do_train: train_example = matcher.get_train_examples(FLAGS.data_dir) matcher.do_train(model, FLAGS.output_dir, train_example, FLAGS.epochs, FLAGS.batch_size, callback=[ clr, ]) if FLAGS.do_eval: dev_example = matcher.get_dev_examples(FLAGS.data_dir) matcher.do_eval(model, FLAGS.output_dir, dev_example, FLAGS.batch_size) if FLAGS.do_predict: test_example = matcher.get_test_examples(FLAGS.data_dir) matcher.do_predict(model, FLAGS.output_dir, test_example, FLAGS.batch_size)
def make_evaluations_gpu(conf,shot_list,loader): os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' #=cpu import theano from keras.utils.generic_utils import Progbar from model_builder import ModelBuilder builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] batch_size = min(len(shot_list),conf['model']['pred_batch_size']) pbar = Progbar(len(shot_list)) print('evaluating {} shots using batchsize {}'.format(len(shot_list),batch_size)) shot_sublists = shot_list.sublists(batch_size,equal_size=False) all_metrics = [] all_weights = [] for (i,shot_sublist) in enumerate(shot_sublists): batch_size = len(shot_sublist) model = builder.build_model(True,custom_batch_size=batch_size) builder.load_model_weights(model) model.reset_states() X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist,custom_batch_size=batch_size) #load data and fit on data all_metrics.append(model.evaluate(X,y,batch_size=batch_size,verbose=False)) all_weights.append(batch_size) model.reset_states() pbar.add(1.0*len(shot_sublist)) loader.verbose=False#True during the first iteration if len(all_metrics) > 1: print('evaluations all: {}'.format(all_metrics)) loss = np.average(all_metrics,weights = all_weights) print('Evaluation Loss: {}'.format(loss)) return loss