def load_model():
if not os.path.exists(Hyperparameters.MODEL_PATH):
model = ModelBuilder.build_model(Hyperparameters.INPUT_SHAPE,
Hyperparameters.OUTPUT_SHAPE)
else:
model = load_model(
Hyperparameters.MODEL_PATH,
custom_objects=ModelBuilder.get_model_custom_objects())
return model
def main(_):
tf.gfile.MakeDirs(FLAGS.output_dir)
if FLAGS.is_fixed_emb:
emb_matrix = utils.get_emb_matrix(FLAGS.data_dir, FLAGS.max_features)
clr = CyclicLR(base_lr=FLAGS.min_lr,
max_lr=FLAGS.max_lr,
step_size=2740,
mode='exp_range',
gamma=0.99994)
matcher = TextMatcher(FLAGS.model_name, FLAGS.vocab_file,
FLAGS.do_lower_case, FLAGS.max_seq_len)
model_builder = ModelBuilder(model_name=FLAGS.model_name,
max_len=FLAGS.max_seq_len,
input_dim=FLAGS.input_dim,
max_features=FLAGS.max_features,
units=FLAGS.units,
num_filter=FLAGS.num_filter)
if FLAGS.is_fixed_emb:
model_builder.set_embedding_matrix(emb_matrix)
model = model_builder.build_model()
print(model.summary())
if FLAGS.do_train:
train_example = matcher.get_train_examples(FLAGS.data_dir)
matcher.do_train(model,
FLAGS.output_dir,
train_example,
FLAGS.epochs,
FLAGS.batch_size,
callback=[
clr,
])
if FLAGS.do_eval:
dev_example = matcher.get_dev_examples(FLAGS.data_dir)
matcher.do_eval(model, FLAGS.output_dir, dev_example, FLAGS.batch_size)
if FLAGS.do_predict:
test_example = matcher.get_test_examples(FLAGS.data_dir)
matcher.do_predict(model, FLAGS.output_dir, test_example,
FLAGS.batch_size)
def make_evaluations_gpu(conf,shot_list,loader):
os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' #=cpu
import theano
from keras.utils.generic_utils import Progbar
from model_builder import ModelBuilder
builder = ModelBuilder(conf)
y_prime = []
y_gold = []
disruptive = []
batch_size = min(len(shot_list),conf['model']['pred_batch_size'])
pbar = Progbar(len(shot_list))
print('evaluating {} shots using batchsize {}'.format(len(shot_list),batch_size))
shot_sublists = shot_list.sublists(batch_size,equal_size=False)
all_metrics = []
all_weights = []
for (i,shot_sublist) in enumerate(shot_sublists):
batch_size = len(shot_sublist)
model = builder.build_model(True,custom_batch_size=batch_size)
builder.load_model_weights(model)
model.reset_states()
X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist,custom_batch_size=batch_size)
#load data and fit on data
all_metrics.append(model.evaluate(X,y,batch_size=batch_size,verbose=False))
all_weights.append(batch_size)
model.reset_states()
pbar.add(1.0*len(shot_sublist))
loader.verbose=False#True during the first iteration
if len(all_metrics) > 1:
print('evaluations all: {}'.format(all_metrics))
loss = np.average(all_metrics,weights = all_weights)
print('Evaluation Loss: {}'.format(loss))
return loss
