def train():
model_config = ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % FLAGS.model_type
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=3,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up gradient clipping function
# def _clip_gradient_by_value(gvs):
# return [(tf.clip_by_value(grad, -training_config.clip_gradients,
# training_config.clip_gradients), var) for grad, var in gvs]
# grad_proc_fn = _clip_gradient_by_value
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver)
def train():
model_config = ModelConfig()
training_config = TrainConfig()
model_config.batch_size = 32
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.model_type, model_config.feat_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# create reader
reader = AttentionDataReader(batch_size=model_config.batch_size,
subset='trainval',
feat_type=model_config.feat_type)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict)
def workflow(resnet_model, model_num):
# Build and train model.
model_config_object = ModelConfig(resnet_model, model_num)
ResNetModel = ModelCNN(model_config_object)
ResNetModel.train_valid()
# Evaluate best model.
model_config_object = ModelConfig(resnet_model, model_num)
ResNetModel = ModelCNN(model_config_object)
best_model = model_config_object.resnet_model
state = torch.load(os.path.join(model_config_object.model_save_dir, model_config_object.save_best_model_name),
map_location=model_config_object.device)
best_model.load_state_dict(state['state_dict'])
best_model = best_model.to(model_config_object.device)
best_model.eval()
# Write test results to csv.
with open('result.csv', 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(["Id", "Expected"])
for i in range(432):
image_name = 'data/test/' + str(i) + '.jpg'
test_image = Image.open(image_name)
test_image_transform = model_config_object.data_transforms['test'](test_image)
outputs = best_model(test_image_transform.unsqueeze(0).to(model_config_object.device))
_, preds = torch.max(outputs.data, 1)
class_ = int(preds.data.cpu().numpy())
class_to_idx = state['class_to_idx']
category = list(class_to_idx.keys())[list(class_to_idx.values()).index(class_)]
writer.writerow([str(i), category])
def vaq_multiple_choices(checkpoint_path=None, subset='kpval'):
need_attr = False
need_im_feat = True
use_answer_type = False
feat_type = 'res152'
model_config = ModelConfig()
model_config.cell_option = 4
# Get model
print(FLAGS.model_type)
model_fn = get_model_creation_fn(FLAGS.model_type)
mc_ctx = MultipleChoiceEvaluater(subset='test',
need_im_feat=need_im_feat,
need_attr=need_attr,
feat_type=feat_type,
use_ans_type=use_answer_type)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir %
FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, phase='condition')
model.build()
losses = model.losses
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' %
os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
num_batches = mc_ctx.num_samples
print('Running multiple choices...')
predictions, answer_ids = [], []
for i in range(num_batches):
if i % 1000 == 0:
print('Running multiple choices: %d/%d' % (i, num_batches))
outputs = mc_ctx.get_task_data()
im, capt, capt_len, _, ans_seq, ans_seq_len, _, _ = outputs
inputs = [im, capt, capt_len, ans_seq, ans_seq_len]
feed_dict = model.fill_feed_dict(inputs)
np_losses = sess.run(losses, feed_dict=feed_dict)
scores = -np_losses.sum(axis=1)[np.newaxis, :]
predictions.append(scores)
answer_ids.append(outputs[-2])
predictions = np.concatenate(predictions, axis=0)
answer_ids = np.array(answer_ids)
# evaluate
mc_ctx.evaluate_results(answer_ids,
predictions,
model_type=FLAGS.model_type)
def test():
from config import ModelConfig
model_fn = get_model_creation_fn('VAQ-CA')
config = ModelConfig()
config.top_answer_file = 'data/top_answer2000_sequences.h5'
config.vqa_agent_ckpt = '/import/vision-ephemeral/fl302/code/vqa2.0/model/' \
'curr_VQA-Soft_Res5c/model.ckpt-325000'
model = model_fn(config, phase='train')
model.build()
def sample_cst_questions(checkpoint_path=None, subset='kptrain'):
model_config = ModelConfig()
model_config.convert = FLAGS.convert
model_config.loss_type = 'pairwise'
model_config.top_k = 3
batch_size = 8
# Get model
create_fn = create_reader(FLAGS.model_type, phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# get data reader
reader = create_fn(batch_size=batch_size,
subset=subset,
version=FLAGS.test_version)
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.
model = ContrastQuestionSampler(model_config)
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
num_batches = reader.num_batches
print('Running beam search inference...')
for i in range(num_batches):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
c_ans, c_ans_len, pathes, scores = model.greedy_inference(
outputs[:-2], sess)
scores, pathes = post_process_prediction(scores, pathes)
k = 3
capt, capt_len = outputs[2:4]
gt = capt[0, :capt_len[0]]
print('gt: %s [%s]' %
(to_sentence.index_to_question(gt),
to_sentence.index_to_answer(c_ans[0, :c_ans_len[0]])))
for ix in range(k):
question = to_sentence.index_to_question(pathes[ix])
answer = to_sentence.index_to_answer(c_ans[ix, :c_ans_len[ix]])
print('%s %d: %s [%s]' %
('pre' if ix == 0 else 'cst', ix, question, answer))
import pdb
pdb.set_trace()
def return_a_config():
config = ModelConfig(max_sequence_length=15, annotation_number=10,
adaption_layer_filters=[1024, 1024, 512],
net_global_dim=[128, 256],
net_max_features_nums=512,
stages=[6],
GRAND_PARENT_PATH='/Users/junr/Documents/prp/pic_data',
IMAGE_PARENT_PATH='/Users/junr/Documents/prp/pic_data/pic_data'
)
config.finish()
return config
def predict_ml_model(data_type,
model_name,
variation,
vectorizer_type,
level,
ngram_range,
checkpoint_dir=None,
return_proba=True,
**kwargs):
config = ModelConfig()
if checkpoint_dir is not None:
config.checkpoint_dir = checkpoint_dir
config.exp_name = '{}_{}_{}_{}_{}'.format(variation, model_name,
vectorizer_type, level,
ngram_range)
print('Logging Info - Experiment: ', config.exp_name)
if model_name == 'svm':
model = SVMModel(config, **kwargs)
elif model_name == 'lr':
model = LRModel(config, **kwargs)
elif model_name == 'sgd':
model = SGDModel(config, **kwargs)
elif model_name == 'gnb':
model = GaussianNBModel(config, **kwargs)
elif model_name == 'mnb':
model = MultinomialNBModel(config, **kwargs)
elif model_name == 'bnb':
model = BernoulliNBModel(config, **kwargs)
elif model_name == 'rf':
model = RandomForestModel(config, **kwargs)
elif model_name == 'gbdt':
model = GBDTModel(config, **kwargs)
elif model_name == 'xgboost':
model = XGBoostModel(config, **kwargs)
else:
raise ValueError('Model Name Not Understood : {}'.format(model_name))
model_save_path = path.join(config.checkpoint_dir,
'{}.hdf5'.format(config.exp_name))
if not path.exists(model_save_path):
raise FileNotFoundError('Model Not Found: {}'.format(model_save_path))
model.load_best_model()
data = load_ngram_data(variation, vectorizer_type, level, ngram_range,
data_type)
if data is None:
return None, config.exp_name
if return_proba:
return model.predict_proba(data), config.exp_name
else:
return model.predict(data), config.exp_name
def evaluateResnetModel():
config = ModelConfig()
config.model_name = 'mnist-resnet'
config.input_data_dir = os.path.join('..', 'data')
config.log_dir = os.path.join('..', 'log')
config.batch_size = 10000
config.max_epoches = 1
config.input_size = 28
config.num_classes = 10
config.fake_data = False
model = ResnetModel(config)
evaluate(model)
def _add(self, src_path, is_first=False, count=0):
try:
model_path = str(src_path)
path_exists = os.path.exists(model_path)
if not path_exists and count > 0:
self.logger.error("{} not found, retry attempt is terminated.".format(model_path))
return
if 'model_demo.yaml' in model_path:
self.logger.warning(
"\n-------------------------------------------------------------------\n"
"- Found that the model_demo.yaml file exists, \n"
"- the loading is automatically ignored. \n"
"- If it is used for the first time, \n"
"- please copy it as a template. \n"
"- and do not use the reserved character \"model_demo.yaml\" as the file name."
"\n-------------------------------------------------------------------"
)
return
if model_path.endswith("yaml"):
model_conf = ModelConfig(self.conf, model_path)
inner_name = model_conf.model_name
inner_size = model_conf.size_string
inner_key = PathUtils.get_file_name(model_path)
for k, v in self.name_map.items():
if inner_size in v:
self.logger.warning(
"\n-------------------------------------------------------------------\n"
"- The current model {} is the same size [{}] as the loaded model {}. \n"
"- Only one of the smart calls can be called. \n"
"- If you want to refer to one of them, \n"
"- please use the model key or model type to find it."
"\n-------------------------------------------------------------------".format(
inner_key, inner_size, k
)
)
break
inner_value = model_conf.model_name
graph_session = GraphSession(model_conf)
if graph_session.loaded:
interface = Interface(graph_session)
if inner_name == self.conf.default_model:
self.interface_manager.set_default(interface)
else:
self.interface_manager.add(interface)
self.logger.info("{} a new model: {} ({})".format(
"Inited" if is_first else "Added", inner_value, inner_key
))
self.name_map[inner_key] = inner_value
if src_path in self.interface_manager.invalid_group:
self.interface_manager.invalid_group.pop(src_path)
else:
self.interface_manager.report(src_path)
if count < 12 and not is_first:
time.sleep(5)
return self._add(src_path, is_first=is_first, count=count+1)
except Exception as e:
self.interface_manager.report(src_path)
self.logger.error(e)
def eval(ckpt, use_emb=False):
# Recommended hyperparameters
args = ModelConfig(batch_size=64, ckpt=ckpt, dropout=0.5, use_emb=use_emb)
m = DictionaryModel(dict_field.vocab,
output_size=att_crit.input_size,
embed_input=args.use_emb,
dropout_rate=args.dropout)
m.load_state_dict(torch.load(args.ckpt)['m_state_dict'])
if torch.cuda.is_available():
m.cuda()
att_crit.cuda()
train_data.atts_matrix.cuda()
val_data.atts_matrix.cuda()
test_data.atts_matrix.cuda()
m.eval()
# Don't take the mean until the end
preds_ = []
labels_ = []
for val_b, (atts, words, defns, perm) in enumerate(tqdm(test_iter)):
preds_.append(att_crit.predict(m(defns, words))[perm])
labels_.append(atts.data.cpu().numpy()[perm])
preds = np.concatenate(preds_, 0)
labels = np.concatenate(labels_, 0)
acc_table = evaluate_accuracy(preds, labels).T.squeeze()
return acc_table, preds
def ivqa_decoding_beam_search(checkpoint_path=None, subset='kpval'):
model_config = ModelConfig()
res_file = 'result/quest_vaq_greedy_%s_%s.json' % (
FLAGS.model_type.upper(), subset)
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
create_fn = create_reader(FLAGS.model_type, phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# get data reader
reader = create_fn(batch_size=100,
subset=subset,
version=FLAGS.test_version)
if checkpoint_path is None:
ckpt_dir = FLAGS.checkpoint_dir % (FLAGS.version, FLAGS.model_type)
# ckpt_dir = '/import/vision-ephemeral/fl302/models/v2_kpvaq_VAQ-RL/'
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
checkpoint_path = ckpt.model_checkpoint_path
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'beam')
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
num_batches = reader.num_batches
print('Running beam search inference...')
results = []
for i in range(num_batches):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
scores, pathes = model.greedy_inference(outputs[:-2], sess)
scores, pathes = post_process_prediction(scores, pathes)
question = to_sentence.index_to_question(pathes[0])
print('%d/%d: %s' % (i, num_batches, question))
for quest_id, image_id, path in zip(quest_ids, image_ids, pathes):
sentence = to_sentence.index_to_question(path)
res_i = {
'image_id': int(image_id),
'question_id': int(quest_id),
'question': sentence
}
results.append(res_i)
save_json(res_file, results)
return res_file
def __init__(self, train_data, train_labels, train_weights):
self._model = None
self.hidden_dim = ModelConfig().hiddenLayerSize
self.train_data = train_data
self.train_labels = train_labels
self.train_weights = train_weights
self._init_model()
def train(self, n_epochs=5):
self._model.fit(self.train_data,
self.train_labels,
epochs=n_epochs,
batch_size=ModelConfig().batchSize,
sample_weight=self.train_weights,
shuffle=True,
validation_split=0.15)
def __init__(self, model_file=None):
self.config = ModelConfig()
if model_file:
self.model = load_model(model_file, compile=True)
else:
self.create()
self.loss_train_op()
def convert():
model_name = 'ivaq_var_restval'
checkpoint_path = 'model/var_ivqa_pretrain_restval/model.ckpt-505000'
# build model
from config import ModelConfig
model_config = ModelConfig()
model_fn = get_model_creation_fn('VAQ-Var')
# create graph
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'beam')
model.build()
tf_embedding = model._answer_embed
tf_answer_feed = model._ans
tf_answer_len_feed = model._ans_len
# Restore from checkpoint
print('Restore from %s' % checkpoint_path)
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
# build reader
top_ans_file = '/import/vision-ephemeral/fl302/code/' \
'VQA-tensorflow/data/vqa_trainval_top2000_answers.txt'
mc_ctx = MultiChoiceQuestionManger(subset='val',
load_ans=True,
top_ans_file=top_ans_file)
to_sentence = SentenceGenerator(trainset='trainval',
top_ans_file=top_ans_file)
answer_encoder = mc_ctx.encoder
top_answer_inds = range(2000)
top_answers = answer_encoder.get_top_answers(top_answer_inds)
answer_seqs = answer_encoder.encode_to_sequence(top_answers)
for i, (ans, seq) in enumerate(zip(top_answers, answer_seqs)):
rec_ans = to_sentence.index_to_answer(seq)
ans = ' '.join(_tokenize_sentence(ans))
print('%d: Raw: %s, Rec: %s' % (i + 1, ans, rec_ans))
assert (ans == rec_ans)
print('Checking passed')
# extract
print('Converting...')
ans_arr, ans_arr_len = put_to_array(answer_seqs)
import pdb
pdb.set_trace()
embedding = sess.run(tf_embedding,
feed_dict={
tf_answer_feed: ans_arr.astype(np.int32),
tf_answer_len_feed: ans_arr_len.astype(np.int32)
})
# save
sv_file = 'data/v1_%s_top2000_lstm_embedding.h5' % model_name
from util import save_hdf5
save_hdf5(sv_file, {'answer_embedding': embedding})
print('Done')
def vaq_decoding_greedy(checkpoint_path=None, subset='kpval'):
model_config = ModelConfig()
res_file = 'result/quest_vaq_greedy_%s.json' % FLAGS.model_type.upper()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
create_fn = create_reader(FLAGS.model_type, phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# build data reader
reader = create_fn(batch_size=32, subset=subset)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir %
FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'greedy')
model.build()
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' %
os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
num_batches = reader.num_batches
print('Running greedy inference...')
results = []
for i in range(num_batches):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
scores, pathes = model.greedy_inference(outputs[:-2], sess)
scores, pathes = post_process_prediction(scores, pathes)
question = to_sentence.index_to_question(pathes[0])
print('%d/%d: %s' % (i, num_batches, question))
for quest_id, image_id, path in zip(quest_ids, image_ids, pathes):
sentence = to_sentence.index_to_question(path)
res_i = {
'image_id': int(image_id),
'question_id': int(quest_id),
'question': sentence
}
results.append(res_i)
save_json(res_file, results)
return res_file
def test(checkpoint_path=None):
batch_size = 1
config = ModelConfig()
# Get model function
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = AttentionFetcher(batch_size=batch_size, subset=TEST_SET)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(
FLAGS.checkpoint_dir % (FLAGS.model_type, config.feat_type))
checkpoint_path = ckpt.model_checkpoint_path
print(checkpoint_path)
# build and restore model
model = model_fn(config, phase='test')
model.build()
g_prob = model.prob
g_att_map = model.attention_map
# sess = tf.Session()
sess = tf.Session(graph=tf.get_default_graph())
tf.logging.info('Restore from model %s' %
os.path.basename(checkpoint_path))
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
# Create the vocabulary.
visualiser = PredictionVisualiser(FLAGS.model_type, do_plot=True)
ans_ids = []
quest_ids = []
print('Running inference on split %s...' % TEST_SET)
for i in range(reader.num_batches):
if i % 100 == 0:
update_progress(i / float(reader.num_batches))
outputs = reader.get_test_batch()
if i < 100:
continue
generated_ans, att_map = sess.run([g_prob, g_att_map],
feed_dict=model.fill_feed_dict(
outputs[:-2]))
# process attention map
att_map = att_map.reshape([batch_size, 14, 14, -1])
att_map = np.transpose(att_map, [0, 3, 1, 2])
generated_ans[:, -1] = 0
top_ans = np.argmax(generated_ans, axis=1)
gt_ans = outputs[3]
ans_ids.append(top_ans)
quest_id = outputs[-2]
quest_ids.append(quest_id)
if np.random.rand() > 0.05:
visualiser.plot(quest_id, generated_ans, att_map)
def get_filenames():
# Read the filenames from the config
gt_conf = GenerateTeamsConfig()
m_conf = ModelConfig()
output = []
for suff in m_conf.targetFilesSuffix:
out_filename = "{}_{}".format(gt_conf.outputFilenamePrefix, suff)
uniq_filename = "{}_{}".format(gt_conf.uniqueFilenamePrefix, suff)
output.append(
(join(config.TEAMS_TXT_DIR,
out_filename), join(config.TEAMS_TXT_DIR, uniq_filename)))
return output
def output_model(project_name: str, model_type: ModelType, key=None):
model_conf = ModelConfig(project_name, is_dev=False)
graph_parent_path = model_conf.compile_model_path
model_suffix = COMPILE_MODEL_MAP[model_type]
model_bytes = pickle.dumps(model_conf.conf)
graph_path = os.path.join(
graph_parent_path, "{}{}".format(model_conf.model_name, model_suffix))
with open(graph_path, "rb") as f:
graph_bytes = f.read()
output_path = graph_path.replace(".pb", ".pl").replace(".onnx", ".pl")
concat_model(output_path, model_bytes, graph_bytes, key)
def __init__(self, model_state: ModelState, title):
model_config = ModelConfig()
self.window = PlotWindow(model_config, title)
self.stationary_nodes = []
self.mobile_node = None
self.draw_points(model_state)
self.lines = []
self.history_line = None
self.curve_fit_line = None
if title == 'Prediction':
ModelConfig.prediction_plot = self
else:
ModelConfig.simulation_plot = self
def vaq_condition(checkpoint_path=None):
subset = 'dev'
model_config = ModelConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = Reader(batch_size=1, subset=subset, output_attr=True, output_im=False,
output_qa=True, output_capt=False)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir % FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'condition')
model.build()
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' % os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
fetch_op = model.losses
num_batches = reader.num_batches
save_file = 'data/%s_vaq_cond_score1000-2000_%s.hdf5' % ((FLAGS.model_type).lower(), subset)
print('Save File: %s' % save_file)
print('Running conditioning...')
nlls, quest_ids = [], []
for i in range(num_batches):
update_progress(i / float(num_batches))
outputs = reader.get_test_batch()
im_feed, quest, _, ans_feed, quest_id, image_id = outputs
losses = sess.run(fetch_op, feed_dict=model.fill_feed_dict(outputs[:-2]))
scores = losses[:, :-1].mean(axis=1)
scores = scores[np.newaxis, ::]
nlls.append(scores)
quest_ids.append(quest_id)
nlls = np.concatenate(nlls, axis=0)
quest_ids = np.concatenate(quest_ids, axis=0)
print('\nSaving result files: %s...' % save_file)
save_hdf5(save_file, {'nll': nlls, 'quest_ids': quest_ids})
def __init__(self, ckpt_file='model/v1_vqa_VQA/v1_vqa_VQA_best2/model.ckpt-135000'):
self.g = tf.Graph()
self.ckpt_file = ckpt_file
from models.vqa_soft_attention import AttentionModel
from vqa_config import ModelConfig
config = ModelConfig()
self.name = ' ------- MLB-attention ------- '
with self.g.as_default():
self.sess = tf.Session()
self.model = AttentionModel(config, phase='test_broadcast')
self.model.build()
vars = tf.trainable_variables()
self.saver = tf.train.Saver(var_list=vars)
self.saver.restore(self.sess, ckpt_file)
def __init__(self, g, sess):
from models.vqa_base import BaseModel
from vqa_config import ModelConfig
config = ModelConfig()
self.sess = sess
ckpt_file = 'model/kprestval_VQA-BaseNorm/model.ckpt-26000'
with g.as_default():
self.sess = tf.Session()
self.model = BaseModel(config, phase='test')
with tf.variable_scope('VQA'):
self.model.build()
# vars = tf.trainable_variables()
vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='VQA')
vars_dict = {v.name.replace('VQA/', '').split(':')[0]: v for v in vars}
self.saver = tf.train.Saver(var_list=vars_dict)
self.saver.restore(self.sess, ckpt_file)
def _get_close_walls(self):
danger_walls = DangerWalls()
area = ModelConfig().area
danger_zone = ModelConfig.danger_range
if self.coords.x < self._get_min_danger_border(area.x_min,
danger_zone):
danger_walls.left_wall = True
if self.coords.x > self._get_max_danger_border(area.x_max,
danger_zone):
danger_walls.right_wall = True
if self.coords.y < self._get_min_danger_border(area.y_min,
danger_zone):
danger_walls.bottom_wall = True
if self.coords.y > self._get_max_danger_border(area.y_max,
danger_zone):
danger_walls.top_wall = True
return danger_walls
def prepare_data(file_names):
try:
# Check if data already exists in the mat dir
m_conf = ModelConfig()
mat_file = "{}.npy".format(m_conf.matrixDataFile)
lab_file = "{}.npy".format(m_conf.matrixLabelFile)
wei_file = "{}.npy".format(m_conf.matrixWeightFile)
onehot_file = "{}.json".format(m_conf.onehotFile)
data_matrix = npy.load(join(config.TEAMS_MAT_DIR, mat_file))
label_matrix = npy.load(join(config.TEAMS_MAT_DIR, lab_file))
weight_matrix = npy.load(join(config.TEAMS_MAT_DIR, wei_file))
with open(join(config.TEAMS_MAT_DIR, onehot_file)) as json_file:
onehot = json.load(json_file)
return onehot, data_matrix, label_matrix, weight_matrix
except:
# If not then load from the filenames
return onehot_encode_data(file_names)
def __init__(self):
self.to_sentence = SentenceGenerator(trainset='trainval')
self.sent_encoder = SentenceEncoder()
self.g = tf.Graph()
self.ckpt_file = 'model/v1_var_kptrain_VAQ-VarDS/model.ckpt-3300000'
from models.variational_ds_ivqa_model import VariationIVQAModel
from config import ModelConfig
config = ModelConfig()
self._top_k = 10
self.name = ' ------- VarIVQA ------- '
with self.g.as_default():
self.sess = tf.Session()
self.model = VariationIVQAModel(config, phase='sampling_beam')
self.model.build()
vars = tf.trainable_variables()
self.saver = tf.train.Saver(var_list=vars)
self.saver.restore(self.sess, self.ckpt_file)
self._init_image_cache()
def test(checkpoint_path=None):
batch_size = 128
config = ModelConfig()
# Get model function
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = TestReader(batch_size=batch_size, subset=TEST_SET)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(
FLAGS.checkpoint_dir %
(FLAGS.version, FLAGS.model_type, FLAGS.delta))
checkpoint_path = ckpt.model_checkpoint_path
print(checkpoint_path)
# build and restore model
model = model_fn(config, phase='test')
model.build()
sess = tf.Session(graph=tf.get_default_graph())
tf.logging.info('Restore from model %s' %
os.path.basename(checkpoint_path))
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
print('Running inference on split %s...' % TEST_SET)
aug_quest_ids, scores = [], []
for i in range(reader.num_batches):
if i % 10 == 0:
update_progress(i / float(reader.num_batches))
outputs = reader.get_test_batch()
rank_score = sess.run(model.prob,
feed_dict=model.fill_feed_dict(outputs[:3]))
_, quest_ids, image_ids = outputs[3:]
scores.append(rank_score)
aug_quest_ids.append(quest_ids)
aug_quest_ids = np.concatenate(aug_quest_ids)
scores = np.concatenate(scores)
return convert_to_questions(aug_quest_ids, scores)
def train_bert(genre, input_level, batch_size):
config = ModelConfig()
config.genre = genre
config.input_level = input_level
config.max_len = config.word_max_len[genre] if input_level == 'word' else config.char_max_len[genre]
config.batch_size = batch_size
model = TFHubBertModel(config, [0, 1, 2], EXTERNAL_WORD_VECTORS_FILENAME['tfhub_bert'])
train_input = load_input_data(genre, input_level, 'train', 'bert')
valid_input = load_input_data(genre, input_level, 'valid', 'bert')
test_input = load_input_data(genre, input_level, 'test', 'bert')
model.train(train_input, valid_input)
model.evaluate(valid_input)
model.evaluate(test_input)
def detect(imagesDir, modelPath, classes, name, masksDir, outputType,
classesColours, format):
# Create model object in inference mode.
config = ModelConfig(name=name, numClasses=len(classes) + 1)
model = modellib.MaskRCNN(mode="inference",
model_dir=modelPath,
config=config)
model.load_weights(modelPath, by_name=True)
classNames = ['BG']
for i in classes:
classNames.append(i)
classesColours = [float(i) for i in classesColours.split(',')]
# ## Run Object Detection
# TODO: Use the whole list instead of iteration
for imageFile in [
file for file in next(os.walk(imagesDir))[2]
if os.path.splitext(file)[1] == format
]:
image = skimage.io.imread(os.path.join(imagesDir, imageFile))
# Run detection
results = model.detect([image], verbose=1)
# Save results
# TODO: More images -> more indices than [0]
r = results[0]
visualize.save_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
classNames,
r['scores'],
outputDir=masksDir,
which=outputType,
title=imageFile,
colours=classesColours)
