def __init__(self, dim, act = nn.ReLU(), num_hiddens = [50], conf = dict()):
super(BNN_SVI, self).__init__()
self.dim = dim
self.act = act
self.num_hiddens = num_hiddens
self.num_iters = conf.get('num_iters', 4000)
self.batch_size = conf.get('batch_size', 32)
self.print_every = conf.get('print_every', 100)
self.lr = conf.get('lr', 1e-2)
self.weight_std = conf.get('weight_std', 1.0)
self.noise_level = conf.get('noise_level', None)
self.nn = NN(dim, self.act, self.num_hiddens, nout = 1)
class DataLoader(object):
def __init__(self, params, db, nn_db):
self.lock = Lock()
self.db = db
self.cur = db.length
self.im_shape = params['im_shape']
self.nn_shape = params['nn_shape']
self.hist_eq = params['hist_eq']
self.indexes = np.arange(db.length)
self.shuffle = params['shuffle']
self.subtract_mean = params['subtract_mean']
if self.subtract_mean:
self.mean_img = self.db.read_mean_img(self.im_shape)
self.im_shape = params['im_shape']
self.load_nn = params['load_nn']
self.nn_query_size = params['nn_query_size']
if self.load_nn:
self.nn_db = nn_db
#nn_ignore = 1 if db.db_root == nn_db.db_root else 0
nn_ignore = 0
self.nn = NN(nn_db, params['nn_db_size'], nn_ignore)
def load_next_data(self):
nid = self.get_next_id()
jp, imgs, segs = self.db.read_instance(nid, size=self.im_shape)
item = {'jp':jp}
for i in xrange(len(imgs)):
img = imgs[i]
if self.hist_eq:
img = correct_hist(img)
item.update({'img_' + shape_str(self.im_shape[i]):img.transpose((2,0,1)), 'seg_' + shape_str(self.im_shape[i]): segs[i]})
if self.load_nn:
nn_id = self.nn.nn_ids(jp, self.nn_query_size)
if hasattr(nn_id, '__len__'):
nn_id = random.choice(nn_id)
nn_jp, nn_imgs, nn_segs = self.nn_db.read_instance(nn_id, size=self.nn_shape)
item.update({'nn_jp':nn_jp})
for i in xrange(len(nn_imgs)):
nn_img = nn_imgs[i]
if self.hist_eq:
nn_img = correct_hist(nn_img)
item.update({'nn_img_' + shape_str(self.nn_shape[i]):nn_img.transpose((2,0,1)), 'nn_seg_' + shape_str(self.nn_shape[i]): nn_segs[i]})
return item
def get_next_id(self):
self.lock.acquire()
if self.cur >= len(self.indexes) - 1:
self.cur = 0
if self.shuffle:
random.shuffle(self.indexes)
else:
self.cur += 1
self.lock.release()
return self.indexes[self.cur]
class EKFNNLayer(caffe.Layer):
def setup(self, bottom, top):
params = eval(self.param_str)
check_params(params,
nn_root=None,
nn_shape=None,
nn_query_size=1,
nn_num=1,
nn_db_size=np.inf,
nn_ignore=1)
self.params = Map(params)
self.nn_db = DartDB(self.params.nn_root)
self.nn = NN(self.nn_db, self.params.nn_db_size, self.params.nn_ignore)
assert self.params.nn_num <= self.params.nn_query_size
def reshape(self, bottom, top):
#Reshape tops
batch_size = bottom[0].shape[0]
assert self.nn_db.jps.shape[1] == bottom[0].shape[1]
cur_top = 0
for nn_id in range(self.params.nn_num):
top[cur_top + 0].reshape(batch_size, 3, self.params.nn_shape[0], self.params.nn_shape[1])
top[cur_top + 1].reshape(batch_size, 1, self.params.nn_shape[0], self.params.nn_shape[1])
top[cur_top + 2].reshape(batch_size, self.nn_db.jps.shape[1])
top[cur_top + 3].reshape(batch_size, 1)
cur_top += 4
#self.top_names.extend(['nn_img_' + str(nn_id), 'nn_seg_' + str(nn_id)])
#self.top_names.append('nn_jp_' + str(nn_id))
#self.top_names.append('nn_w_' + str(nn_id))
def forward(self, bottom, top):
for itt in range(bottom[0].shape[0]):
jp = bottom[0].data[itt]
nn_ids = self.nn.nn_ids(jp, self.params.nn_query_size)
if hasattr(nn_ids, '__len__'):
nn_ids = np.random.choice(nn_ids, size=self.params.nn_num, replace=False)
else:
nn_ids = [nn_ids]
for i in range(len(nn_ids)):
nn_id = nn_ids[i]
nn_jp, nn_img, nn_seg = self.nn_db.read_instance(nn_id, size=self.params.nn_shape)
top[i * 4 + 0].data[itt, ...] = nn_img[0].transpose((2,0,1))
top[i * 4 + 1].data[itt, ...] = nn_seg[0]
top[i * 4 + 2].data[itt, ...] = nn_jp
top[i * 4 + 3].data[itt, ...] = 1
def backward(self, top, propagate_down, bottom):
bottom[0].diff[...] = 0
def forward_jv(self, top, bottom):
for top_id in range(self.params.nn_num * 4):
top[top_id].diff[...] = 0
def build_network(self):
# Define weights
weights = {
'wordEmbedding':
tf.Variable(self.qa.word_embedding,
trainable=True,
name="word_embeddding"),
'crf_W':
tf.get_variable("crf_W",
initializer=tf.random_uniform(
shape=[2 * self.n_hidden, self.n_tags],
minval=-0.08,
maxval=0.08)),
'crf_b':
tf.get_variable("crf_b",
initializer=tf.random_uniform(shape=[self.n_tags],
minval=-0.08,
maxval=0.08))
}
self.is_training = tf.placeholder(tf.bool, name="is_training")
self.question_ids = tf.placeholder(tf.int32, [None, self.n_steps])
self.x_lens = tf.placeholder(tf.int32, [None])
self.y = tf.placeholder(tf.int32, [None, self.n_steps])
x = tf.nn.embedding_lookup(weights['wordEmbedding'],
ids=self.question_ids)
query4ner, _ = NN.bi_gru(x, self.x_lens, self.n_hidden, "query4ner",
self.keep_prob, self.is_training)
crf_x = tf.concat(query4ner, 2)
matricized_x_t = tf.reshape(
crf_x,
[-1, 2 * self.n_hidden]) # maybe it is reasonable but pass it now!
matricized_unary_scores = tf.matmul(
matricized_x_t, weights['crf_W']) + weights['crf_b']
self.unary_scores = tf.reshape(matricized_unary_scores,
[-1, self.n_steps, self.n_tags])
log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood(
self.unary_scores, self.y, self.x_lens)
self.loss = tf.reduce_mean(-log_likelihood)
all_var = tf.global_variables()
grads = tf.gradients(self.loss, all_var)
grads = [tf.clip_by_value(grad, -10, 10) for grad in grads]
self.optimizer = tf.train.MomentumOptimizer(
learning_rate=self.learning_rate,
momentum=0.9).apply_gradients(zip(grads, all_var))
self.saver = tf.train.Saver()
def subject_network(self, x):
'''
:param x: query embedding after look-up
:return:subject_score:[batch_size,cand_sub_size] loss_subject:[1]
'''
with tf.variable_scope("subject_part"):
_, query4subject = NN.bi_gru(x, self.x_lens, self.n_hidden,
"bi_gru4subject_query",
self.keep_prob, self.is_training)
query4subject = tf.concat(query4subject, -1)
# [batch,type_len]
query4subject = tf.sigmoid(
NN.linear(query4subject,
self.qa.type_len,
name="query_trans_subject"))
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
self.type_emb_w_martix = tf.get_variable(
"query_trans_subjectkernel")
self.type_emb_b_martix = tf.get_variable(
"query_trans_subjectbias")
# [batch_size,emb_size]
gold_subject = self.subject_emb[:, 0, :]
self.loss_subject = gold_subject * tf.log(
tf.clip_by_value(
query4subject, 1e-10, 1.0)) + (1 - gold_subject) * tf.log(
tf.clip_by_value(1 - query4subject, 1e-10, 1.0))
# [1]
loss_subject = tf.reduce_mean(
-tf.reduce_sum(self.loss_subject, axis=1), axis=0)
with tf.variable_scope("test"):
# [batch,1,cand_sub]
subject_score = tf.matmul(
tf.expand_dims(query4subject, 1),
tf.transpose(self.subject_emb, perm=[0, 2, 1]))
# [batch,cand_sub,1]
subject_score = tf.squeeze(subject_score, 1)
# subject_score = tf.transpose(subject_score, [0, 2, 1])
return subject_score, loss_subject
def setup(self, bottom, top):
params = eval(self.param_str)
check_params(params,
nn_root=None,
nn_shape=None,
nn_query_size=1,
nn_num=1,
nn_db_size=np.inf,
nn_ignore=1)
self.params = Map(params)
self.nn_db = DartDB(self.params.nn_root)
self.nn = NN(self.nn_db, self.params.nn_db_size, self.params.nn_ignore)
assert self.params.nn_num <= self.params.nn_query_size
def __init__(self, params, db, nn_db):
self.lock = Lock()
self.db = db
self.cur = db.length
self.im_shape = params['im_shape']
self.nn_shape = params['nn_shape']
self.hist_eq = params['hist_eq']
self.indexes = np.arange(db.length)
self.shuffle = params['shuffle']
self.subtract_mean = params['subtract_mean']
if self.subtract_mean:
self.mean_img = self.db.read_mean_img(self.im_shape)
self.im_shape = params['im_shape']
self.load_nn = params['load_nn']
self.nn_query_size = params['nn_query_size']
if self.load_nn:
self.nn_db = nn_db
#nn_ignore = 1 if db.db_root == nn_db.db_root else 0
nn_ignore = 0
self.nn = NN(nn_db, params['nn_db_size'], nn_ignore)
def relation_network(self, x):
'''
:param x: query embedding after look-up
:return: rel_score:[batch_size,relation_voc_size], loss:[1]
'''
with tf.variable_scope("relation_network"):
_, query4relation = NN.bi_gru(x, self.x_lens, self.n_hidden,
"bi_gru4relation_query",
self.keep_prob, self.is_training)
# [batch_size,n_hidden*2]
query4relation = tf.concat(query4relation, -1)
# [batch,relation_voc_size]
rel_score = self.matmul_query_relation(
query4relation, self.weight['relationEmbedding'], "rel_score")
self.only_rel_predict = tf.argmax(rel_score, 1)
# [1]
loss_relation = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
labels=self.relation_index, logits=rel_score),
axis=0)
return rel_score, loss_relation
def matmul_query_relation(self, query_vec, rel_vec, name):
query_vec = NN.linear(query_vec, int(rel_vec.shape[-1]), name=name)
score = tf.matmul(query_vec, tf.transpose(rel_vec))
# [batch,cand_rel]
return score
class BNN_SVI(BNN):
def __init__(self, dim, act = nn.ReLU(), num_hiddens = [50], conf = dict()):
super(BNN_SVI, self).__init__()
self.dim = dim
self.act = act
self.num_hiddens = num_hiddens
self.num_iters = conf.get('num_iters', 4000)
self.batch_size = conf.get('batch_size', 32)
self.print_every = conf.get('print_every', 100)
self.lr = conf.get('lr', 1e-2)
self.weight_std = conf.get('weight_std', 1.0)
self.noise_level = conf.get('noise_level', None)
self.nn = NN(dim, self.act, self.num_hiddens, nout = 1)
def model(self, X, y):
"""
Normal distribution for weights and bias
"""
num_x = X.shape[0]
priors = dict()
for n, p in self.nn.named_parameters():
priors[n] = pyro.distributions.Normal(loc = torch.zeros_like(p), scale = self.weight_std * torch.ones_like(p)).to_event(1)
lifted_module = pyro.random_module("module", self.nn, priors)
lifted_reg_model = lifted_module()
with pyro.plate("map", len(X), subsample_size = min(num_x, self.batch_size)) as ind:
pred = lifted_reg_model(X[ind]).squeeze(-1)
pyro.sample("obs", pyro.distributions.Normal(pred, self.noise_level), obs = y[ind])
def guide(self, X, y):
priors = dict()
softplus = nn.Softplus()
for n, p in self.nn.named_parameters():
loc = pyro.param("mu_" + n, self.weight_std * torch.randn_like(p))
scale = pyro.param("sigma_" + n, torch.randn_like(p))
priors[n] = pyro.distributions.Normal(loc = loc, scale = softplus(scale)).to_event(1)
lifted_module = pyro.random_module("module", self.nn, priors)
return lifted_module()
def train(self, X, y):
if self.noise_level is None:
print("No noise level provided, use noise_level = 0.05 * y.std()")
self.noise_level = 0.05 * y.std()
num_train = X.shape[0]
y = y.reshape(num_train)
optim = pyro.optim.Adam({"lr":self.lr})
svi = pyro.infer.SVI(self.model, self.guide, optim, loss = pyro.infer.Trace_ELBO())
pyro.clear_param_store()
self.rec = []
for i in range(self.num_iters):
loss = svi.step(X, y)
if (i+1) % self.print_every == 0:
self.rec.append(loss / num_train)
print("[Iteration %05d/%05d] loss: %-4.3f" % (i + 1, self.num_iters, loss / num_train))
def sample(self, num_samples = 1):
nns = [self.guide(None, None) for i in range(num_samples)]
return nns
def sample_predict(self, nns, X):
num_x = X.shape[0]
pred = torch.zeros(len(nns), num_x)
for i in range(len(nns)):
pred[i] = nns[i](X).squeeze()
precs = torch.ones(pred.shape) / (self.noise_level**2)
return pred
def report(self):
print(self.nn)
