def __init__(self, dim, act = nn.ReLU(), num_hiddens = [50], conf = dict()): super(BNN_SVI, self).__init__() self.dim = dim self.act = act self.num_hiddens = num_hiddens self.num_iters = conf.get('num_iters', 4000) self.batch_size = conf.get('batch_size', 32) self.print_every = conf.get('print_every', 100) self.lr = conf.get('lr', 1e-2) self.weight_std = conf.get('weight_std', 1.0) self.noise_level = conf.get('noise_level', None) self.nn = NN(dim, self.act, self.num_hiddens, nout = 1)
class DataLoader(object): def __init__(self, params, db, nn_db): self.lock = Lock() self.db = db self.cur = db.length self.im_shape = params['im_shape'] self.nn_shape = params['nn_shape'] self.hist_eq = params['hist_eq'] self.indexes = np.arange(db.length) self.shuffle = params['shuffle'] self.subtract_mean = params['subtract_mean'] if self.subtract_mean: self.mean_img = self.db.read_mean_img(self.im_shape) self.im_shape = params['im_shape'] self.load_nn = params['load_nn'] self.nn_query_size = params['nn_query_size'] if self.load_nn: self.nn_db = nn_db #nn_ignore = 1 if db.db_root == nn_db.db_root else 0 nn_ignore = 0 self.nn = NN(nn_db, params['nn_db_size'], nn_ignore) def load_next_data(self): nid = self.get_next_id() jp, imgs, segs = self.db.read_instance(nid, size=self.im_shape) item = {'jp':jp} for i in xrange(len(imgs)): img = imgs[i] if self.hist_eq: img = correct_hist(img) item.update({'img_' + shape_str(self.im_shape[i]):img.transpose((2,0,1)), 'seg_' + shape_str(self.im_shape[i]): segs[i]}) if self.load_nn: nn_id = self.nn.nn_ids(jp, self.nn_query_size) if hasattr(nn_id, '__len__'): nn_id = random.choice(nn_id) nn_jp, nn_imgs, nn_segs = self.nn_db.read_instance(nn_id, size=self.nn_shape) item.update({'nn_jp':nn_jp}) for i in xrange(len(nn_imgs)): nn_img = nn_imgs[i] if self.hist_eq: nn_img = correct_hist(nn_img) item.update({'nn_img_' + shape_str(self.nn_shape[i]):nn_img.transpose((2,0,1)), 'nn_seg_' + shape_str(self.nn_shape[i]): nn_segs[i]}) return item def get_next_id(self): self.lock.acquire() if self.cur >= len(self.indexes) - 1: self.cur = 0 if self.shuffle: random.shuffle(self.indexes) else: self.cur += 1 self.lock.release() return self.indexes[self.cur]
class EKFNNLayer(caffe.Layer): def setup(self, bottom, top): params = eval(self.param_str) check_params(params, nn_root=None, nn_shape=None, nn_query_size=1, nn_num=1, nn_db_size=np.inf, nn_ignore=1) self.params = Map(params) self.nn_db = DartDB(self.params.nn_root) self.nn = NN(self.nn_db, self.params.nn_db_size, self.params.nn_ignore) assert self.params.nn_num <= self.params.nn_query_size def reshape(self, bottom, top): #Reshape tops batch_size = bottom[0].shape[0] assert self.nn_db.jps.shape[1] == bottom[0].shape[1] cur_top = 0 for nn_id in range(self.params.nn_num): top[cur_top + 0].reshape(batch_size, 3, self.params.nn_shape[0], self.params.nn_shape[1]) top[cur_top + 1].reshape(batch_size, 1, self.params.nn_shape[0], self.params.nn_shape[1]) top[cur_top + 2].reshape(batch_size, self.nn_db.jps.shape[1]) top[cur_top + 3].reshape(batch_size, 1) cur_top += 4 #self.top_names.extend(['nn_img_' + str(nn_id), 'nn_seg_' + str(nn_id)]) #self.top_names.append('nn_jp_' + str(nn_id)) #self.top_names.append('nn_w_' + str(nn_id)) def forward(self, bottom, top): for itt in range(bottom[0].shape[0]): jp = bottom[0].data[itt] nn_ids = self.nn.nn_ids(jp, self.params.nn_query_size) if hasattr(nn_ids, '__len__'): nn_ids = np.random.choice(nn_ids, size=self.params.nn_num, replace=False) else: nn_ids = [nn_ids] for i in range(len(nn_ids)): nn_id = nn_ids[i] nn_jp, nn_img, nn_seg = self.nn_db.read_instance(nn_id, size=self.params.nn_shape) top[i * 4 + 0].data[itt, ...] = nn_img[0].transpose((2,0,1)) top[i * 4 + 1].data[itt, ...] = nn_seg[0] top[i * 4 + 2].data[itt, ...] = nn_jp top[i * 4 + 3].data[itt, ...] = 1 def backward(self, top, propagate_down, bottom): bottom[0].diff[...] = 0 def forward_jv(self, top, bottom): for top_id in range(self.params.nn_num * 4): top[top_id].diff[...] = 0
def build_network(self): # Define weights weights = { 'wordEmbedding': tf.Variable(self.qa.word_embedding, trainable=True, name="word_embeddding"), 'crf_W': tf.get_variable("crf_W", initializer=tf.random_uniform( shape=[2 * self.n_hidden, self.n_tags], minval=-0.08, maxval=0.08)), 'crf_b': tf.get_variable("crf_b", initializer=tf.random_uniform(shape=[self.n_tags], minval=-0.08, maxval=0.08)) } self.is_training = tf.placeholder(tf.bool, name="is_training") self.question_ids = tf.placeholder(tf.int32, [None, self.n_steps]) self.x_lens = tf.placeholder(tf.int32, [None]) self.y = tf.placeholder(tf.int32, [None, self.n_steps]) x = tf.nn.embedding_lookup(weights['wordEmbedding'], ids=self.question_ids) query4ner, _ = NN.bi_gru(x, self.x_lens, self.n_hidden, "query4ner", self.keep_prob, self.is_training) crf_x = tf.concat(query4ner, 2) matricized_x_t = tf.reshape( crf_x, [-1, 2 * self.n_hidden]) # maybe it is reasonable but pass it now! matricized_unary_scores = tf.matmul( matricized_x_t, weights['crf_W']) + weights['crf_b'] self.unary_scores = tf.reshape(matricized_unary_scores, [-1, self.n_steps, self.n_tags]) log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood( self.unary_scores, self.y, self.x_lens) self.loss = tf.reduce_mean(-log_likelihood) all_var = tf.global_variables() grads = tf.gradients(self.loss, all_var) grads = [tf.clip_by_value(grad, -10, 10) for grad in grads] self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.9).apply_gradients(zip(grads, all_var)) self.saver = tf.train.Saver()
def subject_network(self, x): ''' :param x: query embedding after look-up :return:subject_score:[batch_size,cand_sub_size] loss_subject:[1] ''' with tf.variable_scope("subject_part"): _, query4subject = NN.bi_gru(x, self.x_lens, self.n_hidden, "bi_gru4subject_query", self.keep_prob, self.is_training) query4subject = tf.concat(query4subject, -1) # [batch,type_len] query4subject = tf.sigmoid( NN.linear(query4subject, self.qa.type_len, name="query_trans_subject")) with tf.variable_scope(tf.get_variable_scope(), reuse=True): self.type_emb_w_martix = tf.get_variable( "query_trans_subjectkernel") self.type_emb_b_martix = tf.get_variable( "query_trans_subjectbias") # [batch_size,emb_size] gold_subject = self.subject_emb[:, 0, :] self.loss_subject = gold_subject * tf.log( tf.clip_by_value( query4subject, 1e-10, 1.0)) + (1 - gold_subject) * tf.log( tf.clip_by_value(1 - query4subject, 1e-10, 1.0)) # [1] loss_subject = tf.reduce_mean( -tf.reduce_sum(self.loss_subject, axis=1), axis=0) with tf.variable_scope("test"): # [batch,1,cand_sub] subject_score = tf.matmul( tf.expand_dims(query4subject, 1), tf.transpose(self.subject_emb, perm=[0, 2, 1])) # [batch,cand_sub,1] subject_score = tf.squeeze(subject_score, 1) # subject_score = tf.transpose(subject_score, [0, 2, 1]) return subject_score, loss_subject
def setup(self, bottom, top): params = eval(self.param_str) check_params(params, nn_root=None, nn_shape=None, nn_query_size=1, nn_num=1, nn_db_size=np.inf, nn_ignore=1) self.params = Map(params) self.nn_db = DartDB(self.params.nn_root) self.nn = NN(self.nn_db, self.params.nn_db_size, self.params.nn_ignore) assert self.params.nn_num <= self.params.nn_query_size
def __init__(self, params, db, nn_db): self.lock = Lock() self.db = db self.cur = db.length self.im_shape = params['im_shape'] self.nn_shape = params['nn_shape'] self.hist_eq = params['hist_eq'] self.indexes = np.arange(db.length) self.shuffle = params['shuffle'] self.subtract_mean = params['subtract_mean'] if self.subtract_mean: self.mean_img = self.db.read_mean_img(self.im_shape) self.im_shape = params['im_shape'] self.load_nn = params['load_nn'] self.nn_query_size = params['nn_query_size'] if self.load_nn: self.nn_db = nn_db #nn_ignore = 1 if db.db_root == nn_db.db_root else 0 nn_ignore = 0 self.nn = NN(nn_db, params['nn_db_size'], nn_ignore)
def relation_network(self, x): ''' :param x: query embedding after look-up :return: rel_score:[batch_size,relation_voc_size], loss:[1] ''' with tf.variable_scope("relation_network"): _, query4relation = NN.bi_gru(x, self.x_lens, self.n_hidden, "bi_gru4relation_query", self.keep_prob, self.is_training) # [batch_size,n_hidden*2] query4relation = tf.concat(query4relation, -1) # [batch,relation_voc_size] rel_score = self.matmul_query_relation( query4relation, self.weight['relationEmbedding'], "rel_score") self.only_rel_predict = tf.argmax(rel_score, 1) # [1] loss_relation = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=self.relation_index, logits=rel_score), axis=0) return rel_score, loss_relation
def matmul_query_relation(self, query_vec, rel_vec, name): query_vec = NN.linear(query_vec, int(rel_vec.shape[-1]), name=name) score = tf.matmul(query_vec, tf.transpose(rel_vec)) # [batch,cand_rel] return score
class BNN_SVI(BNN): def __init__(self, dim, act = nn.ReLU(), num_hiddens = [50], conf = dict()): super(BNN_SVI, self).__init__() self.dim = dim self.act = act self.num_hiddens = num_hiddens self.num_iters = conf.get('num_iters', 4000) self.batch_size = conf.get('batch_size', 32) self.print_every = conf.get('print_every', 100) self.lr = conf.get('lr', 1e-2) self.weight_std = conf.get('weight_std', 1.0) self.noise_level = conf.get('noise_level', None) self.nn = NN(dim, self.act, self.num_hiddens, nout = 1) def model(self, X, y): """ Normal distribution for weights and bias """ num_x = X.shape[0] priors = dict() for n, p in self.nn.named_parameters(): priors[n] = pyro.distributions.Normal(loc = torch.zeros_like(p), scale = self.weight_std * torch.ones_like(p)).to_event(1) lifted_module = pyro.random_module("module", self.nn, priors) lifted_reg_model = lifted_module() with pyro.plate("map", len(X), subsample_size = min(num_x, self.batch_size)) as ind: pred = lifted_reg_model(X[ind]).squeeze(-1) pyro.sample("obs", pyro.distributions.Normal(pred, self.noise_level), obs = y[ind]) def guide(self, X, y): priors = dict() softplus = nn.Softplus() for n, p in self.nn.named_parameters(): loc = pyro.param("mu_" + n, self.weight_std * torch.randn_like(p)) scale = pyro.param("sigma_" + n, torch.randn_like(p)) priors[n] = pyro.distributions.Normal(loc = loc, scale = softplus(scale)).to_event(1) lifted_module = pyro.random_module("module", self.nn, priors) return lifted_module() def train(self, X, y): if self.noise_level is None: print("No noise level provided, use noise_level = 0.05 * y.std()") self.noise_level = 0.05 * y.std() num_train = X.shape[0] y = y.reshape(num_train) optim = pyro.optim.Adam({"lr":self.lr}) svi = pyro.infer.SVI(self.model, self.guide, optim, loss = pyro.infer.Trace_ELBO()) pyro.clear_param_store() self.rec = [] for i in range(self.num_iters): loss = svi.step(X, y) if (i+1) % self.print_every == 0: self.rec.append(loss / num_train) print("[Iteration %05d/%05d] loss: %-4.3f" % (i + 1, self.num_iters, loss / num_train)) def sample(self, num_samples = 1): nns = [self.guide(None, None) for i in range(num_samples)] return nns def sample_predict(self, nns, X): num_x = X.shape[0] pred = torch.zeros(len(nns), num_x) for i in range(len(nns)): pred[i] = nns[i](X).squeeze() precs = torch.ones(pred.shape) / (self.noise_level**2) return pred def report(self): print(self.nn)