class FastZeroTagModel(AttalosModel): """ Create a tensorflow graph that finds the principal direction of the target word embeddings (with negative sampling), using the loss function from "Fast Zero-Shot Image Tagging". """ def __init__(self, wv_model, datasets, **kwargs): self.wv_model = wv_model self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) word_counts = NegativeSampler.get_wordcount_from_datasets(datasets, self.one_hot) self.negsampler = NegativeSampler(word_counts) self.w = construct_W(wv_model, self.one_hot.get_key_ordering()).T self.learning_rate = kwargs.get("learning_rate", 0.0001) self.optim_words = kwargs.get("optim_words", True) self.hidden_units = kwargs.get("hidden_units", "200") self.use_batch_norm = kwargs.get("use_batch_norm",False) self.opt_type = kwargs.get("opt_type","adam") if self.hidden_units=='0': self.hidden_units=[] else: self.hidden_units = [int(x) for x in self.hidden_units.split(',')] self.model_info = dict() # Placeholders for data self.model_info['input'] = tf.placeholder(shape=(None, datasets[0].img_feat_size), dtype=tf.float32) self.model_info['pos_ids'] = tf.placeholder(dtype=tf.int32) self.model_info['neg_ids'] = tf.placeholder(dtype=tf.int32) self.model_info['w2v'] = tf.Variable(self.w, dtype=tf.float32) self.model_info['pos_vecs'] = tf.transpose(tf.nn.embedding_lookup(self.model_info['w2v'], self.model_info['pos_ids']), perm=[1,0,2]) self.model_info['neg_vecs'] = tf.transpose(tf.nn.embedding_lookup(self.model_info['w2v'], self.model_info['neg_ids']), perm=[1,0,2]) # Construct fully connected layers layer = self.model_info['input'] layers = [] for i, hidden_size in enumerate(self.hidden_units): layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) if self.use_batch_norm: layer = tf.contrib.layers.batch_norm(layer) layers.append(layer) logger.info("Using batch normalization") # Output layer should always be linear layer = tf.contrib.layers.linear(layer, self.w.shape[1]) layers.append(layer) self.model_info['layers'] = layers self.model_info['prediction'] = layer def fztloss( f, pVecs, nVecs ): """ Tensorized cost function from Fast Zero-Shot Learning paper Args: f: The output from the network, a tensor of shape (# images, word embedding size) pVecs: The vector embeddings of the ground truth tags, a tensor of shape (# images, # positive tags, word embedding size) nVecs: The vector embeddings of negatively sampled tags, a tensor of shape (# images, # negative samples, word embedding size) Returns: Scalar tensor representing the batch cost """ posmul = tf.mul(pVecs, f) negmul = tf.mul(nVecs, f) tfpos = tf.reduce_sum(posmul, reduction_indices=2) tfneg = tf.reduce_sum(negmul, reduction_indices=2) tfpos = tf.transpose(tfpos, [1,0]) tfneg = tf.transpose(tfneg, [1,0]) negexpan = tf.tile( tf.expand_dims(tfneg, -1), [1, 1, tf.shape(tfpos)[1]] ) posexpan = tf.tile( tf.transpose(tf.expand_dims(tfpos, -1), [0,2,1]), [1, tf.shape(tfneg)[1], 1]) differences = tf.sub(negexpan, posexpan) return tf.reduce_sum(tf.reduce_sum(tf.log(1 + tf.exp(differences)), reduction_indices=[1,2])) loss = fztloss(self.model_info['prediction'], self.model_info['pos_vecs'], self.model_info['neg_vecs']) self.model_info['loss'] = loss if self.opt_type=='sgd': optimizer=tf.train.GradientDescent else: optimizer=tf.train.AdamOptimizer self.model_info['optimizer'] = optimizer(learning_rate=self.learning_rate).minimize(loss) def predict_feats(self, sess, x): return sess.run(self.model_info['prediction'], feed_dict={self.model_info['input']: x}) def _get_ids(self, tag_ids, numSamps=[5, 10], uniform_sampling=False): """ Takes a batch worth of text tags and returns positive/negative ids """ pos_word_ids = np.ones((len(tag_ids), numSamps[0]), dtype=np.int32) pos_word_ids.fill(-1) for ind, tags in enumerate(tag_ids): if len(tags) > 0: pos_word_ids[ind] = np.random.choice(tags, size=numSamps[0]) neg_word_ids = None if uniform_sampling: neg_word_ids = np.random.randint(0, self.one_hot.vocab_size, size=(len(tag_ids), numSamps[1])) else: neg_word_ids = np.ones((len(tag_ids), numSamps[1]), dtype=np.int32) neg_word_ids.fill(-1) for ind in range(pos_word_ids.shape[0]): # TODO: Check to see if this benefits from the same bug as negsampling code neg_word_ids[ind] = self.negsampler.negsamp_ind(pos_word_ids[ind], numSamps[1]) return pos_word_ids, neg_word_ids def prep_fit(self, data): img_feats, text_feats_list = data text_feat_ids = [] for tags in text_feats_list: text_feat_ids.append([self.one_hot.get_index(tag) for tag in tags if tag in self.one_hot]) pos_ids, neg_ids = self._get_ids(text_feat_ids) self.pos_ids = pos_ids self.neg_ids = neg_ids fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_ids"]: pos_ids, self.model_info["neg_ids"]: neg_ids } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): if cross_eval: self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_w = construct_W(self.wv_model, self.test_one_hot.get_key_ordering()).T else: self.test_one_hot = self.one_hot self.test_w = self.w x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [self.model_info["prediction"], ] feed_dict = { self.model_info["input"]: x } truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): predictions = predict_fetches[0] if cross_eval and self.test_w is None: raise Exception("test_w is not set. Did you call prep_predict?") predictions = np.dot(predictions, self.test_w.T) return predictions def get_training_loss(self, fit_fetches): return fit_fetches[1]
class MultihotModel(AttalosModel): """ This model performs logistic regression via NN using multihot vectors as targets. """ def _construct_model_info(self, input_size, output_size, learning_rate): model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) model_info["y_truth"] = tf.placeholder(shape=(None, output_size), dtype=tf.float32) model_info["predictions"] = tf.contrib.layers.fully_connected(model_info["input"], output_size, activation_fn=tf.sigmoid) model_info["loss"] = tf.reduce_sum(tf.square(model_info["predictions"] - model_info["y_truth"])) model_info["optimizer"] = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(model_info["loss"]) return model_info def __init__(self, wv_model, datasets, **kwargs): #train_dataset, test_dataset, **kwargs): #self.cross_eval = kwargs.get("cross_eval", False) self.wv_model = wv_model #self.one_hot = OneHot([train_dataset] if self.cross_eval else [train_dataset, test_dataset], # valid_vocab=wv_model.vocab) self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) train_dataset = datasets[0] # train_dataset should always be first in datasets iterable self.learning_rate = kwargs.get("learning_rate", 0.0001) self.model_info = self._construct_model_info( input_size = train_dataset.img_feat_size, output_size = self.one_hot.vocab_size, learning_rate = self.learning_rate ) self.test_one_hot = None super(MultihotModel, self).__init__() def prep_fit(self, data): img_feats_list, text_feats_list = data img_feats = np.array(img_feats_list) text_feats = [self.one_hot.get_multiple(text_feats) for text_feats in text_feats_list] text_feats = np.array(text_feats) fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["y_truth"]: text_feats } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): if cross_eval: raise Exception("cross_eval is True. Multihot model does not support cross evaluation.") x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [self.model_info["predictions"],] feed_dict = { self.model_info["input"]: x } truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): if cross_eval: raise Exception("cross_eval is True. Multihot model does not support cross evaluation.") predictions = predict_fetches[0] return predictions def get_training_loss(self, fit_fetches): _, loss = fit_fetches return loss """
class NegSamplingModel(AttalosModel): """ This model performs negative sampling. """ def _construct_model_info(self, input_size, output_size, learning_rate, wv_arr, hidden_units=[200], optim_words=True, opt_type='adam', use_batch_norm=True, weight_decay=0.0): model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) if optim_words: model_info["pos_vecs"] = tf.placeholder(dtype=tf.float32) model_info["neg_vecs"] = tf.placeholder(dtype=tf.float32) logger.info("Optimization on GPU, word vectors are stored separately.") else: model_info["w2v"] = tf.Variable(wv_arr, dtype=tf.float32) model_info["pos_ids"] = tf.placeholder(dtype=tf.int32) model_info["neg_ids"] = tf.placeholder(dtype=tf.int32) model_info["pos_vecs"] = tf.transpose(tf.nn.embedding_lookup(model_info["w2v"], model_info["pos_ids"]), perm=[1,0,2]) model_info["neg_vecs"] = tf.transpose(tf.nn.embedding_lookup(model_info["w2v"], model_info["neg_ids"]), perm=[1,0,2]) logger.info("Not optimizing word vectors.") # Construct fully connected layers layers = [] layer = model_info["input"] for i, hidden_size in enumerate(hidden_units): layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) if use_batch_norm: layer = tf.contrib.layers.batch_norm(layer) layers.append(layer) # Output layer should always be linear layer = tf.contrib.layers.linear(layer, wv_arr.shape[1]) layers.append(layer) model_info["layers"] = layers model_info["prediction"] = layer def meanlogsig(predictions, truth): reduction_indices = 2 return tf.reduce_mean(tf.log(tf.sigmoid(tf.reduce_sum(predictions * truth, reduction_indices=reduction_indices)))) pos_loss = meanlogsig(model_info["prediction"], model_info["pos_vecs"]) neg_loss = meanlogsig(-model_info["prediction"], model_info["neg_vecs"]) model_info["loss"] = -(pos_loss + neg_loss) # Decide whether or not to use SGD or Adam Optimizers if self.opt_type == 'sgd': logger.info("Optimization uses SGD with non-variable rate") optimizer = tf.train.GradientDescentOptimizer else: logger.info("Optimization uses Adam") optimizer = tf.train.AdamOptimizer # Are we manually decaying the words? Create a TF variable in that case. if weight_decay: logger.info("Learning rate is manually adaptive, dropping every ten (hard coded) epoch") model_info['learning_rate'] = tf.placeholder(tf.float32, shape=[]) else: model_info['learning_rate'] = learning_rate model_info["optimizer"] = optimizer(learning_rate=model_info['learning_rate']).minimize(model_info["loss"]) #model_info["init_op"] = tf.initialize_all_variables() #model_info["saver"] = tf.train.Saver() return model_info def __init__(self, wv_model, datasets, **kwargs): self.wv_model = wv_model self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) word_counts = NegativeSampler.get_wordcount_from_datasets(datasets, self.one_hot) self.negsampler = NegativeSampler(word_counts) train_dataset = datasets[0] # train_dataset should always be first in datasets self.w = construct_W(wv_model, self.one_hot.get_key_ordering()).T scale_words = kwargs.get("scale_words",1.0) if scale_words == 0.0: self.w = (self.w.T / np.linalg.norm(self.w,axis=1)).T else: self.w *= scale_words # Optimization parameters # Starting learning rate, currently default to 0.001. This will change iteratively if decay is on. self.learning_rate = kwargs.get("learning_rate", 0.0001) self.weight_decay = kwargs.get("weight_decay", 0.0) self.optim_words = kwargs.get("optim_words", True) self.epoch_num = 0 # Sampling methods self.ignore_posbatch = kwargs.get("ignore_posbatch",False) self.joint_factor = kwargs.get("joint_factor",1.0) self.hidden_units = kwargs.get("hidden_units", "200") if self.hidden_units=='0': self.hidden_units=[] else: self.hidden_units = [int(x) for x in self.hidden_units.split(",")] self.opt_type = kwargs.get("opt_type", "adam") self.use_batch_norm = kwargs.get('use_batch_norm',False) self.model_info = self._construct_model_info( input_size = train_dataset.img_feat_size, output_size = self.one_hot.vocab_size, hidden_units=self.hidden_units, learning_rate = self.learning_rate, optim_words = self.optim_words, use_batch_norm = self.use_batch_norm, wv_arr = self.w, weight_decay = self.weight_decay ) self.test_one_hot = None self.test_w = None super(NegSamplingModel, self).__init__() def iter_batches(self, dataset, batch_size): for x, y in super(NegSamplingModel, self).iter_batches(dataset, batch_size): yield x, y # This will decay the learning rate every ten epochs. Hardcoded ten currently... if self.weight_decay: if self.epoch_num and self.epoch_num % 15 == 0 and self.learning_rate > 1e-6: self.learning_rate *= self.weight_decay logger.info('Learning rate dropped to {}'.format(self.learning_rate)) self.epoch_num+=1 def _get_ids(self, tag_ids, numSamps=[5, 10], uniform_sampling=False): """ Takes a batch worth of text tags and returns positive/negative ids """ pos_word_ids = np.ones((len(tag_ids), numSamps[0]), dtype=np.int32) pos_word_ids.fill(-1) for ind, tags in enumerate(tag_ids): if len(tags) > 0: pos_word_ids[ind] = np.random.choice(tags, size=numSamps[0]) neg_word_ids = None if uniform_sampling: neg_word_ids = np.random.randint(0, self.one_hot.vocab_size, size=(len(tag_ids), numSamps[1])) else: neg_word_ids = np.ones((len(tag_ids), numSamps[1]), dtype=np.int32) neg_word_ids.fill(-1) for ind in range(pos_word_ids.shape[0]): if self.ignore_posbatch: # NOTE: This function call should definitely be pos_word_ids[ind] # but that results in significantly worse performance # I wish I understood why. # I think this means we won't sample any tags that appear in the batch neg_word_ids[ind] = self.negsampler.negsamp_ind(pos_word_ids, numSamps[1]) else: neg_word_ids[ind] = self.negsampler.negsamp_ind(pos_word_ids[ind], numSamps[1]) return pos_word_ids, neg_word_ids def prep_fit(self, data): img_feats, text_feats_list = data text_feat_ids = [] for tags in text_feats_list: text_feat_ids.append([self.one_hot.get_index(tag) for tag in tags if tag in self.one_hot]) pos_ids, neg_ids = self._get_ids(text_feat_ids) self.pos_ids = pos_ids self.neg_ids = neg_ids if not self.optim_words: fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_ids"]: pos_ids, self.model_info["neg_ids"]: neg_ids } else: pvecs = np.zeros((pos_ids.shape[0], pos_ids.shape[1], self.w.shape[1])) nvecs = np.zeros((neg_ids.shape[0], neg_ids.shape[1], self.w.shape[1])) for i, ids in enumerate(pos_ids): pvecs[i] = self.w[ids] for i, ids in enumerate(neg_ids): nvecs[i] = self.w[ids] pvecs = pvecs.transpose((1, 0, 2)) nvecs = nvecs.transpose((1, 0, 2)) fetches = [self.model_info["optimizer"], self.model_info["loss"], self.model_info["prediction"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_vecs"]: pvecs, self.model_info["neg_vecs"]: nvecs } if self.weight_decay: feed_dict[self.model_info['learning_rate']] = self.learning_rate return fetches, feed_dict def _updatewords(self, vpindex, vnindex, vin): for i, (vpi, vni) in enumerate(zip(vpindex, vnindex)): self.w[vpi] += self.joint_factor*self.learning_rate * np.outer(1 - sigmoid(self.w[vpi].dot(vin[i])), vin[i]) self.w[vni] -= self.joint_factor*self.learning_rate * np.outer(sigmoid(self.w[vni].dot(vin[i])), vin[i]) def fit(self, sess, fetches, feed_dict): fit_fetches = super(NegSamplingModel, self).fit(sess, fetches, feed_dict) if self.optim_words: if self.pos_ids is None or self.neg_ids is None: raise Exception("pos_ids or neg_ids is not set; cannot update word vectors. Did you run prep_fit()?") _, _, prediction = fit_fetches self._updatewords(self.pos_ids, self.neg_ids, prediction) return fit_fetches def prep_predict(self, dataset, cross_eval=False): if cross_eval: self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_w = construct_W(self.wv_model, self.test_one_hot.get_key_ordering()).T else: self.test_one_hot = self.one_hot self.test_w = self.w x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [self.model_info["prediction"], ] feed_dict = { self.model_info["input"]: x } truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): predictions = predict_fetches[0] if cross_eval and self.test_w is None: raise Exception("test_w is not set. Did you call prep_predict?") predictions = np.dot(predictions, self.test_w.T) return predictions def get_training_loss(self, fit_fetches): return fit_fetches[1]
class NaiveSumModel(AttalosModel): """ This model performs linear regression via NN using the naive sum of word vectors as targets. """ def _construct_model_info(self, input_size, output_size, learning_rate, hidden_units=[200]): logger.info("Input size: %s" % input_size) logger.info("Output size: %s" % output_size) model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) model_info["y_truth"] = tf.placeholder(shape=(None, output_size), dtype=tf.float32) layers = [] layer = model_info["input"] for i, hidden_size in enumerate(hidden_units): layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) model_info["predictions"] = tf.contrib.layers.fully_connected(layer, output_size, activation_fn=None) model_info["loss"] = tf.reduce_sum(tf.square(model_info["predictions"] - model_info["y_truth"])) model_info["optimizer"] = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(model_info["loss"]) return model_info def __init__(self, wv_model, datasets, **kwargs): self.wv_model = wv_model #self.cross_eval = kwargs.get("cross_eval", False) #self.one_hot = OneHot([train_dataset] if self.cross_eval else [train_dataset, test_dataset], # valid_vocab=wv_model.vocab) self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) self.wv_transformer = NaiveW2V.create_from_vocab(wv_model, self.one_hot, vocab=self.one_hot.get_key_ordering()) train_dataset = datasets[0] # train_dataset should always be first in datasets iterable self.learning_rate = kwargs.get("learning_rate", 0.0001) self.hidden_units = kwargs.get("hidden_units", "200") if self.hidden_units == '0': self.hidden_units = [] else: self.hidden_units = [int(x) for x in self.hidden_units.split(",")] self.model_info = self._construct_model_info( input_size = train_dataset.img_feat_size, output_size = self.wv_model.get_word_vector_shape()[0], learning_rate = self.learning_rate, hidden_units=self.hidden_units, ) self.test_one_hot = None super(NaiveSumModel, self).__init__() def prep_fit(self, data): img_feats_list, text_feats_list = data img_feats = np.array(img_feats_list) text_feats = [self.one_hot.get_multiple(text_feats) for text_feats in text_feats_list] text_feats = np.array(text_feats) text_feats = self.wv_transformer.transform(text_feats) fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["y_truth"]: text_feats } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): if self.test_one_hot is None or self.test_dataset is dataset: self.test_dataset = dataset self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_w = construct_W(self.wv_model, self.test_one_hot.get_key_ordering()).T x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.test_one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) truth = np.asarray(y) fetches = [self.model_info["predictions"], ] feed_dict = { self.model_info["input"]: x } return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): if self.test_one_hot is None: raise Exception("test_one_hot is not set. Did you call prep_predict to initialize it?") predictions = predict_fetches[0] predictions = np.dot(predictions, construct_W(self.wv_model, self.test_one_hot.get_key_ordering())) return predictions def get_training_loss(self, fit_fetches): _, loss = fit_fetches return loss
class MultihotModel(AttalosModel): """ This model performs logistic regression via NN using multihot vectors as targets. """ def _construct_model_info(self, input_size, output_size, learning_rate): model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) model_info["y_truth"] = tf.placeholder(shape=(None, output_size), dtype=tf.float32) model_info["predictions"] = tf.contrib.layers.fully_connected( model_info["input"], output_size, activation_fn=tf.sigmoid) model_info["loss"] = tf.reduce_sum( tf.square(model_info["predictions"] - model_info["y_truth"])) model_info["optimizer"] = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(model_info["loss"]) return model_info def __init__(self, wv_model, datasets, **kwargs): #train_dataset, test_dataset, **kwargs): #self.cross_eval = kwargs.get("cross_eval", False) self.wv_model = wv_model #self.one_hot = OneHot([train_dataset] if self.cross_eval else [train_dataset, test_dataset], # valid_vocab=wv_model.vocab) self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) train_dataset = datasets[ 0] # train_dataset should always be first in datasets iterable self.learning_rate = kwargs.get("learning_rate", 0.0001) self.model_info = self._construct_model_info( input_size=train_dataset.img_feat_size, output_size=self.one_hot.vocab_size, learning_rate=self.learning_rate) self.test_one_hot = None super(MultihotModel, self).__init__() def prep_fit(self, data): img_feats_list, text_feats_list = data img_feats = np.array(img_feats_list) text_feats = [ self.one_hot.get_multiple(text_feats) for text_feats in text_feats_list ] text_feats = np.array(text_feats) fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["y_truth"]: text_feats } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): if cross_eval: raise Exception( "cross_eval is True. Multihot model does not support cross evaluation." ) x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [ self.model_info["predictions"], ] feed_dict = {self.model_info["input"]: x} truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): if cross_eval: raise Exception( "cross_eval is True. Multihot model does not support cross evaluation." ) predictions = predict_fetches[0] return predictions def get_training_loss(self, fit_fetches): _, loss = fit_fetches return loss """
class WDVModel(AttalosModel): """ This model performs logistic regression via NN using word distribution vectors (correlation vectors) as targets. """ def _construct_model_info(self, input_size, output_size, learning_rate, hidden_units=[200]): logger.info("Input size: %s" % input_size) logger.info("Output size: %s" % output_size) model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) model_info["y_truth"] = tf.placeholder(shape=(None, output_size), dtype=tf.float32) layers = [] layer = model_info["input"] for i, hidden_size in enumerate(hidden_units): layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) model_info["predictions"] = tf.contrib.layers.fully_connected(layer, output_size, activation_fn=tf.sigmoid) model_info["loss"] = tf.reduce_sum(tf.square(model_info["predictions"] - model_info["y_truth"])) model_info["optimizer"] = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(model_info["loss"]) return model_info def __init__(self, wv_model, datasets, **kwargs): #train_dataset, test_dataset, **kwargs): self.wv_model = wv_model #self.cross_eval = kwargs.get("cross_eval", False) #self.one_hot = OneHot([train_dataset] if self.cross_eval else [train_dataset, test_dataset], # valid_vocab=wv_model.vocab) self.one_hot = OneHot(datasets, valid_vocab=self.wv_model.vocab) train_dataset = datasets[0] # train_dataset should always be first in datasets iterable self.wv_transformer = WDV.create_from_vocab(wv_model, vocab1=self.one_hot.get_key_ordering(), preprocess_fn=WDV.preprocess) self.learning_rate = kwargs.get("learning_rate", 0.0001) self.hidden_units = kwargs.get("hidden_units", "200") if self.hidden_units == '0': self.hidden_units = [] else: self.hidden_units = [int(x) for x in self.hidden_units.split(",")] self.model_info = self._construct_model_info( input_size = train_dataset.img_feat_size, output_size = self.one_hot.vocab_size, #self.wv_model.get_word_vector_shape()[0], learning_rate = self.learning_rate, hidden_units=self.hidden_units, ) self.transform_cache = {} self.test_one_hot = None self.test_wv_transformer = None super(WDVModel, self).__init__() def generate_key(self, word_list): return " ".join(sorted(word_list)) def add_cache(self, word_list, val): key = self.generate_key(word_list) self.transform_cache[key] = val def get_cache(self, word_list): key = self.generate_key(word_list) if key in self.transform_cache: return self.transform_cache[key] else: return None def prep_fit(self, data): img_feats_list, text_feats_list = data img_feats = np.array(img_feats_list) # normalize img_feats # new_img_feats = (new_img_feats.T / np.linalg.norm(new_img_feats, axis=1)).T new_text_feats_list = [] for text_feats in text_feats_list: new_text_feats = self.get_cache(text_feats) if new_text_feats is None: new_text_feats = [self.one_hot.get_multiple(text_feats)] new_text_feats = np.array(new_text_feats) new_text_feats = self.wv_transformer.transform(new_text_feats, postprocess_fn=WDV.postprocess) new_text_feats = new_text_feats[0] # new_text_feats is a list; get first element self.add_cache(text_feats, new_text_feats) new_text_feats_list.append(new_text_feats) text_feats = np.array(new_text_feats_list) fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["y_truth"]: text_feats } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): x = [] y = [] if cross_eval: self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_wv_transformer = WDV.create_from_vocab(self.wv_model, vocab1=self.one_hot.get_key_ordering(), vocab2=self.test_one_hot.get_key_ordering(), preprocess_fn=scale3) else: self.test_one_hot = self.one_hot self.test_wv_transformer = None for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.test_one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [self.model_info["predictions"], ] feed_dict = { self.model_info["input"]: x } truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): predictions = predict_fetches[0] if cross_eval: if self.test_wv_transformer is None: raise Exception("test_wv_transformers is not set. Did you call prep_predict?") predictions = np.dot(predictions, self.test_wv_transformer.wdv_arr) return predictions def get_training_loss(self, fit_fetches): _, loss = fit_fetches return loss """
class NegSamplingModel(AttalosModel): """ This model performs negative sampling. """ def _construct_model_info(self, input_size, output_size, learning_rate, wv_arr, hidden_units=[200, 200], optim_words=True, use_batch_norm=True): model_info = {} model_info["input"] = tf.placeholder(shape=(None, input_size), dtype=tf.float32) if optim_words: model_info["pos_vecs"] = tf.placeholder(dtype=tf.float32) model_info["neg_vecs"] = tf.placeholder(dtype=tf.float32) logger.info( "Optimization on GPU, word vectors are stored separately.") else: model_info["w2v"] = tf.Variable(wv_arr, dtype=tf.float32) model_info["pos_ids"] = tf.placeholder(dtype=tf.int32) model_info["neg_ids"] = tf.placeholder(dtype=tf.int32) model_info["pos_vecs"] = tf.transpose(tf.nn.embedding_lookup( model_info["w2v"], model_info["pos_ids"]), perm=[1, 0, 2]) model_info["neg_vecs"] = tf.transpose(tf.nn.embedding_lookup( model_info["w2v"], model_info["neg_ids"]), perm=[1, 0, 2]) logger.info("Not optimizing word vectors.") # Construct fully connected layers layers = [] layer = model_info["input"] for i, hidden_size in enumerate(hidden_units[:-1]): layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) if use_batch_norm: layer = tf.contrib.layers.batch_norm(layer) layers.append(layer) # Output layer should always be linear layer = tf.contrib.layers.linear(layer, wv_arr.shape[1]) layers.append(layer) model_info["layers"] = layers model_info["prediction"] = layer def meanlogsig(predictions, truth): reduction_indices = 2 return tf.reduce_mean( tf.log( tf.sigmoid( tf.reduce_sum(predictions * truth, reduction_indices=reduction_indices)))) pos_loss = meanlogsig(model_info["prediction"], model_info["pos_vecs"]) neg_loss = meanlogsig(-model_info["prediction"], model_info["neg_vecs"]) model_info["loss"] = -(pos_loss + neg_loss) model_info["optimizer"] = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(model_info["loss"]) #model_info["init_op"] = tf.initialize_all_variables() #model_info["saver"] = tf.train.Saver() return model_info def __init__(self, wv_model, datasets, **kwargs): self.wv_model = wv_model self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) word_counts = NegativeSampler.get_wordcount_from_datasets( datasets, self.one_hot) self.negsampler = NegativeSampler(word_counts) train_dataset = datasets[ 0] # train_dataset should always be first in datasets self.w = construct_W(wv_model, self.one_hot.get_key_ordering()).T self.learning_rate = kwargs.get("learning_rate", 0.0001) self.optim_words = kwargs.get("optim_words", True) self.hidden_units = kwargs.get("hidden_units", "200,200") self.hidden_units = [int(x) for x in self.hidden_units.split(",")] self.model_info = self._construct_model_info( input_size=train_dataset.img_feat_size, output_size=self.one_hot.vocab_size, hidden_units=self.hidden_units, learning_rate=self.learning_rate, optim_words=self.optim_words, wv_arr=self.w) self.test_one_hot = None self.test_w = None super(NegSamplingModel, self).__init__() def _get_ids(self, tag_ids, numSamps=[5, 10], uniform_sampling=False): """ Takes a batch worth of text tags and returns positive/negative ids """ pos_word_ids = np.ones((len(tag_ids), numSamps[0]), dtype=np.int32) pos_word_ids.fill(-1) for ind, tags in enumerate(tag_ids): if len(tags) > 0: pos_word_ids[ind] = np.random.choice(tags, size=numSamps[0]) neg_word_ids = None if uniform_sampling: neg_word_ids = np.random.randint(0, self.one_hot.vocab_size, size=(len(tag_ids), numSamps[1])) else: neg_word_ids = np.ones((len(tag_ids), numSamps[1]), dtype=np.int32) neg_word_ids.fill(-1) for ind in range(pos_word_ids.shape[0]): if self.optim_words: neg_word_ids[ind] = self.negsampler.negsamp_ind( pos_word_ids[ind], numSamps[1]) else: # NOTE: This function call should definitely be pos_word_ids[ind] # but that results in significantly worse performance # I wish I understood why. # I think this means we won't sample any tags that appear in the batch neg_word_ids[ind] = self.negsampler.negsamp_ind( pos_word_ids, numSamps[1]) return pos_word_ids, neg_word_ids def prep_fit(self, data): img_feats, text_feats_list = data text_feat_ids = [] for tags in text_feats_list: text_feat_ids.append([ self.one_hot.get_index(tag) for tag in tags if tag in self.one_hot ]) pos_ids, neg_ids = self._get_ids(text_feat_ids) self.pos_ids = pos_ids self.neg_ids = neg_ids if not self.optim_words: fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_ids"]: pos_ids, self.model_info["neg_ids"]: neg_ids } else: pvecs = np.zeros( (pos_ids.shape[0], pos_ids.shape[1], self.w.shape[1])) nvecs = np.zeros( (neg_ids.shape[0], neg_ids.shape[1], self.w.shape[1])) for i, ids in enumerate(pos_ids): pvecs[i] = self.w[ids] for i, ids in enumerate(neg_ids): nvecs[i] = self.w[ids] pvecs = pvecs.transpose((1, 0, 2)) nvecs = nvecs.transpose((1, 0, 2)) fetches = [ self.model_info["optimizer"], self.model_info["loss"], self.model_info["prediction"] ] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_vecs"]: pvecs, self.model_info["neg_vecs"]: nvecs } return fetches, feed_dict def _updatewords(self, vpindex, vnindex, vin): for i, (vpi, vni) in enumerate(zip(vpindex, vnindex)): self.w[vpi] += self.learning_rate * np.outer( 1 - sigmoid(self.w[vpi].dot(vin[i])), vin[i]) self.w[vni] -= self.learning_rate * np.outer( sigmoid(self.w[vni].dot(vin[i])), vin[i]) def fit(self, sess, fetches, feed_dict): fit_fetches = super(NegSamplingModel, self).fit(sess, fetches, feed_dict) if self.optim_words: if self.pos_ids is None or self.neg_ids is None: raise Exception( "pos_ids or neg_ids is not set; cannot update word vectors. Did you run prep_fit()?" ) _, _, prediction = fit_fetches self._updatewords(self.pos_ids, self.neg_ids, prediction) return fit_fetches def prep_predict(self, dataset, cross_eval=False): if cross_eval: self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_w = construct_W(self.wv_model, self.test_one_hot.get_key_ordering()).T else: self.test_one_hot = self.one_hot self.test_w = self.w x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [ self.model_info["prediction"], ] feed_dict = {self.model_info["input"]: x} truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): predictions = predict_fetches[0] if cross_eval and self.test_w is None: raise Exception("test_w is not set. Did you call prep_predict?") predictions = np.dot(predictions, self.test_w.T) return predictions def get_training_loss(self, fit_fetches): return fit_fetches[1]
class FastZeroTagModel(AttalosModel): """ Create a tensorflow graph that finds the principal direction of the target word embeddings (with negative sampling), using the loss function from "Fast Zero-Shot Image Tagging". """ def __init__(self, wv_model, datasets, **kwargs): self.wv_model = wv_model self.one_hot = OneHot(datasets, valid_vocab=wv_model.vocab) word_counts = NegativeSampler.get_wordcount_from_datasets(datasets, self.one_hot) self.negsampler = NegativeSampler(word_counts) self.w = construct_W(wv_model, self.one_hot.get_key_ordering()).T self.learning_rate = kwargs.get("learning_rate", 0.0001) self.optim_words = kwargs.get("optim_words", True) self.hidden_units = kwargs.get("hidden_units", "200,200") self.hidden_units = [int(x) for x in self.hidden_units.split(",")] self.model_info = dict() # Placeholders for data self.model_info['input'] = tf.placeholder(shape=(None, datasets[0].img_feat_size), dtype=tf.float32) self.model_info['pos_ids'] = tf.placeholder(dtype=tf.int32) self.model_info['neg_ids'] = tf.placeholder(dtype=tf.int32) self.model_info['w2v'] = tf.Variable(self.w, dtype=tf.float32) self.model_info['pos_vecs'] = tf.transpose(tf.nn.embedding_lookup(self.model_info['w2v'], self.model_info['pos_ids']), perm=[1,0,2]) self.model_info['neg_vecs'] = tf.transpose(tf.nn.embedding_lookup(self.model_info['w2v'], self.model_info['neg_ids']), perm=[1,0,2]) # Construct fully connected layers layers = [] for i, hidden_size in enumerate(self.hidden_units[:-1]): if i == 0: layer = tf.contrib.layers.relu(self.model_info['input'], hidden_size) else: layer = tf.contrib.layers.relu(layer, hidden_size) layers.append(layer) layer = tf.contrib.layers.batch_norm(layer) layers.append(layer) # Output layer should always be linear layer = tf.contrib.layers.linear(layer, self.hidden_units[-1]) layers.append(layer) self.model_info['layers'] = layers self.model_info['prediction'] = layer def fztloss( f, pVecs, nVecs ): """ Tensorized cost function from Fast Zero-Shot Learning paper Args: f: The output from the network, a tensor of shape (# images, word embedding size) pVecs: The vector embeddings of the ground truth tags, a tensor of shape (# images, # positive tags, word embedding size) nVecs: The vector embeddings of negatively sampled tags, a tensor of shape (# images, # negative samples, word embedding size) Returns: Scalar tensor representing the batch cost """ posmul = tf.mul(pVecs, f) negmul = tf.mul(nVecs, f) tfpos = tf.reduce_sum(posmul, reduction_indices=2) tfneg = tf.reduce_sum(negmul, reduction_indices=2) tfpos = tf.transpose(tfpos, [1,0]) tfneg = tf.transpose(tfneg, [1,0]) negexpan = tf.tile( tf.expand_dims(tfneg, -1), [1, 1, tf.shape(tfpos)[1]] ) posexpan = tf.tile( tf.transpose(tf.expand_dims(tfpos, -1), [0,2,1]), [1, tf.shape(tfneg)[1], 1]) differences = tf.sub(negexpan, posexpan) return tf.reduce_sum(tf.reduce_sum(tf.log(1 + tf.exp(differences)), reduction_indices=[1,2])) loss = fztloss(self.model_info['prediction'], self.model_info['pos_vecs'], self.model_info['neg_vecs']) self.model_info['loss'] = loss self.model_info['optimizer'] = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(loss) def predict_feats(self, sess, x): return sess.run(self.model_info['prediction'], feed_dict={self.model_info['input']: x}) def _get_ids(self, tag_ids, numSamps=[5, 10], uniform_sampling=False): """ Takes a batch worth of text tags and returns positive/negative ids """ pos_word_ids = np.ones((len(tag_ids), numSamps[0]), dtype=np.int32) pos_word_ids.fill(-1) for ind, tags in enumerate(tag_ids): if len(tags) > 0: pos_word_ids[ind] = np.random.choice(tags, size=numSamps[0]) neg_word_ids = None if uniform_sampling: neg_word_ids = np.random.randint(0, self.one_hot.vocab_size, size=(len(tag_ids), numSamps[1])) else: neg_word_ids = np.ones((len(tag_ids), numSamps[1]), dtype=np.int32) neg_word_ids.fill(-1) for ind in range(pos_word_ids.shape[0]): # TODO: Check to see if this benefits from the same bug as negsampling code neg_word_ids[ind] = self.negsampler.negsamp_ind(pos_word_ids[ind], numSamps[1]) return pos_word_ids, neg_word_ids def prep_fit(self, data): img_feats, text_feats_list = data text_feat_ids = [] for tags in text_feats_list: text_feat_ids.append([self.one_hot.get_index(tag) for tag in tags if tag in self.one_hot]) pos_ids, neg_ids = self._get_ids(text_feat_ids) self.pos_ids = pos_ids self.neg_ids = neg_ids fetches = [self.model_info["optimizer"], self.model_info["loss"]] feed_dict = { self.model_info["input"]: img_feats, self.model_info["pos_ids"]: pos_ids, self.model_info["neg_ids"]: neg_ids } return fetches, feed_dict def prep_predict(self, dataset, cross_eval=False): if cross_eval: self.test_one_hot = OneHot([dataset], valid_vocab=self.wv_model.vocab) self.test_w = construct_W(self.wv_model, self.test_one_hot.get_key_ordering()).T else: self.test_one_hot = self.one_hot self.test_w = self.w x = [] y = [] for idx in dataset: image_feats, text_feats = dataset.get_index(idx) text_feats = self.one_hot.get_multiple(text_feats) x.append(image_feats) y.append(text_feats) x = np.asarray(x) y = np.asarray(y) fetches = [self.model_info["prediction"], ] feed_dict = { self.model_info["input"]: x } truth = y return fetches, feed_dict, truth def post_predict(self, predict_fetches, cross_eval=False): predictions = predict_fetches[0] if cross_eval and self.test_w is None: raise Exception("test_w is not set. Did you call prep_predict?") predictions = np.dot(predictions, self.test_w.T) return predictions def get_training_loss(self, fit_fetches): return fit_fetches[1]