def main(): parser = argparse.ArgumentParser() parser.add_argument('--caption_file', type=str, default='Data/text.txt', help='caption file') parser.add_argument('--data_dir', type=str, default='Data', help='Data Directory') args = parser.parse_args() model = skipthoughts.load_model() encoded_captions = {} file_path = os.path.join(args.caption_file) dump_path = os.path.join(args.data_dir, 'enc_text.pkl') with open(file_path) as f: str_captions = f.read() captions = str_captions.split('\n') print(captions) encoded_captions['features'] = skipthoughts.encode(model, captions) pickle.dump(encoded_captions, open(dump_path, "wb")) print('Finished extracting Skip-Thought vectors of the given text ' 'descriptions')
def encode_and_save(image_captions, image_classes, data_dir: str, dataset: str): model = skipthoughts.load_model() encoded_captions = {} for i, img in enumerate(image_captions): st = time.time() encoded_captions[img] = skipthoughts.encode(model, image_captions[img]) if i % 20 == 0: print(i, len(image_captions), img) print("Seconds", time.time() - st) img_ids = list(image_captions.keys()) random.shuffle(img_ids) n_train_instances = int(len(img_ids) * 0.9) tr_image_ids = img_ids[0:n_train_instances] val_image_ids = img_ids[n_train_instances:-1] pickle.dump( image_captions, open(os.path.join(data_dir, dataset, dataset + '_caps.pkl'), "wb")) pickle.dump(tr_image_ids, open(os.path.join(data_dir, dataset, 'train_ids.pkl'), "wb")) pickle.dump(val_image_ids, open(os.path.join(data_dir, dataset, 'val_ids.pkl'), "wb")) ec_pkl_path = join(data_dir, dataset, dataset + '_tv.pkl') pickle.dump(encoded_captions, open(ec_pkl_path, "wb")) fc_pkl_path = join(data_dir, dataset, dataset + '_tc.pkl') pickle.dump(image_classes, open(fc_pkl_path, "wb"))
def save_caption_vectors_shapes(data_dir): import time img_dir = join(data_dir, 'shapes/images') image_files = [f for f in os.listdir(img_dir) if 'png' in f] print image_files[300:400] print len(image_files) image_captions = { img_file : [] for img_file in image_files } caption_dir = join(data_dir, 'shapes/texts') caption_files = [f for f in os.listdir(caption_dir) if 'txt' in f] for cap_file in caption_files: with open(join(caption_dir,cap_file)) as f: captions = f.read().split('\n') img_file = cap_file[0:5] + ".png" # 5 captions per image image_captions[img_file] += [cap for cap in captions if len(cap) > 0][0:5] print len(image_captions) model = skipthoughts.load_model() encoded_captions = {} for i, img in enumerate(image_captions): st = time.time() encoded_captions[img] = skipthoughts.encode(model, image_captions[img]) print i, len(image_captions), img print "Seconds", time.time() - st h = h5py.File(join(data_dir, 'shapes_tv.hdf5')) for key in encoded_captions: h.create_dataset(key, data=encoded_captions[key]) h.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--caption_file', type=str, default='Data/captions.txt', help='caption file') parser.add_argument('--data_dir', type=str, default='Data', help='Data Directory') args = parser.parse_args() with open(args.caption_file) as f: captions = f.read().split('\n') # captions : Text description of pictures stored in file sample_captions.txt captions = [cap for cap in captions if len(cap) > 0] print(captions) # create skipthoughts vectors model = skipthoughts.load_model() print('Creation of skipthought vectors : loading ....') caption_vectors = skipthoughts.encode(model, captions) print('Creation of skipthought vectors : DONE !') #print(caption_vectors) #print(np.shape(caption_vectors)).3 # create tensor vectors with skipthought vectors as input print('Save skipthought vector : loading ....') np.save('skipvectors_2000.npy', caption_vectors) print('Save skipthought vector : DONE !')
def __init__(self, datadir, uni_bi="combined"): """ Embed Skip_Thought vectors, using precomputed model in npy format. Args: uni_bi: possible values are "uni", "bi" or "combined" determining what kind of embedding should be used. todo: is argument ndim working properly? """ import skipthoughts self.encode = skipthoughts.encode if datadir is None: datadir = os.path.realpath('__file__') self.datadir = self.datadir # table for memoizing embeddings self.cache_table = {} self.uni_bi = uni_bi if uni_bi in ("uni", "bi"): self.N = 2400 elif uni_bi == "combined": self.N = 4800 else: raise ValueError( "uni_bi has invalid value. Valid values: 'uni', 'bi', 'combined'" ) self.skipthoughts.path_to_models = self.datadir self.skipthoughts.path_to_tables = self.datadir self.skipthoughts.path_to_umodel = skipthoughts.path_to_models + 'uni_skip.npz' self.skipthoughts.path_to_bmodel = skipthoughts.path_to_models + 'bi_skip.npz' self.st = skipthoughts.load_model()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--caption_file', type=str, default='Data/sample_captions.txt', help='caption file') parser.add_argument('--data_dir', type=str, default='Data', help='Data Directory') args = parser.parse_args() with open(args.caption_file) as f: captions = f.read().split('\n') captions = [cap for cap in captions if len(cap) > 0] print(captions) model = skipthoughts.load_model() caption_vectors = skipthoughts.encode(model, captions) if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')): os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5')) h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5')) h.create_dataset('vectors', data=caption_vectors) h.close()
def load_inputs(path, limit=-1): ''' Returns captions,resnet_embeddings,vectors ''' # load captions and resnet embeddings split = pickle.load(open(path, "rb")) ret = None if limit != -1: split = split[:limit] if len(split[0]) == 2: # encode captions via skipthought, if there are not emebddings already include model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) captions, resnet_embeddings = [s[0] for s in split ], [s[1].flatten() for s in split] vectors = encoder.encode(captions) ret = captions, resnet_embeddings, vectors elif len(split[0]) == 3: captions, resnet_embeddings, vectors = [s[0] for s in split], [ s[1].flatten() for s in split ], [s[2].flatten() for s in split] vectors = np.array(vectors) ret = captions, resnet_embeddings, vectors else: raise "Input pickled vectors should be a list of two tuples (caption,resnet) or\ three-tuples (caption,resnet,embedding)" return ret
def save_caption_vectors(all_captions, target_dir, split, experiment): h = h5py.File( os.path.join(target_dir, experiment, '{}_captions.hdf5'.format(split))) model = skipthoughts.load_model() for class_name, image_captions in all_captions.items(): print("number of images: ", len(image_captions)) img_batches = [[] for i in range(NUM_BATCHES)] caption_batches = [[] for i in range(NUM_BATCHES)] counter = 0 for img, captions in image_captions.items(): counter = counter % NUM_BATCHES img_batches[counter].append(img) caption_batches[counter] += captions counter += 1 print("batched for {}".format(class_name)) group = h.create_group(class_name) for i in range(NUM_BATCHES): imgs = img_batches[i] captions = caption_batches[i] encoded_captions = skipthoughts.encode(model, captions) cstart = 0 for img in imgs: num_caps = len(image_captions[img]) print(cstart, num_caps, len(encoded_captions)) group.create_dataset(img, data=encoded_captions[cstart:cstart + num_caps]) cstart += num_caps print("Batch {} of {} Done".format(i + 1, NUM_BATCHES)) h.close()
def load_all(): """ Load everything we need for generating """ print config.paths['decmodel'] # Skip-thoughts print 'Loading skip-thoughts...' stv = skipthoughts.load_model(config.paths['skmodels'], config.paths['sktables']) # Decoder print 'Loading decoder...' dec = decoder.load_model(config.paths['decmodel'], config.paths['dictionary']) # Image-sentence embedding print 'Loading image-sentence embedding...' vse = embedding.load_model(config.paths['vsemodel']) # VGG-19 print 'Loading and initializing ConvNet...' if config.FLAG_CPU_MODE: sys.path.insert(0, config.paths['pycaffe']) import caffe caffe.set_mode_cpu() net = caffe.Net(config.paths['vgg_proto_caffe'], config.paths['vgg_model_caffe'], caffe.TEST) else: net = build_convnet(config.paths['vgg']) # Captions print 'Loading captions...' cap = [] with open(config.paths['captions'], 'rb') as f: for line in f: cap.append(line.strip()) # Caption embeddings print 'Embedding captions...' cvec = embedding.encode_sentences(vse, cap, verbose=False) # Biases print 'Loading biases...' bneg = numpy.load(config.paths['negbias']) bpos = numpy.load(config.paths['posbias']) # Pack up z = {} z['stv'] = stv z['dec'] = dec z['vse'] = vse z['net'] = net z['cap'] = cap z['cvec'] = cvec z['bneg'] = bneg z['bpos'] = bpos return z
def __init__(self, session, robot_IP, robot_PORT, ref_joints_path, model_path, skipthoughts_path): self.memory = session.service("ALMemory") self.motion = ALProxy("ALMotion", robot_IP, robot_PORT) self.robot_joints = [ 'HeadYaw', 'HeadPitch', 'HipRoll', 'HipPitch', 'KneePitch', 'LShoulderPitch', 'LShoulderRoll', 'LElbowYaw', 'LElbowRoll', 'LWristYaw', 'LHand', 'RShoulderPitch', 'RShoulderRoll', 'RElbowYaw', 'RElbowRoll', 'RWristYaw', 'RHand' ] self.fractionMaxSpeed = 0.20 self.time_delay = 0.04 # Initialize the pre-trained models self.ref_joints_path = ref_joints_path + "ref_joints.pickle" self.model_path = model_path + "model.ckpt" self.caption_embedding_size = 2400 self.t_z = tf.placeholder(tf.float32, shape=(None, 100)) self.t_real_caption = tf.placeholder( tf.float32, shape=(None, self.caption_embedding_size)) self.G_z = generator(self.t_z, self.t_real_caption) self.sess = tf.InteractiveSession() saver = tf.train.Saver() saver.restore(self.sess, self.model_path) self.embedding_model = skipthoughts.load_model(skipthoughts_path)
def __init__(self, datadir, uni_bi="combined"): """ Embed Skip_Thought vectors, using precomputed model in npy format. Args: uni_bi: possible values are "uni", "bi" or "combined" determining what kind of embedding should be used. todo: is argument ndim working properly? """ import skipthoughts self.encode = skipthoughts.encode if datadir is None: datadir = os.path.realpath('__file__') self.datadir = self.datadir # table for memoizing embeddings self.cache_table = {} self.uni_bi = uni_bi if uni_bi in ("uni", "bi"): self.N = 2400 elif uni_bi == "combined": self.N = 4800 else: raise ValueError("uni_bi has invalid value. Valid values: 'uni', 'bi', 'combined'") self.skipthoughts.path_to_models = self.datadir self.skipthoughts.path_to_tables = self.datadir self.skipthoughts.path_to_umodel = skipthoughts.path_to_models + 'uni_skip.npz' self.skipthoughts.path_to_bmodel = skipthoughts.path_to_models + 'bi_skip.npz' self.st = skipthoughts.load_model()
def main(): with open(sys.argv[1]) as f: features = f.readlines() features = [re.match('[0-9]*,(.*)', x).groups()[0] for x in features] print(features) model = load_model() encoder = Encoder(model) ret = encoder.encode(features) np.save('features.npy', ret)
def __init__(self, trainedModel=False): self.trained = trainedModel print('Word Vectors have been loaded...') if self.trained: print('LOADING TRAINED MODEL') self.model = tools.load_model() else: print('LOADING PRE TRAINED MODEL') self.model = skipthoughts.load_model() print('loaded model')
def skipthought_encode(sentences): """ Obtains sentence embeddings for each sentence in the text """ # print('Loading pre-trained models...') model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) # print('Encoding sentences...') encoded = encoder.encode(sentences) return encoded
def main(): beta1 = 0.5 lr = 2e-4 z_dim = 100 t_dim = 256 batch_size = 64 image_size = 64 gfc_dim = 1024 caption_vector_length = 4800 epochs = 600 path = sys.argv[1] test_data, test_id = load_test_data(path) embed_model = skipthoughts.load_model() caption_vectors = skipthoughts.encode(embed_model, test_data) #np.save("test_embedd.npy", caption_vectors) #exit() #caption_vectors = np.load("test_embedd.npy") caption_vectors = np.tile(caption_vectors, (5, 1)) model_options = { 'z_dim': 100, 't_dim': 256, 'batch_size': len(test_data) * 5, 'image_size': 64, 'gf_dim': 64, 'df_dim': 64, 'gfc_dim': 1024, 'caption_vector_length': 4800 } gan = model.GAN(model_options) input_tensors, variables, loss, outputs, checks = gan.build_model() with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: tf.global_variables_initializer().run() saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state("Data/Models/") saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored.") z_noise = np.random.uniform(-1, 1, [5 * len(test_data), z_dim]) gen = sess.run(outputs['generator'], feed_dict={ input_tensors['t_real_caption']: caption_vectors, input_tensors['t_z']: z_noise }) print "Saving Images, Model" save_image(gen, test_id)
def load(): stv = skipthoughts.load_model(config.paths['skmodels'], config.paths['sktables']) console.log("Loaded skipthoughts model") bneg = numpy.load(config.paths['negbias']) console.log("Loaded negative bias") bpos = numpy.load(config.paths['posbias']) console.log("Loaded positive bias") dec = decoder.load_model(config.paths['decmodel'], config.paths['dictionary']) console.log("Loaded decoder") return (stv, bpos, bneg, dec)
def get_image_tag_pair(tag_dict_in_use, img_path="data/faces/"): print("start loading skipthoughts model") # get text vector model = skipthoughts.load_model() list_text = skipthoughts.encode(model, list(tag_dict_in_use.values())) # get image list_image = [] for key, item in tag_dict_in_use.items(): img = skimage.io.imread(os.path.join(img_path, str(key) + ".jpg")) img = skimage.transform.resize(img, (64, 64)) list_image.append(img) return list_image, list_text
def transform_sentences(sentences): """ Builds sentence embeddings using the Skip-thoughts model. :param _df: Input data frame with column of sentences. :return: Torch matrix of embeddings, size 1024. """ model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) #sentences = _df['sentText'].tolist() _sent_embs = encoder.encode(sentences, verbose=True) _sent_tensors = [torch.from_numpy(j) for j in _sent_embs] return torch.stack(_sent_tensors)
def load_input(path): input_images = load_input_images(path) input_sentences, labels = load_input_sentences() inputs = list(zip(input_images, input_sentences, labels)) random.shuffle(inputs) input_images, input_sentences, labels = zip(*inputs) model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) input_sentences = encoder.encode(input_sentences) return input_images, input_sentences, labels
def load_all(): """ Load everything we need for generating """ print config.paths['decmodel'] # Skip-thoughts print 'Loading skip-thoughts...' stv = skipthoughts.load_model(config.paths['skmodels'], config.paths['sktables']) # Decoder print 'Loading decoder...' dec = decoder.load_model(config.paths['decmodel'], config.paths['dictionary']) # Image-sentence embedding print 'Loading image-sentence embedding...' vse = embedding.load_model(config.paths['vsemodel']) # Captions print 'Loading captions...' cap = [] with open(config.paths['captions'], 'rb') as f: for line in f: cap.append(line.strip()) # Caption embeddings print 'Embedding captions...' cvec = embedding.encode_sentences(vse, cap, verbose=False) # Biases print 'Loading biases...' bneg = numpy.load(config.paths['negbias']) bpos = numpy.load(config.paths['posbias']) # VGG Net. net = load_vgg() # Pack up z = {} z['stv'] = stv z['dec'] = dec z['vse'] = vse z['net'] = net z['cap'] = cap z['cvec'] = cvec z['bneg'] = bneg z['bpos'] = bpos return z
def main(): parser = argparse.ArgumentParser() parser.add_argument('--caption_file', type=str, default='Data/sample_captions.txt', help='caption file') parser.add_argument('--data_dir', type=str, default='Data', help='Data Directory') parser.add_argument('--data_set', type=str, default='flowers', help="Define the name of data sets") args = parser.parse_args() _n_labels = 4096 if args.data_set == "ImageNet": with open("./Data/sample_caption_ImageNet.txt") as f: captions = f.read().split('\n') captions = [cap for cap in captions if len(cap) > 0] caption_vector_list = [] for cap in captions: _n = cap.split(',') _zeros0 = np.zeros(_n_labels) _zeros1 = np.zeros(_n_labels) _zeros0[int(_n[0])] = 1 _zeros1[int(_n[0])] = 1 _onehot = np.concatenate([_zeros0, _zeros1], axis=0) caption_vector_list.append(_onehot) print(len(caption_vector_list), len(caption_vector_list[0])) h = h5py.File(join(args.data_dir, 'sample_caption_ImageNet.hdf5')) h.create_dataset('vectors', data=caption_vector_list) h.close() if args.data_set == "flowers": with open(args.caption_file) as f: captions = f.read().split('\n') captions = [cap for cap in captions if len(cap) > 0] print(captions) model = skipthoughts.load_model() caption_vectors = skipthoughts.encode(model, captions) print(caption_vectors) print(caption_vectors.shape, len(caption_vectors[0])) if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')): os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5')) h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5')) h.create_dataset('vectors', data=caption_vectors) h.close()
def encodeSentences(reviews): enc_reviews = [None] * len(reviews) cum_sum_sentences = [0] sent_count = 0 for review in reviews: sent_count += len(review) cum_sum_sentences.append(sent_count) print('Loading pretrained model...') model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) print('Encoding sentences...') enc_sentences = encoder.encode(reviews, verbose=False) return enc_sentences
def load_all(): """ Load everything we need for generating """ print path_to_decmodel # Skip-thoughts print 'Loading skip-thoughts...' stv = skipthoughts.load_model(path_to_skmodels, path_to_sktables) # Decoder print 'Loading decoder...' dec = decoder.load_model(path_to_decmodel, path_to_dictionary) # Image-sentence embedding print 'Loading image-sentence embedding...' vse = embedding.load_model(path_to_vsemodel) # VGG-19 print 'Loading and initializing ConvNet...' net = build_convnet(path_to_vgg) # Captions print 'Loading captions...' cap = [] with open(path_to_captions, 'rb') as f: for line in f: cap.append(line.strip()) # Caption embeddings print 'Embedding captions...' cvec = embedding.encode_sentences(vse, cap, verbose=False) # Biases print 'Loading biases...' bneg = numpy.load(path_to_negbias) bpos = numpy.load(path_to_posbias) # Pack up z = {} z['stv'] = stv z['dec'] = dec z['vse'] = vse z['net'] = net z['cap'] = cap z['cvec'] = cvec z['bneg'] = bneg z['bpos'] = bpos return z
def build_examples(): import skipthoughts model = skipthoughts.load_model() caption_file = pickle.load( open(os.path.join(file_path, utils.back("dict.pkl")), "rb")) path_to_save_examples = os.path.join(file_path, utils.back("examples")) if not os.path.exists(path_to_save_examples): os.makedirs(path_to_save_examples) nb_file_per_tfrecords = 1000 for name in ["train", "val"]: # Path where images are path = os.path.join(file_path, utils.back("{}2014".format(name))) # Number of images in the folder nb_file = len([n for n in os.listdir(path)]) # Number of tfRecords already created nb_records = len([ n for n in os.listdir(path_to_save_examples) if n.startswith(name) ]) # The file number to restart from iter_to_restart = (nb_records - 1) * nb_file_per_tfrecords iter_to_restart = 79000 for iter in trange(iter_to_restart, nb_file, nb_file_per_tfrecords): writer = tf.python_io.TFRecordWriter( os.path.join( path_to_save_examples, "{}{}".format(name, iter // nb_file_per_tfrecords) + ".tfrecords")) for index, filename in tqdm(enumerate(os.listdir(path)), leave=False, desc="TfRecord{}".format( iter // nb_file_per_tfrecords)): if index < iter: continue if index >= iter + nb_file_per_tfrecords: break img = np.array(Image.open(os.path.join(path, filename))) caption = caption_file[filename.split(".")[0]] emb = skipthoughts.encode(model, caption, verbose=False) img = img.tostring() ex = make_example(img, emb) writer.write(ex.SerializeToString()) writer.close() break
def get_pretrained_encodings(pretrained=False): ''' Get encodings using the pre-trained models. ''' word_set = set() dict_f = open(os.path.join(DATA_PATH, 'word2vec/dict.txt'), 'r') for line in dict_f: word_set.add(line.strip()) dict_f.close() # Getting the data. with open(os.path.join(DATA_PATH, 'amazon_food/train_data.pkl'), 'r') as f: train_data = cPickle.load(f) train_preprocessed = preprocess(train_data[0], word_set) with open(os.path.join(DATA_PATH, 'amazon_food/test_data.pkl'), 'r') as f: test_data = cPickle.load(f) test_preprocessed = preprocess(test_data[0], word_set) if pretrained: model = skipthoughts.load_model() encoder = skipthoughts.Encoder(model) test_save_path = os.path.join( DATA_PATH, 'amazon_food/skip_thought_vecs/skip_thought_vecs_test_pretrained.npy' ) train_save_path = os.path.join( DATA_PATH, 'amazon_food/skip_thought_vecs/skip_thought_vecs_train_pretrained.npy' ) print('Encoding training vectors') train_vectors = encoder.encode(train_preprocessed) print('Encoding test vectors') test_vectors = encoder.encode(test_preprocessed) else: model = tools.load_model(None) test_save_path = os.path.join( DATA_PATH, 'amazon_food/skip_thought_vecs/skip_thought_vecs_test_bi.npy') train_save_path = os.path.join( DATA_PATH, 'amazon_food/skip_thought_vecs/skip_thought_vecs_train_bi.npy') print('Encoding training vectors') train_vectors = tools.encode(model, train_preprocessed) print('Encoding test vectors') test_vectors = tools.encode(model, test_preprocessed) np.save(train_save_path, train_vectors) np.save(test_save_path, test_vectors)
def save_caption_vectors_faces(data_dir): import time data_dir = 'Data/' img_dir = join(data_dir, 'faces/jpg') image_files = [f for f in os.listdir(img_dir) if 'jpg' in f] print image_files[1:20] print len(image_files) image_captions = {img_file: [] for img_file in image_files} caption_dir = join(data_dir, 'faces/tags/tags_clean.csv') with open(caption_dir, 'r') as fin: for line in fin: l = line.split(',') img_id = str(l[0]) + '.jpg' tags = l[1].split('\t') captions = [] for t in tags: t2 = t.split(':') if len(t2) > 1: captions.append([str(t2[0]), int(t2[1])]) captions.sort(key=lambda tup: tup[1], reverse=True) for ind, c in enumerate(captions): # ================================== # TODO: top 10 tags # if ind >= 10: # break # image_captions[img_id].append(c[0]) # =================================== # TODO: eyes and hair tags only if 'eye' in c[0] or 'hair' in c[0]: image_captions[img_id].append(c[0]) print len(image_captions) model = skipthoughts.load_model() encoded_captions = {} for i, img in enumerate(image_captions): st = time.time() encoded_captions[img] = skipthoughts.encode(model, image_captions[img]) print i, len(image_captions), img print "Seconds", time.time() - st h = h5py.File(join(data_dir, 'faces_tv.hdf5')) for key in encoded_captions: h.create_dataset(key, data=encoded_captions[key]) h.close()
def __init__(self, model_name=''): self.loaded_custom_model = False if model_name == '': print 'Loading BookCorpus encoding model...' self.model = skipthoughts.load_model() self.sentences = None self.vectors = None else: print 'Loading custom encoding model: ' + model_name self.loaded_custom_model = True self.model = penseur_utils.load_encoder(model_name) self.sentences = pickle.load(open('data/' + model_name + '_sen.p', 'r')) self.encode(self.sentences, verbose=True) self.analogy_vector = None self.word_table = None
def prune_thoughts(dataset, questions, input_dir): i = open(input_dir) text = i.read() clean = re.sub("[0-9]", "", text) sent_detector = nltk.data.load('tokenizers/punkt/english.pickle') sents = sent_detector.tokenize(clean) X = sents #print sents Y = dataset model = skipthoughts.load_model() vectors = skipthoughts.encode(model, X) #nearest_neighbor = skipthoughts.nn(model, X, vectors, Y, k=5) #print dataset #print questions return X
def generate(testing_file, generate_num): model = DCGAN(batch_size=generate_num, image_size=image_size, caption_vec_size=caption_vec_size, noise_dim=noise_dim, channel_dim=channel_dim, reduced_text_dim=reduced_text_dim, momentum=momentum) _, _, _ = model.build_model() inputs, image = model.build_generator() captions = [] with open(testing_file, 'r') as f: reader = csv.reader(f, delimiter=',') for line in reader: captions.append(line[1]) sent2vec = skipthoughts.load_model() caption_vecs = skipthoughts.encode(sent2vec, captions) generated_images = [] # with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: with tf.Session() as sess: model.restore_model(sess, model_file) for vec in caption_vecs: noise = np.random.uniform(-1, 1, [generate_num, noise_dim]) caption = [vec] * generate_num generated_images.append( sess.run(image, feed_dict={ inputs['caption']: caption, inputs['noise']: noise })) if not os.path.isdir(sample_path): os.mkdir(sample_path) for i, images in enumerate(generated_images, start=1): for j, image in enumerate(images, start=1): scipy.misc.imsave( os.path.join(sample_path, 'sample_{}_{}.jpg'.format(i, j)), image) return
def save_caption_vectors_flowers(data_dir): import time img_dir = join(data_dir, 'flowers/jpg') print(img_dir) image_files = [f for f in os.listdir(img_dir) if 'jpg' in f] #print(image_files[300:400]) print(len(image_files)) image_captions = {img_file: [] for img_file in image_files} caption_dir = join(data_dir, 'flowers/text_c10') class_dirs = [] for i in range(1, 103): class_dir_name = 'class_%.5d' % (i) class_dirs.append(join(caption_dir, class_dir_name)) for class_dir in class_dirs: caption_files = [f for f in os.listdir(class_dir) if 'txt' in f] for cap_file in caption_files: cap_file = str(cap_file).replace("._", "") #print(cap_file) with open(join(class_dir, cap_file)) as f: captions = f.read().split('\n') img_file = cap_file[0:11] + ".jpg" #_caption_collections = [cap for cap in captions if len(cap) > 0][0:5] # 5 captions per image _caption_collections = [cap for cap in captions if len(cap) > 0][0:1] # 1 captions per image # リストの結合 image_captions[img_file] += _caption_collections print(len(image_captions)) model = skipthoughts.load_model() encoded_captions = {} for i, img in enumerate(image_captions): st = time.time() encoded_captions[img] = skipthoughts.encode(model, image_captions[img]) print(i, len(image_captions), img) print("Seconds", time.time() - st) print("Finish the task!!!") """
def load_story(): # Skip-thoughts print 'Loading skip-thoughts...' stv = skipthoughts.load_model(config.paths['skmodels'], config.paths['sktables']) # Decoder print 'Loading decoder...' dec = decoder.load_model(config.paths['decmodel'], config.paths['dictionary']) # Biases print 'Loading biases...' bneg = numpy.load(config.paths['negbias']) bpos = numpy.load(config.paths['posbias']) return {'stv': stv, 'dec': dec, 'bneg': bneg, 'bpos': bpos}
def load_model(desc, tfidf_doc='split_plot', tfidf_wthr=1): """Load appropriate model based on descriptor type. """ model = None if desc.startswith('tfidf'): model = encode_tfidf_model(tfidf_doc, tfidf_wthr) desc = desc + '-' + tfidf_doc + '-' + str(tfidf_wthr) elif desc == 'word2vec': model = w2v.load('models/movie_plots_1364.d-300.mc1.w2v', kind='bin') elif desc == 'skipthought': model = skipthoughts.load_model() elif desc == 'vis-text-embed': raise ValueError('Visual-Text embeddings are not yet supported.') # model = VisTextEncoder() return model, desc
def save_caption_vectors_flowers(data_dir): import time img_dir = join(data_dir, 'flowers/jpg') image_files = [f for f in os.listdir(img_dir) if 'jpg' in f] print image_files[300:400] print len(image_files) image_captions = { img_file : [] for img_file in image_files } caption_dir = join(data_dir, 'flowers/text_c10') class_dirs = [] for i in range(1, 103): class_dir_name = 'class_%.5d'%(i) class_dirs.append( join(caption_dir, class_dir_name)) for class_dir in class_dirs: caption_files = [f for f in os.listdir(class_dir) if 'txt' in f] for cap_file in caption_files: with open(join(class_dir,cap_file)) as f: captions = f.read().split('\n') img_file = cap_file[0:11] + ".jpg" # 5 captions per image image_captions[img_file] += [cap for cap in captions if len(cap) > 0][0:5] print len(image_captions) model = skipthoughts.load_model() encoded_captions = {} for i, img in enumerate(image_captions): st = time.time() encoded_captions[img] = skipthoughts.encode(model, image_captions[img]) print i, len(image_captions), img print "Seconds", time.time() - st h = h5py.File(join(data_dir, 'flower_tv.hdf5')) for key in encoded_captions: h.create_dataset(key, data=encoded_captions[key]) h.close()
def save_caption_vectors_ms_coco(data_dir, split, batch_size): meta_data = {} ic_file = join(data_dir, 'annotations/captions_{}2014.json'.format(split)) with open(ic_file) as f: ic_data = json.loads(f.read()) meta_data['data_length'] = len(ic_data['annotations']) with open(join(data_dir, 'meta_{}.pkl'.format(split)), 'wb') as f: pickle.dump(meta_data, f) model = skipthoughts.load_model() batch_no = 0 print "Total Batches", len(ic_data['annotations'])/batch_size while batch_no*batch_size < len(ic_data['annotations']): captions = [] image_ids = [] idx = batch_no for i in range(batch_no*batch_size, (batch_no+1)*batch_size): idx = i%len(ic_data['annotations']) captions.append(ic_data['annotations'][idx]['caption']) image_ids.append(ic_data['annotations'][idx]['image_id']) print captions print image_ids # Thought Vectors tv_batch = skipthoughts.encode(model, captions) h5f_tv_batch = h5py.File( join(data_dir, 'tvs/'+split + '_tvs_' + str(batch_no)), 'w') h5f_tv_batch.create_dataset('tv', data=tv_batch) h5f_tv_batch.close() h5f_tv_batch_image_ids = h5py.File( join(data_dir, 'tvs/'+split + '_tv_image_id_' + str(batch_no)), 'w') h5f_tv_batch_image_ids.create_dataset('tv', data=image_ids) h5f_tv_batch_image_ids.close() print "Batches Done", batch_no, len(ic_data['annotations'])/batch_size batch_no += 1
def main(): model = skipthoughts.load_model() evaluate(model, evaltest=True)
def main(): model = skipthoughts.load_model() extract_sentence_vectors(model)
import sys import numpy as np sys.path.append('../skip-thoughts') import skipthoughts from tqdm import tqdm import hickle import threading import pickle import word2vec from nltk.tokenize import sent_tokenize from nltk.tokenize import word_tokenize model = skipthoughts.load_model() class Dataset(object): """This is a dataset ADT that contains story, QA. Args: param1 (dictionay) : (IMdb key, video clip name value) pair dictionary param2 (list) : QAInfo type list. We are able to get param1 and param2 by mqa.get_story_qa_data() function. """ def __init__(self, story, qa): self.story = story self.qa = qa # embedding matrix Z = Word2Vec or Skipthoughts
def read_model(): model = skipthoughts.load_model() return model
sys.path.append('/data/movieQA/MovieQA_benchmark') sys.path.append('/data/movieQA/skip-thoughts') import skipthoughts as skip import data_loader as MovieQA import gensim_w2v import configs import time flags = configs.tf_flag() FLAGS = flags.FLAGS qa_representation = load('/data/movieQA/skip_full.hkl') mqa = MovieQA.DataLoader() story, qa = mqa.get_story_qa_data('full', 'split_plot') skip_model = skip.load_model() INF = 987654321 def prepare_data(query, imdb_key): query_representation = skip.encode(skip_model, [query]) candidate_qa = [QAInfo for QAInfo in qa if QAInfo.imdb_key == imdb_key] skip_encode = list() for QAInfo in candidate_qa: try: skip_encode.append(qa_representation[QAInfo.qid]) except: pass similarity = [(np.inner(query_representation, rep)[0][0], i) for i, rep in enumerate(skip_encode)] similarity.sort(reverse=True) most_similar = [candidate_qa[i] for score, i in similarity[:1]] retrieved_question = most_similar[0].question retrieved_answer = most_similar[0].answers
def main(): parser = argparse.ArgumentParser() parser.add_argument('--caption_file', type=str, default='Data/sample_captions.txt', help='caption file') parser.add_argument('--data_dir', type=str, default='Data', help='Data Directory') args = parser.parse_args() with open( args.caption_file ) as f: captions = f.read().split('\n') captions = [cap for cap in captions if len(cap) > 0] print captions model = skipthoughts.load_model() caption_vectors = skipthoughts.encode(model, captions) if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')): os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5')) h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5')) h.create_dataset('vectors', data=caption_vectors) h.close()