Ejemplo n.º 1
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--caption_file',
                        type=str,
                        default='Data/text.txt',
                        help='caption file')
    parser.add_argument('--data_dir',
                        type=str,
                        default='Data',
                        help='Data Directory')

    args = parser.parse_args()

    model = skipthoughts.load_model()
    encoded_captions = {}
    file_path = os.path.join(args.caption_file)
    dump_path = os.path.join(args.data_dir, 'enc_text.pkl')
    with open(file_path) as f:
        str_captions = f.read()
        captions = str_captions.split('\n')
        print(captions)
        encoded_captions['features'] = skipthoughts.encode(model, captions)

    pickle.dump(encoded_captions, open(dump_path, "wb"))
    print('Finished extracting Skip-Thought vectors of the given text '
          'descriptions')
Ejemplo n.º 2
0
def encode_and_save(image_captions, image_classes, data_dir: str,
                    dataset: str):
    model = skipthoughts.load_model()
    encoded_captions = {}
    for i, img in enumerate(image_captions):
        st = time.time()
        encoded_captions[img] = skipthoughts.encode(model, image_captions[img])
        if i % 20 == 0:
            print(i, len(image_captions), img)
            print("Seconds", time.time() - st)

    img_ids = list(image_captions.keys())

    random.shuffle(img_ids)
    n_train_instances = int(len(img_ids) * 0.9)
    tr_image_ids = img_ids[0:n_train_instances]
    val_image_ids = img_ids[n_train_instances:-1]

    pickle.dump(
        image_captions,
        open(os.path.join(data_dir, dataset, dataset + '_caps.pkl'), "wb"))

    pickle.dump(tr_image_ids,
                open(os.path.join(data_dir, dataset, 'train_ids.pkl'), "wb"))
    pickle.dump(val_image_ids,
                open(os.path.join(data_dir, dataset, 'val_ids.pkl'), "wb"))

    ec_pkl_path = join(data_dir, dataset, dataset + '_tv.pkl')
    pickle.dump(encoded_captions, open(ec_pkl_path, "wb"))

    fc_pkl_path = join(data_dir, dataset, dataset + '_tc.pkl')
    pickle.dump(image_classes, open(fc_pkl_path, "wb"))
Ejemplo n.º 3
0
def save_caption_vectors_shapes(data_dir):
	import time
	
	img_dir = join(data_dir, 'shapes/images')
	image_files = [f for f in os.listdir(img_dir) if 'png' in f]
	print image_files[300:400]
	print len(image_files)
	image_captions = { img_file : [] for img_file in image_files }

	caption_dir = join(data_dir, 'shapes/texts')
        caption_files = [f for f in os.listdir(caption_dir) if 'txt' in f]
        for cap_file in caption_files:
                with open(join(caption_dir,cap_file)) as f:
                        captions = f.read().split('\n')
                img_file = cap_file[0:5] + ".png"
                # 5 captions per image
                image_captions[img_file] += [cap for cap in captions if len(cap) > 0][0:5]

	print len(image_captions)

	model = skipthoughts.load_model()
	encoded_captions = {}

	for i, img in enumerate(image_captions):
		st = time.time()
		encoded_captions[img] = skipthoughts.encode(model, image_captions[img])
		print i, len(image_captions), img
		print "Seconds", time.time() - st
		
	
	h = h5py.File(join(data_dir, 'shapes_tv.hdf5'))
	for key in encoded_captions:
		h.create_dataset(key, data=encoded_captions[key])
	h.close()
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--caption_file',
                        type=str,
                        default='Data/captions.txt',
                        help='caption file')
    parser.add_argument('--data_dir',
                        type=str,
                        default='Data',
                        help='Data Directory')

    args = parser.parse_args()
    with open(args.caption_file) as f:
        captions = f.read().split('\n')

    # captions : Text description of pictures stored in file sample_captions.txt
    captions = [cap for cap in captions if len(cap) > 0]
    print(captions)

    # create skipthoughts vectors
    model = skipthoughts.load_model()
    print('Creation of skipthought vectors : loading ....')
    caption_vectors = skipthoughts.encode(model, captions)
    print('Creation of skipthought vectors : DONE !')
    #print(caption_vectors)
    #print(np.shape(caption_vectors)).3

    # create tensor vectors with skipthought vectors as input
    print('Save skipthought vector : loading ....')
    np.save('skipvectors_2000.npy', caption_vectors)
    print('Save skipthought vector : DONE !')
Ejemplo n.º 5
0
    def __init__(self, datadir, uni_bi="combined"):
        """ Embed Skip_Thought vectors, using precomputed model in npy format.

        Args:
            uni_bi: possible values are "uni", "bi" or "combined" determining what kind of embedding should be used.


        todo: is argument ndim working properly?
        """

        import skipthoughts
        self.encode = skipthoughts.encode

        if datadir is None:
            datadir = os.path.realpath('__file__')
        self.datadir = self.datadir

        # table for memoizing embeddings
        self.cache_table = {}

        self.uni_bi = uni_bi
        if uni_bi in ("uni", "bi"):
            self.N = 2400
        elif uni_bi == "combined":
            self.N = 4800
        else:
            raise ValueError(
                "uni_bi has invalid value. Valid values: 'uni', 'bi', 'combined'"
            )

        self.skipthoughts.path_to_models = self.datadir
        self.skipthoughts.path_to_tables = self.datadir
        self.skipthoughts.path_to_umodel = skipthoughts.path_to_models + 'uni_skip.npz'
        self.skipthoughts.path_to_bmodel = skipthoughts.path_to_models + 'bi_skip.npz'
        self.st = skipthoughts.load_model()
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--caption_file',
                        type=str,
                        default='Data/sample_captions.txt',
                        help='caption file')
    parser.add_argument('--data_dir',
                        type=str,
                        default='Data',
                        help='Data Directory')

    args = parser.parse_args()
    with open(args.caption_file) as f:
        captions = f.read().split('\n')

    captions = [cap for cap in captions if len(cap) > 0]
    print(captions)
    model = skipthoughts.load_model()
    caption_vectors = skipthoughts.encode(model, captions)

    if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')):
        os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5'))
    h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5'))
    h.create_dataset('vectors', data=caption_vectors)
    h.close()
Ejemplo n.º 7
0
def load_inputs(path, limit=-1):
    '''
	Returns captions,resnet_embeddings,vectors
	'''

    # load captions and resnet embeddings
    split = pickle.load(open(path, "rb"))
    ret = None
    if limit != -1:
        split = split[:limit]

    if len(split[0]) == 2:
        # encode captions via skipthought, if there are not emebddings already include
        model = skipthoughts.load_model()
        encoder = skipthoughts.Encoder(model)
        captions, resnet_embeddings = [s[0] for s in split
                                       ], [s[1].flatten() for s in split]
        vectors = encoder.encode(captions)
        ret = captions, resnet_embeddings, vectors
    elif len(split[0]) == 3:
        captions, resnet_embeddings, vectors = [s[0] for s in split], [
            s[1].flatten() for s in split
        ], [s[2].flatten() for s in split]
        vectors = np.array(vectors)
        ret = captions, resnet_embeddings, vectors
    else:
        raise "Input pickled vectors should be a list of two tuples (caption,resnet) or\
		three-tuples (caption,resnet,embedding)"

    return ret
Ejemplo n.º 8
0
def save_caption_vectors(all_captions, target_dir, split, experiment):
    h = h5py.File(
        os.path.join(target_dir, experiment, '{}_captions.hdf5'.format(split)))
    model = skipthoughts.load_model()

    for class_name, image_captions in all_captions.items():
        print("number of images: ", len(image_captions))

        img_batches = [[] for i in range(NUM_BATCHES)]
        caption_batches = [[] for i in range(NUM_BATCHES)]
        counter = 0
        for img, captions in image_captions.items():
            counter = counter % NUM_BATCHES
            img_batches[counter].append(img)
            caption_batches[counter] += captions
            counter += 1
        print("batched for {}".format(class_name))

        group = h.create_group(class_name)
        for i in range(NUM_BATCHES):
            imgs = img_batches[i]
            captions = caption_batches[i]
            encoded_captions = skipthoughts.encode(model, captions)

            cstart = 0
            for img in imgs:
                num_caps = len(image_captions[img])
                print(cstart, num_caps, len(encoded_captions))
                group.create_dataset(img,
                                     data=encoded_captions[cstart:cstart +
                                                           num_caps])
                cstart += num_caps

            print("Batch {} of {} Done".format(i + 1, NUM_BATCHES))
    h.close()
Ejemplo n.º 9
0
def load_all():
    """
    Load everything we need for generating
    """
    print config.paths['decmodel']

    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(config.paths['skmodels'],
                                  config.paths['sktables'])

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(config.paths['decmodel'],
                             config.paths['dictionary'])

    # Image-sentence embedding
    print 'Loading image-sentence embedding...'
    vse = embedding.load_model(config.paths['vsemodel'])

    # VGG-19
    print 'Loading and initializing ConvNet...'

    if config.FLAG_CPU_MODE:
        sys.path.insert(0, config.paths['pycaffe'])
        import caffe
        caffe.set_mode_cpu()
        net = caffe.Net(config.paths['vgg_proto_caffe'],
                        config.paths['vgg_model_caffe'], caffe.TEST)
    else:
        net = build_convnet(config.paths['vgg'])

    # Captions
    print 'Loading captions...'
    cap = []
    with open(config.paths['captions'], 'rb') as f:
        for line in f:
            cap.append(line.strip())

    # Caption embeddings
    print 'Embedding captions...'
    cvec = embedding.encode_sentences(vse, cap, verbose=False)

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(config.paths['negbias'])
    bpos = numpy.load(config.paths['posbias'])

    # Pack up
    z = {}
    z['stv'] = stv
    z['dec'] = dec
    z['vse'] = vse
    z['net'] = net
    z['cap'] = cap
    z['cvec'] = cvec
    z['bneg'] = bneg
    z['bpos'] = bpos

    return z
Ejemplo n.º 10
0
    def __init__(self, session, robot_IP, robot_PORT, ref_joints_path,
                 model_path, skipthoughts_path):

        self.memory = session.service("ALMemory")
        self.motion = ALProxy("ALMotion", robot_IP, robot_PORT)
        self.robot_joints = [
            'HeadYaw', 'HeadPitch', 'HipRoll', 'HipPitch', 'KneePitch',
            'LShoulderPitch', 'LShoulderRoll', 'LElbowYaw', 'LElbowRoll',
            'LWristYaw', 'LHand', 'RShoulderPitch', 'RShoulderRoll',
            'RElbowYaw', 'RElbowRoll', 'RWristYaw', 'RHand'
        ]
        self.fractionMaxSpeed = 0.20
        self.time_delay = 0.04

        # Initialize the pre-trained models
        self.ref_joints_path = ref_joints_path + "ref_joints.pickle"
        self.model_path = model_path + "model.ckpt"

        self.caption_embedding_size = 2400
        self.t_z = tf.placeholder(tf.float32, shape=(None, 100))
        self.t_real_caption = tf.placeholder(
            tf.float32, shape=(None, self.caption_embedding_size))
        self.G_z = generator(self.t_z, self.t_real_caption)
        self.sess = tf.InteractiveSession()
        saver = tf.train.Saver()
        saver.restore(self.sess, self.model_path)

        self.embedding_model = skipthoughts.load_model(skipthoughts_path)
Ejemplo n.º 11
0
    def __init__(self, datadir, uni_bi="combined"):
        """ Embed Skip_Thought vectors, using precomputed model in npy format.

        Args:
            uni_bi: possible values are "uni", "bi" or "combined" determining what kind of embedding should be used.


        todo: is argument ndim working properly?
        """

        import skipthoughts
        self.encode = skipthoughts.encode

        if datadir is None:
            datadir = os.path.realpath('__file__')
        self.datadir = self.datadir

        # table for memoizing embeddings
        self.cache_table = {}

        self.uni_bi = uni_bi
        if uni_bi in ("uni", "bi"):
            self.N = 2400
        elif uni_bi == "combined":
            self.N = 4800
        else:
            raise ValueError("uni_bi has invalid value. Valid values: 'uni', 'bi', 'combined'")

        self.skipthoughts.path_to_models = self.datadir
        self.skipthoughts.path_to_tables = self.datadir
        self.skipthoughts.path_to_umodel = skipthoughts.path_to_models + 'uni_skip.npz'
        self.skipthoughts.path_to_bmodel = skipthoughts.path_to_models + 'bi_skip.npz'
        self.st = skipthoughts.load_model()
Ejemplo n.º 12
0
def main():
    with open(sys.argv[1]) as f:
        features = f.readlines()
    features = [re.match('[0-9]*,(.*)', x).groups()[0] for x in features]
    print(features)
    model = load_model()
    encoder = Encoder(model)
    ret = encoder.encode(features)
    np.save('features.npy', ret)
Ejemplo n.º 13
0
 def __init__(self, trainedModel=False):
     self.trained = trainedModel
     print('Word Vectors have been loaded...')
     if self.trained:
         print('LOADING TRAINED MODEL')
         self.model = tools.load_model()
     else:
         print('LOADING PRE TRAINED MODEL')
         self.model = skipthoughts.load_model()
     print('loaded model')
Ejemplo n.º 14
0
def skipthought_encode(sentences):
    """
    Obtains sentence embeddings for each sentence in the text
    """
    # print('Loading pre-trained models...')
    model = skipthoughts.load_model()
    encoder = skipthoughts.Encoder(model)
    # print('Encoding sentences...')
    encoded = encoder.encode(sentences)
    return encoded
Ejemplo n.º 15
0
def main():

    beta1 = 0.5
    lr = 2e-4
    z_dim = 100
    t_dim = 256
    batch_size = 64
    image_size = 64
    gfc_dim = 1024
    caption_vector_length = 4800
    epochs = 600
    path = sys.argv[1]

    test_data, test_id = load_test_data(path)
    embed_model = skipthoughts.load_model()
    caption_vectors = skipthoughts.encode(embed_model, test_data)

    #np.save("test_embedd.npy", caption_vectors)
    #exit()
    #caption_vectors = np.load("test_embedd.npy")
    caption_vectors = np.tile(caption_vectors, (5, 1))

    model_options = {
        'z_dim': 100,
        't_dim': 256,
        'batch_size': len(test_data) * 5,
        'image_size': 64,
        'gf_dim': 64,
        'df_dim': 64,
        'gfc_dim': 1024,
        'caption_vector_length': 4800
    }

    gan = model.GAN(model_options)
    input_tensors, variables, loss, outputs, checks = gan.build_model()

    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
        tf.global_variables_initializer().run()

        saver = tf.train.Saver()
        ckpt = tf.train.get_checkpoint_state("Data/Models/")
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("Model restored.")

        z_noise = np.random.uniform(-1, 1, [5 * len(test_data), z_dim])

        gen = sess.run(outputs['generator'],
                       feed_dict={
                           input_tensors['t_real_caption']: caption_vectors,
                           input_tensors['t_z']: z_noise
                       })
        print "Saving Images, Model"

        save_image(gen, test_id)
Ejemplo n.º 16
0
def load():
    stv = skipthoughts.load_model(config.paths['skmodels'],
                                  config.paths['sktables'])
    console.log("Loaded skipthoughts model")
    bneg = numpy.load(config.paths['negbias'])
    console.log("Loaded negative bias")
    bpos = numpy.load(config.paths['posbias'])
    console.log("Loaded positive bias")
    dec = decoder.load_model(config.paths['decmodel'],
                             config.paths['dictionary'])
    console.log("Loaded decoder")
    return (stv, bpos, bneg, dec)
Ejemplo n.º 17
0
def get_image_tag_pair(tag_dict_in_use, img_path="data/faces/"):
    print("start loading skipthoughts model")
    # get text vector
    model = skipthoughts.load_model()
    list_text = skipthoughts.encode(model, list(tag_dict_in_use.values()))

    # get image
    list_image = []
    for key, item in tag_dict_in_use.items():
        img = skimage.io.imread(os.path.join(img_path, str(key) + ".jpg"))
        img = skimage.transform.resize(img, (64, 64))
        list_image.append(img)
    return list_image, list_text
Ejemplo n.º 18
0
def transform_sentences(sentences):
    """
    Builds sentence embeddings using the Skip-thoughts model.

    :param _df: Input data frame with column of sentences.
    :return: Torch matrix of embeddings, size 1024.
    """
    model = skipthoughts.load_model()
    encoder = skipthoughts.Encoder(model)
    #sentences = _df['sentText'].tolist()
    _sent_embs = encoder.encode(sentences, verbose=True)
    _sent_tensors = [torch.from_numpy(j) for j in _sent_embs]
    return torch.stack(_sent_tensors)
Ejemplo n.º 19
0
def load_input(path):
    input_images = load_input_images(path)
    input_sentences, labels = load_input_sentences()

    inputs = list(zip(input_images, input_sentences, labels))
    random.shuffle(inputs)
    input_images, input_sentences, labels = zip(*inputs)

    model = skipthoughts.load_model()
    encoder = skipthoughts.Encoder(model)
    input_sentences = encoder.encode(input_sentences)

    return input_images, input_sentences, labels
Ejemplo n.º 20
0
def load_all():
    """
    Load everything we need for generating
    """
    print config.paths['decmodel']

    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(config.paths['skmodels'],
                                  config.paths['sktables'])

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(config.paths['decmodel'],
                             config.paths['dictionary'])

    # Image-sentence embedding
    print 'Loading image-sentence embedding...'
    vse = embedding.load_model(config.paths['vsemodel'])

    # Captions
    print 'Loading captions...'
    cap = []
    with open(config.paths['captions'], 'rb') as f:
        for line in f:
            cap.append(line.strip())

    # Caption embeddings
    print 'Embedding captions...'
    cvec = embedding.encode_sentences(vse, cap, verbose=False)

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(config.paths['negbias'])
    bpos = numpy.load(config.paths['posbias'])

    # VGG Net.
    net = load_vgg()

    # Pack up
    z = {}
    z['stv'] = stv
    z['dec'] = dec
    z['vse'] = vse
    z['net'] = net
    z['cap'] = cap
    z['cvec'] = cvec
    z['bneg'] = bneg
    z['bpos'] = bpos

    return z
Ejemplo n.º 21
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--caption_file',
                        type=str,
                        default='Data/sample_captions.txt',
                        help='caption file')
    parser.add_argument('--data_dir',
                        type=str,
                        default='Data',
                        help='Data Directory')
    parser.add_argument('--data_set',
                        type=str,
                        default='flowers',
                        help="Define the name of data sets")

    args = parser.parse_args()
    _n_labels = 4096
    if args.data_set == "ImageNet":
        with open("./Data/sample_caption_ImageNet.txt") as f:
            captions = f.read().split('\n')
        captions = [cap for cap in captions if len(cap) > 0]
        caption_vector_list = []
        for cap in captions:
            _n = cap.split(',')
            _zeros0 = np.zeros(_n_labels)
            _zeros1 = np.zeros(_n_labels)
            _zeros0[int(_n[0])] = 1
            _zeros1[int(_n[0])] = 1
            _onehot = np.concatenate([_zeros0, _zeros1], axis=0)
            caption_vector_list.append(_onehot)
        print(len(caption_vector_list), len(caption_vector_list[0]))
        h = h5py.File(join(args.data_dir, 'sample_caption_ImageNet.hdf5'))
        h.create_dataset('vectors', data=caption_vector_list)
        h.close()

    if args.data_set == "flowers":
        with open(args.caption_file) as f:
            captions = f.read().split('\n')

        captions = [cap for cap in captions if len(cap) > 0]
        print(captions)
        model = skipthoughts.load_model()
        caption_vectors = skipthoughts.encode(model, captions)
        print(caption_vectors)
        print(caption_vectors.shape, len(caption_vectors[0]))

        if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')):
            os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5'))
        h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5'))
        h.create_dataset('vectors', data=caption_vectors)
        h.close()
Ejemplo n.º 22
0
def encodeSentences(reviews):
    enc_reviews = [None] * len(reviews)
    cum_sum_sentences = [0]
    sent_count = 0
    for review in reviews:
        sent_count += len(review)
        cum_sum_sentences.append(sent_count)

    print('Loading pretrained model...')
    model = skipthoughts.load_model()
    encoder = skipthoughts.Encoder(model)
    print('Encoding sentences...')
    enc_sentences = encoder.encode(reviews, verbose=False)
    return enc_sentences
Ejemplo n.º 23
0
def load_all():
    """
    Load everything we need for generating
    """
    print path_to_decmodel

    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(path_to_skmodels, path_to_sktables)

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(path_to_decmodel, path_to_dictionary)

    # Image-sentence embedding
    print 'Loading image-sentence embedding...'
    vse = embedding.load_model(path_to_vsemodel)

    # VGG-19
    print 'Loading and initializing ConvNet...'
    net = build_convnet(path_to_vgg)

    # Captions
    print 'Loading captions...'
    cap = []
    with open(path_to_captions, 'rb') as f:
        for line in f:
            cap.append(line.strip())

    # Caption embeddings
    print 'Embedding captions...'
    cvec = embedding.encode_sentences(vse, cap, verbose=False)

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(path_to_negbias)
    bpos = numpy.load(path_to_posbias)

    # Pack up
    z = {}
    z['stv'] = stv
    z['dec'] = dec
    z['vse'] = vse
    z['net'] = net
    z['cap'] = cap
    z['cvec'] = cvec
    z['bneg'] = bneg
    z['bpos'] = bpos

    return z
Ejemplo n.º 24
0
def load_all():
    """
    Load everything we need for generating
    """
    print path_to_decmodel

    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(path_to_skmodels, path_to_sktables)

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(path_to_decmodel, path_to_dictionary)

    # Image-sentence embedding
    print 'Loading image-sentence embedding...'
    vse = embedding.load_model(path_to_vsemodel)

    # VGG-19
    print 'Loading and initializing ConvNet...'
    net = build_convnet(path_to_vgg)

    # Captions
    print 'Loading captions...'
    cap = []
    with open(path_to_captions, 'rb') as f:
        for line in f:
            cap.append(line.strip())

    # Caption embeddings
    print 'Embedding captions...'
    cvec = embedding.encode_sentences(vse, cap, verbose=False)

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(path_to_negbias)
    bpos = numpy.load(path_to_posbias)

    # Pack up
    z = {}
    z['stv'] = stv
    z['dec'] = dec
    z['vse'] = vse
    z['net'] = net
    z['cap'] = cap
    z['cvec'] = cvec
    z['bneg'] = bneg
    z['bpos'] = bpos

    return z
Ejemplo n.º 25
0
def build_examples():
    import skipthoughts
    model = skipthoughts.load_model()

    caption_file = pickle.load(
        open(os.path.join(file_path, utils.back("dict.pkl")), "rb"))

    path_to_save_examples = os.path.join(file_path, utils.back("examples"))
    if not os.path.exists(path_to_save_examples):
        os.makedirs(path_to_save_examples)

    nb_file_per_tfrecords = 1000
    for name in ["train", "val"]:
        # Path where images are
        path = os.path.join(file_path, utils.back("{}2014".format(name)))
        # Number of images in the folder
        nb_file = len([n for n in os.listdir(path)])

        # Number of tfRecords already created
        nb_records = len([
            n for n in os.listdir(path_to_save_examples) if n.startswith(name)
        ])
        # The file number to restart from
        iter_to_restart = (nb_records - 1) * nb_file_per_tfrecords
        iter_to_restart = 79000
        for iter in trange(iter_to_restart, nb_file, nb_file_per_tfrecords):
            writer = tf.python_io.TFRecordWriter(
                os.path.join(
                    path_to_save_examples,
                    "{}{}".format(name, iter // nb_file_per_tfrecords) +
                    ".tfrecords"))
            for index, filename in tqdm(enumerate(os.listdir(path)),
                                        leave=False,
                                        desc="TfRecord{}".format(
                                            iter // nb_file_per_tfrecords)):
                if index < iter:
                    continue
                if index >= iter + nb_file_per_tfrecords:
                    break

                img = np.array(Image.open(os.path.join(path, filename)))
                caption = caption_file[filename.split(".")[0]]
                emb = skipthoughts.encode(model, caption, verbose=False)
                img = img.tostring()
                ex = make_example(img, emb)
                writer.write(ex.SerializeToString())

            writer.close()
            break
Ejemplo n.º 26
0
def get_pretrained_encodings(pretrained=False):
    '''
    Get encodings using the pre-trained models.
    '''

    word_set = set()
    dict_f = open(os.path.join(DATA_PATH, 'word2vec/dict.txt'), 'r')
    for line in dict_f:
        word_set.add(line.strip())
    dict_f.close()

    # Getting the data.
    with open(os.path.join(DATA_PATH, 'amazon_food/train_data.pkl'), 'r') as f:
        train_data = cPickle.load(f)
        train_preprocessed = preprocess(train_data[0], word_set)
    with open(os.path.join(DATA_PATH, 'amazon_food/test_data.pkl'), 'r') as f:
        test_data = cPickle.load(f)
        test_preprocessed = preprocess(test_data[0], word_set)

    if pretrained:
        model = skipthoughts.load_model()
        encoder = skipthoughts.Encoder(model)
        test_save_path = os.path.join(
            DATA_PATH,
            'amazon_food/skip_thought_vecs/skip_thought_vecs_test_pretrained.npy'
        )
        train_save_path = os.path.join(
            DATA_PATH,
            'amazon_food/skip_thought_vecs/skip_thought_vecs_train_pretrained.npy'
        )
        print('Encoding training vectors')
        train_vectors = encoder.encode(train_preprocessed)
        print('Encoding test vectors')
        test_vectors = encoder.encode(test_preprocessed)
    else:
        model = tools.load_model(None)
        test_save_path = os.path.join(
            DATA_PATH,
            'amazon_food/skip_thought_vecs/skip_thought_vecs_test_bi.npy')
        train_save_path = os.path.join(
            DATA_PATH,
            'amazon_food/skip_thought_vecs/skip_thought_vecs_train_bi.npy')
        print('Encoding training vectors')
        train_vectors = tools.encode(model, train_preprocessed)
        print('Encoding test vectors')
        test_vectors = tools.encode(model, test_preprocessed)

    np.save(train_save_path, train_vectors)
    np.save(test_save_path, test_vectors)
Ejemplo n.º 27
0
def save_caption_vectors_faces(data_dir):
    import time

    data_dir = 'Data/'

    img_dir = join(data_dir, 'faces/jpg')
    image_files = [f for f in os.listdir(img_dir) if 'jpg' in f]
    print image_files[1:20]
    print len(image_files)
    image_captions = {img_file: [] for img_file in image_files}

    caption_dir = join(data_dir, 'faces/tags/tags_clean.csv')
    with open(caption_dir, 'r') as fin:
        for line in fin:
            l = line.split(',')
            img_id = str(l[0]) + '.jpg'
            tags = l[1].split('\t')
            captions = []
            for t in tags:
                t2 = t.split(':')
                if len(t2) > 1:
                    captions.append([str(t2[0]), int(t2[1])])
            captions.sort(key=lambda tup: tup[1], reverse=True)
            for ind, c in enumerate(captions):
                # ==================================
                # TODO: top 10 tags
                # if ind >= 10:
                # 	break
                # image_captions[img_id].append(c[0])
                # ===================================
                # TODO: eyes and hair tags only
                if 'eye' in c[0] or 'hair' in c[0]:
                    image_captions[img_id].append(c[0])

    print len(image_captions)

    model = skipthoughts.load_model()
    encoded_captions = {}

    for i, img in enumerate(image_captions):
        st = time.time()
        encoded_captions[img] = skipthoughts.encode(model, image_captions[img])
        print i, len(image_captions), img
        print "Seconds", time.time() - st

    h = h5py.File(join(data_dir, 'faces_tv.hdf5'))
    for key in encoded_captions:
        h.create_dataset(key, data=encoded_captions[key])
    h.close()
Ejemplo n.º 28
0
	def __init__(self, model_name=''):
		self.loaded_custom_model = False
		if model_name == '':
			print 'Loading BookCorpus encoding model...'
			self.model = skipthoughts.load_model()
			self.sentences = None
			self.vectors = None
		else:
			print 'Loading custom encoding model: ' + model_name
			self.loaded_custom_model = True
			self.model = penseur_utils.load_encoder(model_name)
			self.sentences = pickle.load(open('data/' + model_name + '_sen.p', 'r'))
			self.encode(self.sentences, verbose=True)
		self.analogy_vector = None
		self.word_table = None
Ejemplo n.º 29
0
def prune_thoughts(dataset, questions, input_dir):
    i = open(input_dir)
    text = i.read()
    clean = re.sub("[0-9]", "", text)
    sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
    sents = sent_detector.tokenize(clean)
    X = sents
    #print sents
    Y = dataset
    model = skipthoughts.load_model()
    vectors = skipthoughts.encode(model, X)
    #nearest_neighbor = skipthoughts.nn(model, X, vectors, Y, k=5)
    #print dataset
    #print questions
    return X
Ejemplo n.º 30
0
def generate(testing_file, generate_num):
    model = DCGAN(batch_size=generate_num,
                  image_size=image_size,
                  caption_vec_size=caption_vec_size,
                  noise_dim=noise_dim,
                  channel_dim=channel_dim,
                  reduced_text_dim=reduced_text_dim,
                  momentum=momentum)
    _, _, _ = model.build_model()

    inputs, image = model.build_generator()

    captions = []

    with open(testing_file, 'r') as f:
        reader = csv.reader(f, delimiter=',')
        for line in reader:
            captions.append(line[1])

    sent2vec = skipthoughts.load_model()
    caption_vecs = skipthoughts.encode(sent2vec, captions)

    generated_images = []

    #    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
    with tf.Session() as sess:
        model.restore_model(sess, model_file)

        for vec in caption_vecs:
            noise = np.random.uniform(-1, 1, [generate_num, noise_dim])
            caption = [vec] * generate_num
            generated_images.append(
                sess.run(image,
                         feed_dict={
                             inputs['caption']: caption,
                             inputs['noise']: noise
                         }))

    if not os.path.isdir(sample_path):
        os.mkdir(sample_path)

    for i, images in enumerate(generated_images, start=1):
        for j, image in enumerate(images, start=1):
            scipy.misc.imsave(
                os.path.join(sample_path, 'sample_{}_{}.jpg'.format(i, j)),
                image)

    return
Ejemplo n.º 31
0
def save_caption_vectors_flowers(data_dir):
    import time

    img_dir = join(data_dir, 'flowers/jpg')
    print(img_dir)
    image_files = [f for f in os.listdir(img_dir) if 'jpg' in f]
    #print(image_files[300:400])
    print(len(image_files))
    image_captions = {img_file: [] for img_file in image_files}

    caption_dir = join(data_dir, 'flowers/text_c10')
    class_dirs = []
    for i in range(1, 103):
        class_dir_name = 'class_%.5d' % (i)
        class_dirs.append(join(caption_dir, class_dir_name))

    for class_dir in class_dirs:
        caption_files = [f for f in os.listdir(class_dir) if 'txt' in f]
        for cap_file in caption_files:
            cap_file = str(cap_file).replace("._", "")
            #print(cap_file)
            with open(join(class_dir, cap_file)) as f:
                captions = f.read().split('\n')
            img_file = cap_file[0:11] + ".jpg"

            #_caption_collections = [cap for cap in captions if len(cap) > 0][0:5]
            # 5 captions per image

            _caption_collections = [cap for cap in captions
                                    if len(cap) > 0][0:1]
            # 1 captions per image

            # リストの結合
            image_captions[img_file] += _caption_collections
    print(len(image_captions))

    model = skipthoughts.load_model()
    encoded_captions = {}

    for i, img in enumerate(image_captions):
        st = time.time()
        encoded_captions[img] = skipthoughts.encode(model, image_captions[img])
        print(i, len(image_captions), img)
        print("Seconds", time.time() - st)

    print("Finish the task!!!")
    """ 
Ejemplo n.º 32
0
def load_story():
    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(config.paths['skmodels'],
                                  config.paths['sktables'])

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(config.paths['decmodel'],
                             config.paths['dictionary'])

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(config.paths['negbias'])
    bpos = numpy.load(config.paths['posbias'])

    return {'stv': stv, 'dec': dec, 'bneg': bneg, 'bpos': bpos}
def load_model(desc, tfidf_doc='split_plot', tfidf_wthr=1):
    """Load appropriate model based on descriptor type.
    """

    model = None
    if desc.startswith('tfidf'):
        model = encode_tfidf_model(tfidf_doc, tfidf_wthr)
        desc = desc + '-' + tfidf_doc + '-' + str(tfidf_wthr)

    elif desc == 'word2vec':
        model = w2v.load('models/movie_plots_1364.d-300.mc1.w2v', kind='bin')

    elif desc == 'skipthought':
        model = skipthoughts.load_model()

    elif desc == 'vis-text-embed':
        raise ValueError('Visual-Text embeddings are not yet supported.')
    #     model = VisTextEncoder()

    return model, desc
Ejemplo n.º 34
0
def save_caption_vectors_flowers(data_dir):
	import time
	
	img_dir = join(data_dir, 'flowers/jpg')
	image_files = [f for f in os.listdir(img_dir) if 'jpg' in f]
	print image_files[300:400]
	print len(image_files)
	image_captions = { img_file : [] for img_file in image_files }

	caption_dir = join(data_dir, 'flowers/text_c10')
	class_dirs = []
	for i in range(1, 103):
		class_dir_name = 'class_%.5d'%(i)
		class_dirs.append( join(caption_dir, class_dir_name))

	for class_dir in class_dirs:
		caption_files = [f for f in os.listdir(class_dir) if 'txt' in f]
		for cap_file in caption_files:
			with open(join(class_dir,cap_file)) as f:
				captions = f.read().split('\n')
			img_file = cap_file[0:11] + ".jpg"
			# 5 captions per image
			image_captions[img_file] += [cap for cap in captions if len(cap) > 0][0:5]

	print len(image_captions)

	model = skipthoughts.load_model()
	encoded_captions = {}


	for i, img in enumerate(image_captions):
		st = time.time()
		encoded_captions[img] = skipthoughts.encode(model, image_captions[img])
		print i, len(image_captions), img
		print "Seconds", time.time() - st
		
	
	h = h5py.File(join(data_dir, 'flower_tv.hdf5'))
	for key in encoded_captions:
		h.create_dataset(key, data=encoded_captions[key])
	h.close()
Ejemplo n.º 35
0
def save_caption_vectors_ms_coco(data_dir, split, batch_size):
	meta_data = {}
	ic_file = join(data_dir, 'annotations/captions_{}2014.json'.format(split))
	with open(ic_file) as f:
		ic_data = json.loads(f.read())

	meta_data['data_length'] = len(ic_data['annotations'])
	with open(join(data_dir, 'meta_{}.pkl'.format(split)), 'wb') as f:
		pickle.dump(meta_data, f)

	model = skipthoughts.load_model()
	batch_no = 0
	print "Total Batches", len(ic_data['annotations'])/batch_size

	while batch_no*batch_size < len(ic_data['annotations']):
		captions = []
		image_ids = []
		idx = batch_no
		for i in range(batch_no*batch_size, (batch_no+1)*batch_size):
			idx = i%len(ic_data['annotations'])
			captions.append(ic_data['annotations'][idx]['caption'])
			image_ids.append(ic_data['annotations'][idx]['image_id'])

		print captions
		print image_ids
		# Thought Vectors
		tv_batch = skipthoughts.encode(model, captions)
		h5f_tv_batch = h5py.File( join(data_dir, 'tvs/'+split + '_tvs_' + str(batch_no)), 'w')
		h5f_tv_batch.create_dataset('tv', data=tv_batch)
		h5f_tv_batch.close()

		h5f_tv_batch_image_ids = h5py.File( join(data_dir, 'tvs/'+split + '_tv_image_id_' + str(batch_no)), 'w')
		h5f_tv_batch_image_ids.create_dataset('tv', data=image_ids)
		h5f_tv_batch_image_ids.close()

		print "Batches Done", batch_no, len(ic_data['annotations'])/batch_size
		batch_no += 1
Ejemplo n.º 36
0
def main():
    model = skipthoughts.load_model()
    evaluate(model, evaltest=True)
Ejemplo n.º 37
0
def main():
	model = skipthoughts.load_model()
	extract_sentence_vectors(model)
Ejemplo n.º 38
0
import sys
import numpy as np
sys.path.append('../skip-thoughts')
import skipthoughts
from tqdm import tqdm
import hickle
import threading
import pickle
import word2vec
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize


model = skipthoughts.load_model()


class Dataset(object):
    """This is a dataset ADT that contains story, QA.

    Args:
        param1 (dictionay) : (IMdb key, video clip name value) pair dictionary
        param2 (list) : QAInfo type list.

        We are able to get param1 and param2 by mqa.get_story_qa_data() function.
    """

    def __init__(self, story, qa):
        self.story = story
        self.qa = qa

        # embedding matrix Z = Word2Vec or Skipthoughts
Ejemplo n.º 39
0
def read_model():
    model = skipthoughts.load_model()
    return model
Ejemplo n.º 40
0
sys.path.append('/data/movieQA/MovieQA_benchmark')
sys.path.append('/data/movieQA/skip-thoughts')
import skipthoughts as skip
import data_loader as MovieQA
import gensim_w2v
import configs
import time
flags = configs.tf_flag()
FLAGS = flags.FLAGS


qa_representation = load('/data/movieQA/skip_full.hkl')

mqa = MovieQA.DataLoader()
story, qa = mqa.get_story_qa_data('full', 'split_plot')
skip_model = skip.load_model()
INF = 987654321

def prepare_data(query, imdb_key):
  query_representation = skip.encode(skip_model, [query])
  candidate_qa = [QAInfo for QAInfo in qa if QAInfo.imdb_key == imdb_key]
  skip_encode = list()
  for QAInfo in candidate_qa:
    try: skip_encode.append(qa_representation[QAInfo.qid])
    except: pass
  similarity = [(np.inner(query_representation, rep)[0][0], i) for i, rep in enumerate(skip_encode)]
  similarity.sort(reverse=True)
  most_similar = [candidate_qa[i] for score, i in similarity[:1]]

  retrieved_question = most_similar[0].question
  retrieved_answer = most_similar[0].answers
Ejemplo n.º 41
0
def load_all():
    """
    Load everything we need for generating
    """
    print config.paths['decmodel']

    # Skip-thoughts
    print 'Loading skip-thoughts...'
    stv = skipthoughts.load_model(config.paths['skmodels'],
                                  config.paths['sktables'])

    # Decoder
    print 'Loading decoder...'
    dec = decoder.load_model(config.paths['decmodel'],
                             config.paths['dictionary'])

    # Image-sentence embedding
    print 'Loading image-sentence embedding...'
    vse = embedding.load_model(config.paths['vsemodel'])

    # VGG-19
    print 'Loading and initializing ConvNet...'

    if config.FLAG_CPU_MODE:
        sys.path.insert(0, config.paths['pycaffe'])
        import caffe
        caffe.set_mode_cpu()
        net = caffe.Net(config.paths['vgg_proto_caffe'],
                        config.paths['vgg_model_caffe'],
                        caffe.TEST)
    else:
        net = build_convnet(config.paths['vgg'])

    # Captions
    print 'Loading captions...'
    cap = []
    with open(config.paths['captions'], 'rb') as f:
        for line in f:
            cap.append(line.strip())

    # Caption embeddings
    print 'Embedding captions...'
    cvec = embedding.encode_sentences(vse, cap, verbose=False)

    # Biases
    print 'Loading biases...'
    bneg = numpy.load(config.paths['negbias'])
    bpos = numpy.load(config.paths['posbias'])

    # Pack up
    z = {}
    z['stv'] = stv
    z['dec'] = dec
    z['vse'] = vse
    z['net'] = net
    z['cap'] = cap
    z['cvec'] = cvec
    z['bneg'] = bneg
    z['bpos'] = bpos

    return z
def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('--caption_file', type=str, default='Data/sample_captions.txt',
					   help='caption file')
	parser.add_argument('--data_dir', type=str, default='Data',
					   help='Data Directory')

	args = parser.parse_args()
	with open( args.caption_file ) as f:
		captions = f.read().split('\n')

	captions = [cap for cap in captions if len(cap) > 0]
	print captions
	model = skipthoughts.load_model()
	caption_vectors = skipthoughts.encode(model, captions)

	if os.path.isfile(join(args.data_dir, 'sample_caption_vectors.hdf5')):
		os.remove(join(args.data_dir, 'sample_caption_vectors.hdf5'))
	h = h5py.File(join(args.data_dir, 'sample_caption_vectors.hdf5'))
	h.create_dataset('vectors', data=caption_vectors)		
	h.close()