def story(z, image_loc, k=20, bw=5, lyric=False):
"""
Generate a story for an image at location image_loc
"""
# Load the image
rawim, im = load_image(image_loc)
# Run image through convnet
feats = compute_features(z['net'], im).flatten()
feats /= norm(feats)
# Embed image into joint space
feats = embedding.encode_images(z['vse'], feats[None, :])
# Compute the nearest neighbours
scores = numpy.dot(feats, z['cvec'].T).flatten()
sorted_args = numpy.argsort(scores)[::-1]
sentences = [z['cap'][a] for a in sorted_args[:k]]
#pipeline broken here and one good caption whose image name is same is added
f = open('/Users/shreyajain/Downloads/output.txt').read()
#f2 = open('/Users/shreyajain/Downloads/input_story.txt','w')
image_name = image_loc.split('/')[-1]
text = f.split('\n')
for t in range(4, len(text)):
l = text[t].split()
if l != []:
name = l[0].split('/')[-1]
if name == image_name:
caption = l[1:]
#caption = ' '.join(caption)
sentences = caption + sentences
break
# print 'NEAREST-CAPTIONS: '
# for s in sentences[:5]:
# print s
# print ''
# Compute skip-thought vectors for sentences
svecs = skipthoughts.encode(z['stv'], sentences, verbose=False)
# Style shifting
shift = svecs.mean(0) - z['bneg'] + z['bpos']
# Generate story conditioned on shift
passage = decoder.run_sampler(z['dec'], shift, beam_width=bw)
#print 'OUTPUT: '
if lyric:
for line in passage.split(','):
if line[0] != ' ':
print line
else:
print line[1:]
else:
return passage
def generate(sentences, stv, bpos, bneg, dec):
# Compute skip-thought vectors for sentences
svecs = skipthoughts.encode(stv, sentences, verbose=False)
console.log("Encoded skipthought vector")
# Style shifting
shift = svecs.mean(0) - bneg + bpos
console.log("Shifted style")
# TODO: clean up here
# Generate story conditioned on shift
passage = decoder.run_sampler(dec, shift, beam_width=500)
console.log("Sampled passage")
return passage
def story(z, image_loc, k=100, bw=50, lyric=False):
"""
Generate a story for an image at location image_loc
"""
# Load the image
rawim, im = load_image(image_loc)
# Run image through convnet
feats = compute_features(z['net'], im).flatten()
feats /= norm(feats)
# Embed image into joint space
feats = embedding.encode_images(z['vse'], feats[None, :])
# Compute the nearest neighbours
scores = numpy.dot(feats, z['cvec'].T).flatten()
sorted_args = numpy.argsort(scores)[::-1]
sentences = [z['cap'][a] for a in sorted_args[:k]]
print 'NEAREST-CAPTIONS: '
output = []
for s in sentences[:1]:
output.append(s)
print s
print ''
# Compute skip-thought vectors for sentences
svecs = skipthoughts.encode(z['stv'], sentences, verbose=False)
# Style shifting
shift = svecs.mean(0) - z['bneg'] + z['bpos']
# Generate story conditioned on shift
passage = decoder.run_sampler(z['dec'], shift, beam_width=bw)
print 'OUTPUT: '
if lyric:
for line in passage.split(','):
if line[0] != ' ':
print line
else:
print line[1:]
else:
output.append(passage)
print passage
print "............................................."
return output
def story():
z = load_models()
z = load_inputs(z)
#Generate a story based on a given input
inputs = [z['cap'][a] for a in range(0, k)]
print 'Target inputs: '
for s in inputs[:5]:
print s
print ''
# encode the inputs into skip-thought vectors
vectors = encoder.encode(z['stv'], inputs)
# decode the vectors back into text
story = decoder.run_sampler(z['dec'], vectors, 50)
print 'OUTPUT: '
print story
def story(z, image_loc, k=100, bw=50, lyric=False):
"""
Generate a story for an image at location image_loc
"""
# Load the image
rawim, im = load_image(image_loc)
# Run image through convnet
feats = compute_features(z['net'], im).flatten()
feats /= norm(feats)
# Embed image into joint space
feats = embedding.encode_images(z['vse'], feats[None,:])
# Compute the nearest neighbours
scores = numpy.dot(feats, z['cvec'].T).flatten()
sorted_args = numpy.argsort(scores)[::-1]
sentences = [z['cap'][a] for a in sorted_args[:k]]
print 'NEAREST-CAPTIONS: '
for s in sentences[:5]:
print s
print ''
# Compute skip-thought vectors for sentences
svecs = skipthoughts.encode(z['stv'], sentences, verbose=False)
# Style shifting
shift = svecs.mean(0) - z['bneg'] + z['bpos']
# Generate story conditioned on shift
passage = decoder.run_sampler(z['dec'], shift, beam_width=bw)
print 'OUTPUT: '
if lyric:
for line in passage.split(','):
if line[0] != ' ':
print line
else:
print line[1:]
else:
print passage
return passage
def improved_story(z, story_in_sequence, k=100, bw=50, lyric=False):
'''
Inputs a story-in-sequence and outputs a styled story
'''
'''
TODO: Want to start with a story here, but it doesn't seem to be generated
in the original code.
'''
# Embed image into joint space
feats = embedding.encode_images(z['vse'], feats[None, :])
# Compute the nearest neighbours
scores = numpy.dot(feats, z['cvec'].T).flatten()
sorted_args = numpy.argsort(scores)[::-1]
stories_in_sequence = [z['cap'][a] for a in sorted_args[:k]]
print('NEAREST-STORIES-IN-SEQUENCE: ')
for s in stories_in_sequence[:5]:
print(s)
print('')
# Compute skip-thought vectors for stories-in-sequence
svecs = skipthoughts.encode(z['stv'], sentences, verbose=False)
# Style shifting
shift = svecs.mean(0) - z['bneg'] + z['bpos']
# Generate story conditioned on shift
passage = decoder.run_sampler(z['dec'], shift, beam_width=bw)
print('OUTPUT: ')
if lyric:
for line in passage.split(','):
if line[0] != ' ':
print(line)
else:
print((line[1:]))
else:
print(passage)
def story(z, image_locs, k=100, bw=50, lyric=False):
"""
Generate a story for an image at location image_loc
"""
if not isinstance(image_locs, list):
image_locs = [image_locs]
len_image_locs = len(image_locs)
# Num sentences per image_loc
num_sentences = k / len_image_locs
sentences = []
for image_loc in image_locs:
# Load single image
rawim, im = load_image(image_loc)
# Run image through convnet
feats = compute_features(z['net'], im).flatten()
feats /= norm(feats)
# Embed image into joint space
feats = embedding.encode_images(z['vse'], feats[None, :])
# Compute the nearest neighbours
scores = numpy.dot(feats, z['cvec'].T).flatten()
sorted_args = numpy.argsort(scores)[::-1]
sentences += [z['cap'][a] for a in sorted_args[:num_sentences]]
'''
Example as to what the following line is doing:
a = 100
b = 5
c = [x for x in range(a)]
print [c[a/b*j+i] for i in range(a/b) for j in range(b)] # [0, 20, 40, 80, 1, 21, 41, 61, ...]
'''
# Rearrange sentences round robin for image_locs
sentences = [
sentences[num_sentences * j + i] for i in range(num_sentences)
for j in range(len_image_locs)
]
print 'NEAREST-CAPTIONS: '
for s in sentences[:5]:
print s
print ''
# Compute skip-thought vectors for sentences
svecs = skipthoughts.encode(z['stv'], sentences, verbose=False)
# Style shifting
shift = svecs.mean(0) - z['bneg'] + z['bpos']
# Generate story conditioned on shift
passage = decoder.run_sampler(z['dec'], shift, beam_width=bw)
print 'OUTPUT: '
if lyric:
for line in passage.split(','):
if line[0] != ' ':
print line
else:
print line[1:]
else:
print passage
