def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1, image_encoding_size))
model['Ws'] = initw(vocabulary_size,
word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = {
'model': model,
'update': update,
'regularize': regularize
}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size,
output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
lda = params.get('lda',0) # LDA size used (0 if no LDA used)
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
guide_input = params.get('guide',"image")
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
# Added for LDA
if lda:
model['Wlda'] = initw(lda,image_encoding_size)
update.append('Wlda')
regularize.append('Wlda')
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
#ADDED for gLSTM
if(generator != 'glstm'):
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
else:
if params['lda']:
guide_size = get_guide_size(guide_input,params['lda'])
else:
guide_size = get_guide_size(guide_input,misc['ccaweights'])
generator_init_struct = Generator.init(word_encoding_size, hidden_size, guide_size, output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
'''
Generator: word vector와 hidden layer의 size 받아서 RNN/LSTM 초기화,
model, update, regularize를 return
model: RNN/LSTM 모델 자체
update: 학습시킬 weight들 model 중 학습
regularize: weight 중 regularize시킬 것들
'''
merge_init_structs(init_struct, generator_init_struct)
'''
init_struct: We, be, Ws (각각 image encoder, image bias, word encoder
generator_init_struct : RNN/LSTM 관련 weight들 ( decoder )
'''
return init_struct
def init(params, misc):
# inputs
feat_encoding_size = params.get('feat_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'mvlstm')
output_size = len(misc['ixtoclass']) # these should match though
v1_feat_size = params.get('v1_feat_size', 72) # hog
v2_feat_size = params.get('v2_feat_size', 90) # hof
v3_feat_size = params.get('v3_feat_size', 3) # head
# initialize the encoder models
model = {}
model['We_v1'] = initw(v1_feat_size,
feat_encoding_size) # feature encoder
model['be_v1'] = np.zeros((1, feat_encoding_size))
model['We_v2'] = initw(v2_feat_size,
feat_encoding_size) # feature encoder
model['be_v2'] = np.zeros((1, feat_encoding_size))
model['We_v3'] = initw(v3_feat_size,
feat_encoding_size) # feature encoder
model['be_v3'] = np.zeros((1, feat_encoding_size))
update = ['We_v1', 'be_v1', 'We_v2', 'be_v2', 'We_v3', 'be_v3']
regularize = ['We_v1', 'We_v2', 'We_v3']
init_struct = {
'model': model,
'update': update,
'regularize': regularize
}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(feat_encoding_size, hidden_size,
output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
