def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1, image_encoding_size))
model['Ws'] = initw(vocabulary_size,
word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = {
'model': model,
'update': update,
'regularize': regularize
}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size,
output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(input_size, hidden_size, output_size):
model = {}
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size)
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) # decoder
model['bd'] = np.zeros((1, output_size))
update = ['WLSTM', 'Wd', 'bd']
regularize = ['WLSTM', 'Wd']
return {'model': model, 'update': update, 'regularize': regularize}
def init(input_size, hidden_size, output_size):
model = {}
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size)
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) # decoder
model['bd'] = np.zeros((1, output_size))
update = ['WLSTM', 'Wd', 'bd']
regularize = ['WLSTM', 'Wd']
return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(input_size, hidden_size, output_size):
model = {}
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
model["WLSTM"] = initw(input_size + hidden_size + 1, 4 * hidden_size)
# Decoder weights (e.g. mapping to vocabulary)
model["Wd"] = initw(hidden_size, output_size) # decoder
model["bd"] = np.zeros((1, output_size))
update = ["WLSTM", "Wd", "bd"]
regularize = ["WLSTM", "Wd"]
return {"model": model, "update": update, "regularize": regularize}
def init(input_size, hidden_size, output_size):
model = {}
# connections to h_{t-1}
model['Whh'] = initw(hidden_size, hidden_size)
model['bhh'] = np.zeros((1, hidden_size))
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) * 0.01 # decoder
model['bd'] = np.zeros((1, output_size))
update = ['Whh', 'bhh', 'Wd', 'bd']
regularize = ['Whh', 'Wd']
return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(input_size, hidden_size, output_size):
model = {}
# connections to h_{t-1}
model['Whh'] = initw(hidden_size, hidden_size)
model['bhh'] = np.zeros((1, hidden_size))
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) * 0.01 # decoder
model['bd'] = np.zeros((1, output_size))
update = ['Whh', 'bhh', 'Wd', 'bd']
regularize = ['Whh', 'Wd']
return {'model': model, 'update': update, 'regularize': regularize}
def forward(self,H,n,model,cache):
''' Produces one forward step of the given model based on the given hidden layers H '''
H2 = initw(n, self.d)
A = model['A'+str(self.id)]
# print(A)
# print(str(A.size))
M = fillMMatrix(A, n ,self.N)
#M = np.zeros((n,n))
#print(M)
#U2=0
'''
if self.drop_prob_decoder > 0: # if we want dropout on the decoder
if not self.predict_mode: # and we are in training mode
scale2 = 1.0 / (1.0 - self.drop_prob_decoder)
U2 = (np.random.rand(*(H.shape)) < (1 - self.drop_prob_decoder)) * scale2 # generate scaled mask
H *= U2 # drop!
'''
# Update Hmem
Hmem = np.maximum(M.dot(H), 0)
# Hidden Layer 2
for t in xrange(n):
mem = Hmem[t - 1]
H2[t] = np.maximum(H[t].dot(model['Whh'+str(self.id)]) + mem + model['bhh'+str(self.id)], 0)
if not self.predict_mode:
cache['Whh'+str(self.id)] = model['Whh'+str(self.id)]
cache['H'+str(self.id)] = H2
cache['bhh'+str(self.id)] = model['bhh'+str(self.id)]
cache['M'+str(self.id)] = M
cache['Hmem'+str(self.id)] = Hmem
#cache['U2']= U2
return H2,cache,M
def init(input_size, hidden_size, output_size):
model = {}
# connections to x_t
model["Wxh"] = initw(input_size, hidden_size)
model["bxh"] = np.zeros((1, hidden_size))
# connections to h_{t-1}
model["Whh"] = initw(hidden_size, hidden_size)
model["bhh"] = np.zeros((1, hidden_size))
# Decoder weights (e.g. mapping to vocabulary)
model["Wd"] = initw(hidden_size, output_size) * 0.1 # decoder
model["bd"] = np.zeros((1, output_size))
update = ["Whh", "bhh", "Wxh", "bxh", "Wd", "bd"]
regularize = ["Whh", "Wxh", "Wd"]
return {"model": model, "update": update, "regularize": regularize}
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
lda = params.get('lda',0) # LDA size used (0 if no LDA used)
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
guide_input = params.get('guide',"image")
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
# Added for LDA
if lda:
model['Wlda'] = initw(lda,image_encoding_size)
update.append('Wlda')
regularize.append('Wlda')
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
#ADDED for gLSTM
if(generator != 'glstm'):
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
else:
if params['lda']:
guide_size = get_guide_size(guide_input,params['lda'])
else:
guide_size = get_guide_size(guide_input,misc['ccaweights'])
generator_init_struct = Generator.init(word_encoding_size, hidden_size, guide_size, output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(input_size, hidden_size, output_size):
model = {}
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size)
# model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size,1.) / np.sqrt(input_size + hidden_size)
# model['WLSTM'][0,:] = 0 # initialize biases to zero
# fancy_forget_bias_init = 3
# model['WLSTM'][0,hidden_size:2*hidden_size] = fancy_forget_bias_init
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) # decoder
model['bd'] = np.zeros((1, output_size))
update = ['WLSTM', 'Wd', 'bd']
regularize = ['WLSTM', 'Wd']
return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(params, misc):
# inputs
image_encoding_size = params.get('image_encoding_size', 128)
word_encoding_size = params.get('word_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'lstm')
vocabulary_size = len(misc['wordtoix'])
output_size = len(misc['ixtoword']) # these should match though
image_size = 4096 # size of CNN vectors hardcoded here
if generator == 'lstm':
assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters'
# initialize the encoder models
model = {}
model['We'] = initw(image_size, image_encoding_size) # image encoder
model['be'] = np.zeros((1,image_encoding_size))
model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder
update = ['We', 'be', 'Ws']
regularize = ['We', 'Ws']
init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size)
'''
Generator: word vector와 hidden layer의 size 받아서 RNN/LSTM 초기화,
model, update, regularize를 return
model: RNN/LSTM 모델 자체
update: 학습시킬 weight들 model 중 학습
regularize: weight 중 regularize시킬 것들
'''
merge_init_structs(init_struct, generator_init_struct)
'''
init_struct: We, be, Ws (각각 image encoder, image bias, word encoder
generator_init_struct : RNN/LSTM 관련 weight들 ( decoder )
'''
return init_struct
def init(params, misc):
# inputs
feat_encoding_size = params.get('feat_encoding_size', 128)
hidden_size = params.get('hidden_size', 128)
generator = params.get('generator', 'mvlstm')
output_size = len(misc['ixtoclass']) # these should match though
v1_feat_size = params.get('v1_feat_size', 72) # hog
v2_feat_size = params.get('v2_feat_size', 90) # hof
v3_feat_size = params.get('v3_feat_size', 3) # head
# initialize the encoder models
model = {}
model['We_v1'] = initw(v1_feat_size,
feat_encoding_size) # feature encoder
model['be_v1'] = np.zeros((1, feat_encoding_size))
model['We_v2'] = initw(v2_feat_size,
feat_encoding_size) # feature encoder
model['be_v2'] = np.zeros((1, feat_encoding_size))
model['We_v3'] = initw(v3_feat_size,
feat_encoding_size) # feature encoder
model['be_v3'] = np.zeros((1, feat_encoding_size))
update = ['We_v1', 'be_v1', 'We_v2', 'be_v2', 'We_v3', 'be_v3']
regularize = ['We_v1', 'We_v2', 'We_v3']
init_struct = {
'model': model,
'update': update,
'regularize': regularize
}
# descend into the specific Generator and initialize it
Generator = decodeGenerator(generator)
generator_init_struct = Generator.init(feat_encoding_size, hidden_size,
output_size)
merge_init_structs(init_struct, generator_init_struct)
return init_struct
def init(input_size, hidden_size, output_size):
model = {}
# Recurrent weights: take x_t, h_{t-1}, and bias unit
# and produce the 3 gates and the input to cell signal
# model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) #original
# model['WLSTM'] = np.random.randn(input_size + hidden_size + 1, 4 * hidden_size) / np.sqrt(input_size + hidden_size) #Karpathy
model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) / np.sqrt(input_size + hidden_size) #Andrey's suggestion
model['WLSTM'][0,:] = 0 # initialize biases to zero
fancy_forget_bias_init = 3
# if fancy_forget_bias_init != 0:
# forget gates get little bit negative bias initially to encourage them to be turned off
# remember that due to Xavier initialization above, the raw output activations from gates before
# nonlinearity are zero mean and on order of standard deviation ~1
model['WLSTM'][0,hidden_size:2*hidden_size] = fancy_forget_bias_init
# Decoder weights (e.g. mapping to vocabulary)
model['Wd'] = initw(hidden_size, output_size) # decoder
model['bd'] = np.zeros((1, output_size))
update = ['WLSTM', 'Wd', 'bd']
regularize = ['WLSTM', 'Wd']
return { 'model' : model, 'update' : update, 'regularize' : regularize }
