def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1, image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model': model, 'update': update, 'regularize': regularize } # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(input_size, hidden_size, output_size): model = {} # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) # decoder model['bd'] = np.zeros((1, output_size)) update = ['WLSTM', 'Wd', 'bd'] regularize = ['WLSTM', 'Wd'] return {'model': model, 'update': update, 'regularize': regularize}
def init(input_size, hidden_size, output_size): model = {} # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) # decoder model['bd'] = np.zeros((1, output_size)) update = ['WLSTM', 'Wd', 'bd'] regularize = ['WLSTM', 'Wd'] return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(input_size, hidden_size, output_size): model = {} # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal model["WLSTM"] = initw(input_size + hidden_size + 1, 4 * hidden_size) # Decoder weights (e.g. mapping to vocabulary) model["Wd"] = initw(hidden_size, output_size) # decoder model["bd"] = np.zeros((1, output_size)) update = ["WLSTM", "Wd", "bd"] regularize = ["WLSTM", "Wd"] return {"model": model, "update": update, "regularize": regularize}
def init(input_size, hidden_size, output_size): model = {} # connections to h_{t-1} model['Whh'] = initw(hidden_size, hidden_size) model['bhh'] = np.zeros((1, hidden_size)) # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) * 0.01 # decoder model['bd'] = np.zeros((1, output_size)) update = ['Whh', 'bhh', 'Wd', 'bd'] regularize = ['Whh', 'Wd'] return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(input_size, hidden_size, output_size): model = {} # connections to h_{t-1} model['Whh'] = initw(hidden_size, hidden_size) model['bhh'] = np.zeros((1, hidden_size)) # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) * 0.01 # decoder model['bd'] = np.zeros((1, output_size)) update = ['Whh', 'bhh', 'Wd', 'bd'] regularize = ['Whh', 'Wd'] return {'model': model, 'update': update, 'regularize': regularize}
def forward(self,H,n,model,cache): ''' Produces one forward step of the given model based on the given hidden layers H ''' H2 = initw(n, self.d) A = model['A'+str(self.id)] # print(A) # print(str(A.size)) M = fillMMatrix(A, n ,self.N) #M = np.zeros((n,n)) #print(M) #U2=0 ''' if self.drop_prob_decoder > 0: # if we want dropout on the decoder if not self.predict_mode: # and we are in training mode scale2 = 1.0 / (1.0 - self.drop_prob_decoder) U2 = (np.random.rand(*(H.shape)) < (1 - self.drop_prob_decoder)) * scale2 # generate scaled mask H *= U2 # drop! ''' # Update Hmem Hmem = np.maximum(M.dot(H), 0) # Hidden Layer 2 for t in xrange(n): mem = Hmem[t - 1] H2[t] = np.maximum(H[t].dot(model['Whh'+str(self.id)]) + mem + model['bhh'+str(self.id)], 0) if not self.predict_mode: cache['Whh'+str(self.id)] = model['Whh'+str(self.id)] cache['H'+str(self.id)] = H2 cache['bhh'+str(self.id)] = model['bhh'+str(self.id)] cache['M'+str(self.id)] = M cache['Hmem'+str(self.id)] = Hmem #cache['U2']= U2 return H2,cache,M
def init(input_size, hidden_size, output_size): model = {} # connections to x_t model["Wxh"] = initw(input_size, hidden_size) model["bxh"] = np.zeros((1, hidden_size)) # connections to h_{t-1} model["Whh"] = initw(hidden_size, hidden_size) model["bhh"] = np.zeros((1, hidden_size)) # Decoder weights (e.g. mapping to vocabulary) model["Wd"] = initw(hidden_size, output_size) * 0.1 # decoder model["bd"] = np.zeros((1, output_size)) update = ["Whh", "bhh", "Wxh", "bxh", "Wd", "bd"] regularize = ["Whh", "Wxh", "Wd"] return {"model": model, "update": update, "regularize": regularize}
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) lda = params.get('lda',0) # LDA size used (0 if no LDA used) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here guide_input = params.get('guide',"image") if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] # Added for LDA if lda: model['Wlda'] = initw(lda,image_encoding_size) update.append('Wlda') regularize.append('Wlda') init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) #ADDED for gLSTM if(generator != 'glstm'): generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) else: if params['lda']: guide_size = get_guide_size(guide_input,params['lda']) else: guide_size = get_guide_size(guide_input,misc['ccaweights']) generator_init_struct = Generator.init(word_encoding_size, hidden_size, guide_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(input_size, hidden_size, output_size): model = {} # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) # model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size,1.) / np.sqrt(input_size + hidden_size) # model['WLSTM'][0,:] = 0 # initialize biases to zero # fancy_forget_bias_init = 3 # model['WLSTM'][0,hidden_size:2*hidden_size] = fancy_forget_bias_init # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) # decoder model['bd'] = np.zeros((1, output_size)) update = ['WLSTM', 'Wd', 'bd'] regularize = ['WLSTM', 'Wd'] return { 'model' : model, 'update' : update, 'regularize' : regularize }
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) ''' Generator: word vector와 hidden layer의 size 받아서 RNN/LSTM 초기화, model, update, regularize를 return model: RNN/LSTM 모델 자체 update: 학습시킬 weight들 model 중 학습 regularize: weight 중 regularize시킬 것들 ''' merge_init_structs(init_struct, generator_init_struct) ''' init_struct: We, be, Ws (각각 image encoder, image bias, word encoder generator_init_struct : RNN/LSTM 관련 weight들 ( decoder ) ''' return init_struct
def init(params, misc): # inputs feat_encoding_size = params.get('feat_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'mvlstm') output_size = len(misc['ixtoclass']) # these should match though v1_feat_size = params.get('v1_feat_size', 72) # hog v2_feat_size = params.get('v2_feat_size', 90) # hof v3_feat_size = params.get('v3_feat_size', 3) # head # initialize the encoder models model = {} model['We_v1'] = initw(v1_feat_size, feat_encoding_size) # feature encoder model['be_v1'] = np.zeros((1, feat_encoding_size)) model['We_v2'] = initw(v2_feat_size, feat_encoding_size) # feature encoder model['be_v2'] = np.zeros((1, feat_encoding_size)) model['We_v3'] = initw(v3_feat_size, feat_encoding_size) # feature encoder model['be_v3'] = np.zeros((1, feat_encoding_size)) update = ['We_v1', 'be_v1', 'We_v2', 'be_v2', 'We_v3', 'be_v3'] regularize = ['We_v1', 'We_v2', 'We_v3'] init_struct = { 'model': model, 'update': update, 'regularize': regularize } # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(feat_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(input_size, hidden_size, output_size): model = {} # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal # model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) #original # model['WLSTM'] = np.random.randn(input_size + hidden_size + 1, 4 * hidden_size) / np.sqrt(input_size + hidden_size) #Karpathy model['WLSTM'] = initw(input_size + hidden_size + 1, 4 * hidden_size) / np.sqrt(input_size + hidden_size) #Andrey's suggestion model['WLSTM'][0,:] = 0 # initialize biases to zero fancy_forget_bias_init = 3 # if fancy_forget_bias_init != 0: # forget gates get little bit negative bias initially to encourage them to be turned off # remember that due to Xavier initialization above, the raw output activations from gates before # nonlinearity are zero mean and on order of standard deviation ~1 model['WLSTM'][0,hidden_size:2*hidden_size] = fancy_forget_bias_init # Decoder weights (e.g. mapping to vocabulary) model['Wd'] = initw(hidden_size, output_size) # decoder model['bd'] = np.zeros((1, output_size)) update = ['WLSTM', 'Wd', 'bd'] regularize = ['WLSTM', 'Wd'] return { 'model' : model, 'update' : update, 'regularize' : regularize }