def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1, image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model': model, 'update': update, 'regularize': regularize } # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) lda = params.get('lda',0) # LDA size used (0 if no LDA used) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here guide_input = params.get('guide',"image") if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] # Added for LDA if lda: model['Wlda'] = initw(lda,image_encoding_size) update.append('Wlda') regularize.append('Wlda') init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) #ADDED for gLSTM if(generator != 'glstm'): generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) else: if params['lda']: guide_size = get_guide_size(guide_input,params['lda']) else: guide_size = get_guide_size(guide_input,misc['ccaweights']) generator_init_struct = Generator.init(word_encoding_size, hidden_size, guide_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct
def init(params, misc): # inputs image_encoding_size = params.get('image_encoding_size', 128) word_encoding_size = params.get('word_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'lstm') vocabulary_size = len(misc['wordtoix']) output_size = len(misc['ixtoword']) # these should match though image_size = 4096 # size of CNN vectors hardcoded here if generator == 'lstm': assert image_encoding_size == word_encoding_size, 'this implementation does not support different sizes for these parameters' # initialize the encoder models model = {} model['We'] = initw(image_size, image_encoding_size) # image encoder model['be'] = np.zeros((1,image_encoding_size)) model['Ws'] = initw(vocabulary_size, word_encoding_size) # word encoder update = ['We', 'be', 'Ws'] regularize = ['We', 'Ws'] init_struct = { 'model' : model, 'update' : update, 'regularize' : regularize} # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(word_encoding_size, hidden_size, output_size) ''' Generator: word vector와 hidden layer의 size 받아서 RNN/LSTM 초기화, model, update, regularize를 return model: RNN/LSTM 모델 자체 update: 학습시킬 weight들 model 중 학습 regularize: weight 중 regularize시킬 것들 ''' merge_init_structs(init_struct, generator_init_struct) ''' init_struct: We, be, Ws (각각 image encoder, image bias, word encoder generator_init_struct : RNN/LSTM 관련 weight들 ( decoder ) ''' return init_struct
def init(params, misc): # inputs feat_encoding_size = params.get('feat_encoding_size', 128) hidden_size = params.get('hidden_size', 128) generator = params.get('generator', 'mvlstm') output_size = len(misc['ixtoclass']) # these should match though v1_feat_size = params.get('v1_feat_size', 72) # hog v2_feat_size = params.get('v2_feat_size', 90) # hof v3_feat_size = params.get('v3_feat_size', 3) # head # initialize the encoder models model = {} model['We_v1'] = initw(v1_feat_size, feat_encoding_size) # feature encoder model['be_v1'] = np.zeros((1, feat_encoding_size)) model['We_v2'] = initw(v2_feat_size, feat_encoding_size) # feature encoder model['be_v2'] = np.zeros((1, feat_encoding_size)) model['We_v3'] = initw(v3_feat_size, feat_encoding_size) # feature encoder model['be_v3'] = np.zeros((1, feat_encoding_size)) update = ['We_v1', 'be_v1', 'We_v2', 'be_v2', 'We_v3', 'be_v3'] regularize = ['We_v1', 'We_v2', 'We_v3'] init_struct = { 'model': model, 'update': update, 'regularize': regularize } # descend into the specific Generator and initialize it Generator = decodeGenerator(generator) generator_init_struct = Generator.init(feat_encoding_size, hidden_size, output_size) merge_init_structs(init_struct, generator_init_struct) return init_struct