def lstm(self, data, markers, lstm_static=None, lstm_hidden=1000, weight_filler=None, bias_filler=None, learning_param_lstm=None): #default params if not weight_filler: weight_filler = self.uniform_weight_filler(-.08, .08) if not bias_filler: bias_filler = self.constant_filler(0) if not learning_param_lstm: learning_param_lstm = self.init_params([[1, 1], [1, 1], [1, 1]]) if lstm_static: return L.LSTM(data, markers, lstm_static, param=learning_param_lstm, recurrent_param=dict(num_output=lstm_hidden, weight_filler=weight_filler, bias_filler=bias_filler)) else: return L.LSTM(data, markers, param=learning_param_lstm, recurrent_param=dict(num_output=lstm_hidden, weight_filler=weight_filler, bias_filler=bias_filler))
def PlateNetBody(net, data_layer, time_step, num_classes): # lstm_kwargs = { # 'weight_filler': dict(type='xavier'), # 'bias_filler': dict(type='constant', value=0)} kwargs = { 'param': [dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)], 'weight_filler': dict(type='xavier'), 'bias_filler': dict(type='constant', value=0) } # assert from_layer in net.keys() # 48 x 48 recurrent_param = { 'num_output': 100, 'weight_filler': dict(type='xavier'), 'bias_filler': dict(type='constant', value=0) } net.indicator = L.ContinuationIndicator(time_step=time_step, batch_size=512) net.permuted_data = L.Permute(data_layer, order=[3, 0, 1, 2]) net.lstm1 = L.LSTM(net.permuted_data, net.indicator, recurrent_param=recurrent_param) net.lstm2 = L.LSTM(net.lstm1, net.indicator, recurrent_param=recurrent_param) net.fc1 = L.InnerProduct(net.lstm2, num_output=num_classes + 1, axis=2, **kwargs) return net
def exp_proto(mode, batchsize, T, exp_T, question_vocab_size, exp_vocab_size): n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) n.exp_att_feature, n.exp, n.exp_out, n.exp_cont_1, n.exp_cont_2 = \ L.Python(module='exp_data_provider_layer', layer='ExpDataProviderLayer', param_str=mode_str, ntop=5) n.exp_embed_ba = L.Embed(n.exp, input_dim=exp_vocab_size, num_output=300, \ weight_filler=dict(type='uniform', min=-0.08, max=0.08)) n.exp_embed = L.TanH(n.exp_embed_ba) # LSTM1 for Explanation n.exp_lstm1 = L.LSTM(\ n.exp_embed, n.exp_cont_1,\ recurrent_param=dict(\ num_output=2048,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) n.exp_lstm1_dropped = L.Dropout(n.exp_lstm1, dropout_param={'dropout_ratio': 0.3}) # Merge with LSTM1 for explanation n.exp_att_resh = L.Reshape( n.exp_att_feature, reshape_param=dict(shape=dict(dim=[1, -1, 2048]))) n.exp_att_tiled = L.Tile(n.exp_att_resh, axis=0, tiles=exp_T) n.exp_eltwise_all = L.Eltwise(n.exp_lstm1_dropped, n.exp_att_tiled, eltwise_param={'operation': P.Eltwise.PROD}) n.exp_eltwise_all_sqrt = L.SignedSqrt(n.exp_eltwise_all) n.exp_eltwise_all_l2 = L.L2Normalize(n.exp_eltwise_all_sqrt) n.exp_eltwise_all_drop = L.Dropout(n.exp_eltwise_all_l2, dropout_param={'dropout_ratio': 0.3}) # LSTM2 for Explanation n.exp_lstm2 = L.LSTM(\ n.exp_eltwise_all_drop, n.exp_cont_2,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) n.exp_lstm2_dropped = L.Dropout(n.exp_lstm2, dropout_param={'dropout_ratio': 0.3}) n.exp_prediction = L.InnerProduct(n.exp_lstm2_dropped, num_output=exp_vocab_size, weight_filler=dict(type='xavier'), axis=2) n.silence_exp_prediction = L.Silence(n.exp_prediction, ntop=0) return n.to_proto()
def lstm(bottom, clip, nout, param_name=''): recurrent_param = dict(num_output=nout, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) return L.LSTM(bottom, clip, recurrent_param=recurrent_param, param=[ dict(name=param_name + '-w'), dict(name=param_name + '-u'), dict(name=param_name + '-b', decay_mult=0) ])
def generate_scores(split, config): n = caffe.NetSpec() batch_size = config.N mode_str = str(dict(split=split, batch_size=batch_size)) n.language, n.cont, n.img_feature, n.spatial, n.label = L.Python(module=config.data_provider, layer='TossLayer', param_str=mode_str, ntop=5) # embedding n.embed = L.Embed(n.language, input_dim=config.vocab_size, num_output=config.embed_dim, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) # LSTM n.lstm = L.LSTM(n.embed, n.cont, recurrent_param=dict(num_output=config.lstm_dim, weight_filler=dict(type='uniform', min=-0.08, max=0.08), bias_filler=dict(type='constant', value=0))) tops = L.Slice(n.lstm, ntop=config.T, slice_param=dict(axis=0)) for i in range(config.T - 1): n.__setattr__('slice'+str(i), tops[i]) n.__setattr__('silence'+str(i), L.Silence(tops[i], ntop=0)) n.lstm_out = tops[-1] n.lstm_feat = L.Reshape(n.lstm_out, reshape_param=dict(shape=dict(dim=[-1, config.lstm_dim]))) # L2 Normalize image and language features n.img_l2norm = L.L2Normalize(n.img_feature) n.lstm_l2norm = L.L2Normalize(n.lstm_feat) n.img_l2norm_resh = L.Reshape(n.img_l2norm, reshape_param=dict(shape=dict(dim=[-1, config.D_im]))) n.lstm_l2norm_resh = L.Reshape(n.lstm_l2norm, reshape_param=dict(shape=dict(dim=[-1, config.D_text]))) # Concatenate n.feat_all = L.Concat(n.lstm_l2norm_resh, n.img_l2norm_resh, n.spatial, concat_param=dict(axis=1)) # MLP Classifier over concatenated feature n.mlp_l1, n.mlp_relu1 = fc_relu(n.feat_all, config.mlp_hidden_dims) if config.mlp_dropout: n.mlp_drop1 = L.Dropout(n.mlp_relu1, dropout_ratio=0.5, in_place=True) n.scores = fc(n.mlp_drop1, 1) else: n.scores = fc(n.mlp_relu1, 1) # Loss Layer n.loss = L.SigmoidCrossEntropyLoss(n.scores, n.label) return n.to_proto()
def language_model_lstm_no_embed(self, sent_bottom, cont_bottom, text_name='embedding_text', tag=''): lstm_lr = self.args.lstm_lr embedding_lr = self.args.language_embedding_lr lstm = L.LSTM( sent_bottom, cont_bottom, recurrent_param=dict(num_output=self.language_embedding_dim[0], weight_filler=self.uniform_weight_filler( -0.08, 0.08), bias_filler=self.constant_filler(0)), param=self.learning_params( [[lstm_lr, lstm_lr], [lstm_lr, lstm_lr], [lstm_lr, lstm_lr]], ['lstm1' + tag, 'lstm2' + tag, 'lstm3' + tag])) lstm_slices = L.Slice(lstm, slice_point=self.params['sentence_length'] - 1, axis=0, ntop=2) self.n.tops['silence_cell_' + str(self.silence_count)] = L.Silence( lstm_slices[0], ntop=0) self.silence_count += 1 top_lstm = L.Reshape( lstm_slices[1], shape=dict(dim=[-1, self.language_embedding_dim[0]])) top_text = L.InnerProduct( top_lstm, num_output=self.language_embedding_dim[1], weight_filler=self.uniform_weight_filler(-0.08, .08), bias_filler=self.constant_filler(0), param=self.learning_params( [[embedding_lr, embedding_lr], [embedding_lr * 2, 0]], ['lstm_embed1' + tag, 'lstm_embed_1b' + tag])) setattr(self.n, text_name, top_text) return top_text
def _make_module(model_path, in1_shape, time_step, batch_size, num_output): ns = caffe.NetSpec() ns.data1 = L.Input(name="data1", input_param={"shape": {"dim": in1_shape}}) ns.data2 = L.ContinuationIndicator(name="data2", continuation_indicator_param={ "time_step": time_step, "batch_size": batch_size }) ns.lstm = L.LSTM(ns.data1, ns.data2, name="lstm", recurrent_param={"num_output": num_output}) with open(os.path.join(model_path, 'test.prototxt'), 'w') as f: f.write(str(ns.to_proto())) net = caffe.Net(f.name, caffe.TEST) for l in net.layers: for b in l.blobs: if np.count_nonzero(b.data) == 0: b.data[...] = np.random.randn(*b.data.shape) net.save(os.path.join(model_path, 'test.caffemodel'))
def jsonToPrototxt(net, net_name): # assumption: a layer can accept only one input blob # the data layer produces two blobs: data and label # the loss layer requires two blobs: <someData> and label # the label blob is hardcoded. # layers name have to be unique # custom DFS of the network input_dim = None def get_iterable(x): if isinstance(x, collections.Iterable): return x else: return (x, ) stack = [] layersProcessed = {} processOrder = [] blobNames = {} for layerId in net: layersProcessed[layerId] = False blobNames[layerId] = { 'bottom': [], 'top': [], } blobId = 0 def isProcessPossible(layerId): inputs = net[layerId]['connection']['input'] for layerId in inputs: if layersProcessed[layerId] is False: return False return True # finding the data layer for layerId in net: if (net[layerId]['info']['type'] == 'Data' or net[layerId]['info']['type'] == 'Input' or net[layerId]['info']['type'] == 'HDF5Data'): stack.append(layerId) def changeTopBlobName(layerId, newName): blobNames[layerId]['top'] = newName while len(stack): i = len(stack) - 1 while isProcessPossible(stack[i]) is False: i = i - 1 layerId = stack[i] stack.remove(stack[i]) inputs = net[layerId]['connection']['input'] if len(inputs) > 0: if len(inputs) == 2 and (net[inputs[0]]['info']['phase'] is not None) \ and (net[inputs[1]]['info']['phase']): commonBlobName = blobNames[inputs[0]]['top'] changeTopBlobName(inputs[1], commonBlobName) blobNames[layerId]['bottom'] = commonBlobName else: inputBlobNames = [] for inputId in inputs: inputBlobNames.extend(blobNames[inputId]['top']) blobNames[layerId]['bottom'] = inputBlobNames blobNames[layerId]['top'] = ['blob' + str(blobId)] blobId = blobId + 1 for outputId in net[layerId]['connection']['output']: if outputId not in stack: stack.append(outputId) layersProcessed[layerId] = True processOrder.append(layerId) ns_train = caffe.NetSpec() ns_test = caffe.NetSpec() for layerId in processOrder: layer = net[layerId] layerParams = layer['params'] layerType = layer['info']['type'] layerPhase = layer['info']['phase'] if (layerType == 'Data' or layerType == 'Input'): # This is temporary # Has to be improved later # If we have data layer then it is converted to input layer with some default dimensions ''' data_param = {} if layerParams['source'] != '': data_param['source'] = layerParams['source'] # hardcoding mnsit dataset -change this later if layerPhase is not None: if int(layerPhase) == 0: data_param['source'] = 'examples/mnist/mnist_train_lmdb' elif int(layerPhase) == 1: data_param['source'] = 'examples/mnist/mnist_test_lmdb' if layerParams['batch_size'] != '': data_param['batch_size'] = int(float(layerParams['batch_size'])) if layerParams['backend'] != '': backend = layerParams['backend'] if(backend == 'LEVELDB'): backend = 0 elif(backend == 'LMDB'): backend = 1 data_param['backend'] = backend transform_param = {} if layerParams['scale'] != '': transform_param['scale'] = float(layerParams['scale']) if layerPhase is not None: caffeLayer = get_iterable(L.Data( ntop=1, transform_param=transform_param, data_param=data_param, include={ 'phase': int(layerPhase) })) if int(layerPhase) == 0: #for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_train[key] = value elif int(layerPhase) == 1: #for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_test[key] = value else: for ns in (ns_train,ns_test): caffeLayer = get_iterable(L.Data( ntop=2, transform_param=transform_param, data_param=data_param)) #for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value ''' if 'dim' not in layerParams: layerParams['dim'] = '10,3,224,224' input_dim = layerParams['dim'] if layerPhase is not None: caffeLayer = get_iterable( L.Input(input_param={ 'shape': { 'dim': map(int, layerParams['dim'].split(',')) } }, include={'phase': int(layerPhase)})) if int(layerPhase) == 0: # for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_train[key] = value elif int(layerPhase) == 1: # for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_test[key] = value else: for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Input( input_param={ 'shape': { 'dim': map(int, layerParams['dim'].split( ',')) } })) # for key, value in zip(blobNames[layerId]['top'] + ['label'], caffeLayer): for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Crop'): crop_param = {} if layerParams['axis'] != '': crop_param['axis'] = int(float(layerParams['axis'])) if layerParams['offset'] != '': crop_param['offset'] = int(float(layerParams['offset'])) for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Crop(*[ns[x] for x in blobNames[layerId]['bottom']], crop_param=crop_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Convolution'): convolution_param = {} if layerParams['kernel_h'] != '': convolution_param['kernel_h'] = int( float(layerParams['kernel_h'])) if layerParams['kernel_w'] != '': convolution_param['kernel_w'] = int( float(layerParams['kernel_w'])) if layerParams['stride_h'] != '': convolution_param['stride_h'] = int( float(layerParams['stride_h'])) if layerParams['stride_w'] != '': convolution_param['stride_w'] = int( float(layerParams['stride_w'])) if layerParams['num_output'] != '': convolution_param['num_output'] = int( float(layerParams['num_output'])) if layerParams['pad_h'] != '': convolution_param['pad_h'] = int(float(layerParams['pad_h'])) if layerParams['pad_w'] != '': convolution_param['pad_w'] = int(float(layerParams['pad_w'])) if layerParams['weight_filler'] != '': convolution_param['weight_filler'] = {} convolution_param['weight_filler']['type'] = layerParams[ 'weight_filler'] if layerParams['bias_filler'] != '': convolution_param['bias_filler'] = {} convolution_param['bias_filler']['type'] = layerParams[ 'bias_filler'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Convolution( *[ns[x] for x in blobNames[layerId]['bottom']], convolution_param=convolution_param, param=[{ 'lr_mult': 1 }, { 'lr_mult': 2 }])) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Deconvolution'): convolution_param = {} if layerParams['kernel_h'] != '': convolution_param['kernel_h'] = int( float(layerParams['kernel_h'])) if layerParams['kernel_w'] != '': convolution_param['kernel_w'] = int( float(layerParams['kernel_w'])) if layerParams['stride_h'] != '': convolution_param['stride_h'] = int( float(layerParams['stride_h'])) if layerParams['stride_w'] != '': convolution_param['stride_w'] = int( float(layerParams['stride_w'])) if layerParams['num_output'] != '': convolution_param['num_output'] = int( float(layerParams['num_output'])) if layerParams['pad_h'] != '': convolution_param['pad_h'] = int(float(layerParams['pad_h'])) if layerParams['pad_w'] != '': convolution_param['pad_w'] = int(float(layerParams['pad_w'])) if layerParams['weight_filler'] != '': convolution_param['weight_filler'] = {} convolution_param['weight_filler']['type'] = layerParams[ 'weight_filler'] if layerParams['bias_filler'] != '': convolution_param['bias_filler'] = {} convolution_param['bias_filler']['type'] = layerParams[ 'bias_filler'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Deconvolution( *[ns[x] for x in blobNames[layerId]['bottom']], convolution_param=convolution_param, param=[{ 'lr_mult': 1 }, { 'lr_mult': 2 }])) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'ReLU'): inplace = layerParams['inplace'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.ReLU(*[ns[x] for x in blobNames[layerId]['bottom']], in_place=inplace)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Pooling'): pooling_param = {} if layerParams['kernel_h'] != '': pooling_param['kernel_h'] = int(float(layerParams['kernel_h'])) if layerParams['kernel_w'] != '': pooling_param['kernel_w'] = int(float(layerParams['kernel_w'])) if layerParams['stride_h'] != '': pooling_param['stride_h'] = int(float(layerParams['stride_h'])) if layerParams['stride_w'] != '': pooling_param['stride_w'] = int(float(layerParams['stride_w'])) if layerParams['pad_h'] != '': pooling_param['pad_h'] = int(float(layerParams['pad_h'])) if layerParams['pad_w'] != '': pooling_param['pad_w'] = int(float(layerParams['pad_w'])) if layerParams['pool'] != '': pool = layerParams['pool'] if (pool == 'MAX'): pool = 0 elif (pool == 'AVE'): pool = 1 elif (pool == 'STOCHASTIC'): pool = 2 pooling_param['pool'] = pool for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Pooling(*[ns[x] for x in blobNames[layerId]['bottom']], pooling_param=pooling_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'InnerProduct'): inner_product_param = {} if layerParams['num_output'] != '': inner_product_param['num_output'] = int( float(layerParams['num_output'])) if layerParams['weight_filler'] != '': inner_product_param['weight_filler'] = {} inner_product_param['weight_filler']['type'] = layerParams[ 'weight_filler'] if layerParams['bias_filler'] != '': inner_product_param['bias_filler'] = {} inner_product_param['bias_filler']['type'] = layerParams[ 'bias_filler'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.InnerProduct( *[ns[x] for x in blobNames[layerId]['bottom']], inner_product_param=inner_product_param, param=[{ 'lr_mult': 1 }, { 'lr_mult': 2 }])) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'SoftmaxWithLoss'): pass for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.SoftmaxWithLoss( # try L['SoftmaxWithLoss'] *([ns[x] for x in blobNames[layerId]['bottom']]))) # *([ns[x] for x in blobNames[layerId]['bottom']] + [ns.label]))) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Accuracy'): pass if layerPhase is not None: caffeLayer = get_iterable( L.Accuracy( *([ns[x] for x in blobNames[layerId]['bottom']]), # *([ns[x] for x in blobNames[layerId]['bottom']] + [ns.label]), include={'phase': int(layerPhase)})) if int(layerPhase) == 0: for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_train[key] = value elif int(layerPhase) == 1: for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns_test[key] = value else: for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Accuracy( *([ns[x] for x in blobNames[layerId]['bottom']]))) # *([ns[x] for x in blobNames[layerId]['bottom']] + [ns.label]))) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Dropout'): # inplace dropout? caffe-tensorflow do not work inplace = layerParams['inplace'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Dropout(*[ns[x] for x in blobNames[layerId]['bottom']], in_place=inplace)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'LRN'): for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.LRN(*[ns[x] for x in blobNames[layerId]['bottom']])) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Concat'): for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Concat(*[ns[x] for x in blobNames[layerId]['bottom']], ntop=len(blobNames[layerId]['top']))) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Eltwise'): eltwise_param = {} if layerParams['operation'] != '': elt = layerParams['operation'] if (elt == 'PROD'): elt = 0 elif (elt == 'SUM'): elt = 1 elif (elt == 'MAX'): elt = 2 else: elt = 1 # Default is sum eltwise_param['operation'] = elt for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Eltwise(*[ns[x] for x in blobNames[layerId]['bottom']], eltwise_param=eltwise_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Softmax'): for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Softmax(*([ns[x] for x in blobNames[layerId]['bottom']]))) # *([ns[x] for x in blobNames[layerId]['bottom']] + [ns.label]))) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Embed'): for ns in (ns_train, ns_test): print ns.tops caffeLayer = get_iterable( L.Embed(*[ns[x] for x in blobNames[layerId]['bottom']], param=[{ 'lr_mult': 1, 'decay_mult': 1 }, { 'lr_mult': 2, 'decay_mult': 0 }])) # *([ns[x] for x in blobNames[layerId]['bottom']] + [ns.label]))) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'LSTM'): recurrent_param = {} if layerParams['num_output'] != '': recurrent_param['num_output'] = int(layerParams['num_output']) if layerParams['weight_filler'] != '': recurrent_param['weight_filler'] = { 'type': layerParams['weight_filler'] } if layerParams['bias_filler'] != '': recurrent_param['bias_filler'] = { 'type': layerParams['bias_filler'] } for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.LSTM(*[ns[x] for x in blobNames[layerId]['bottom']], recurrent_param=recurrent_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Reshape'): reshape_param = { 'shape': { 'dim': map(int, layerParams['dim'].split(',')) } } for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Reshape(*[ns[x] for x in blobNames[layerId]['bottom']], reshape_param=reshape_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'HDF5Data'): layerPhase = layer['info']['phase'] hdf5_data_param = {} if layerParams['source'] != '': hdf5_data_param['source'] = layerParams['source'] if layerParams['batch_size'] != '': hdf5_data_param['batch_size'] = layerParams['batch_size'] for ns in (ns_train, ns_test): if layerPhase is not None: caffeLayer = get_iterable( L.HDF5Data( *[ns[x] for x in blobNames[layerId]['bottom']], hdf5_data_param=hdf5_data_param, include={'phase': int(layerPhase)})) else: caffeLayer = get_iterable( L.HDF5Data( *[ns[x] for x in blobNames[layerId]['bottom']], hdf5_data_param=hdf5_data_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'BatchNorm'): batch_norm_param = {} if layerParams['use_global_stats'] != '': batch_norm_param['use_global_stats'] = layerParams[ 'use_global_stats'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.BatchNorm(*[ns[x] for x in blobNames[layerId]['bottom']], batch_norm_param=batch_norm_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Scale'): scale_param = {} if layerParams['bias_term'] != '': scale_param['bias_term'] = layerParams['bias_term'] for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Scale(*[ns[x] for x in blobNames[layerId]['bottom']], scale_param=scale_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value elif (layerType == 'Eltwise'): eltwise_param = {} if layerParams['operation'] != '': eltwise_param['operation'] = int(layerParams['operation']) for ns in (ns_train, ns_test): caffeLayer = get_iterable( L.Eltwise(*[ns[x] for x in blobNames[layerId]['bottom']], eltwise_param=eltwise_param)) for key, value in zip(blobNames[layerId]['top'], caffeLayer): ns[key] = value train = 'name: "' + net_name + '"\n' + str(ns_train.to_proto()) test = str(ns_test.to_proto()) # merge the train and test prototxt to get a single train_test prototxt testIndex = [m.start() for m in re.finditer('layer', test)] previousIndex = -1 for i in range(len(testIndex)): if i < len(testIndex) - 1: layer = test[testIndex[i]:testIndex[i + 1]] else: layer = test[testIndex[i]:] a = train.find(layer) if a != -1: l = test[testIndex[previousIndex + 1]:testIndex[i]] train = train[0:a] + l + train[a:] previousIndex = i if previousIndex < len(testIndex) - 1: l = test[testIndex[previousIndex + 1]:] train = train + l prototxt = train return prototxt, input_dim
def generator_proto(mode, batchsize, T, exp_T, question_vocab_size, exp_vocab_size, use_gt=True): n = caffe.NetSpec() mode_str = json.dumps({'mode':mode, 'batchsize':batchsize}) n.data, n.cont, n.img_feature, n.label, n.exp, n.exp_out, n.exp_cont_1, n.exp_cont_2 = \ L.Python(module='vqa_data_provider_layer', layer='VQADataProviderLayer', param_str=mode_str, ntop=8) n.embed_ba = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='uniform',min=-0.08,max=0.08), param=fixed_weights) n.embed = L.TanH(n.embed_ba) # LSTM1 n.lstm1 = L.LSTM(\ n.embed, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0)), param=fixed_weights_lstm) tops1 = L.Slice(n.lstm1, ntop=T, slice_param={'axis':0}) for i in range(T-1): n.__setattr__('slice_first'+str(i), tops1[int(i)]) n.__setattr__('silence_data_first'+str(i), L.Silence(tops1[int(i)],ntop=0)) n.lstm1_out = tops1[T-1] n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm1_reshaped_droped = L.Dropout(n.lstm1_reshaped,dropout_param={'dropout_ratio':0.3}) n.lstm1_droped = L.Dropout(n.lstm1,dropout_param={'dropout_ratio':0.3}) # LSTM2 n.lstm2 = L.LSTM(\ n.lstm1_droped, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0)), param=fixed_weights_lstm) tops2 = L.Slice(n.lstm2, ntop=T, slice_param={'axis':0}) for i in range(T-1): n.__setattr__('slice_second'+str(i), tops2[int(i)]) n.__setattr__('silence_data_second'+str(i), L.Silence(tops2[int(i)],ntop=0)) n.lstm2_out = tops2[T-1] n.lstm2_reshaped = L.Reshape(n.lstm2_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm2_reshaped_droped = L.Dropout(n.lstm2_reshaped,dropout_param={'dropout_ratio':0.3}) concat_botom = [n.lstm1_reshaped_droped, n.lstm2_reshaped_droped] n.lstm_12 = L.Concat(*concat_botom) # Tile question feature n.q_emb_resh = L.Reshape(n.lstm_12, reshape_param=dict(shape=dict(dim=[-1,2048,1,1]))) n.q_emb_tiled_1 = L.Tile(n.q_emb_resh, axis=2, tiles=14) n.q_emb_resh_tiled = L.Tile(n.q_emb_tiled_1, axis=3, tiles=14) # Embed image feature n.i_emb = L.Convolution(n.img_feature, kernel_size=1, stride=1, num_output=2048, pad=0, weight_filler=dict(type='xavier'), param=fixed_weights) # Eltwise product and normalization n.eltwise = L.Eltwise(n.q_emb_resh_tiled, n.i_emb, eltwise_param={'operation': P.Eltwise.PROD}) n.eltwise_sqrt = L.SignedSqrt(n.eltwise) n.eltwise_l2 = L.L2Normalize(n.eltwise_sqrt) n.eltwise_drop = L.Dropout(n.eltwise_l2, dropout_param={'dropout_ratio': 0.3}) # Attention for VQA n.att_conv1 = L.Convolution(n.eltwise_drop, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier'), param=fixed_weights) n.att_conv1_relu = L.ReLU(n.att_conv1) n.att_conv2 = L.Convolution(n.att_conv1_relu, kernel_size=1, stride=1, num_output=1, pad=0, weight_filler=dict(type='xavier'), param=fixed_weights) n.att_reshaped = L.Reshape(n.att_conv2,reshape_param=dict(shape=dict(dim=[-1,1,14*14]))) n.att_softmax = L.Softmax(n.att_reshaped, axis=2) n.att_map = L.Reshape(n.att_softmax,reshape_param=dict(shape=dict(dim=[-1,1,14,14]))) dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.att_feature = L.SoftAttention(n.img_feature, n.att_map, dummy) n.att_feature_resh = L.Reshape(n.att_feature, reshape_param=dict(shape=dict(dim=[-1,2048]))) # eltwise product + normalization again for VQA n.i_emb2 = L.InnerProduct(n.att_feature_resh, num_output=2048, weight_filler=dict(type='xavier'), param=fixed_weights) n.eltwise2 = L.Eltwise(n.lstm_12, n.i_emb2, eltwise_param={'operation': P.Eltwise.PROD}) n.eltwise2_sqrt = L.SignedSqrt(n.eltwise2) n.eltwise2_l2 = L.L2Normalize(n.eltwise2_sqrt) n.eltwise2_drop = L.Dropout(n.eltwise2_l2, dropout_param={'dropout_ratio': 0.3}) n.prediction = L.InnerProduct(n.eltwise2_drop, num_output=3000, weight_filler=dict(type='xavier'), param=fixed_weights) # Take GT answer or Take the logits of the VQA model and get predicted answer to embed if use_gt: n.exp_emb_ans = L.Embed(n.label, input_dim=3000, num_output=300, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) else: n.vqa_ans = L.ArgMax(n.prediction, axis=1) n.exp_emb_ans = L.Embed(n.vqa_ans, input_dim=3000, num_output=300, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) n.exp_emb_ans_tanh = L.TanH(n.exp_emb_ans) n.exp_emb_ans2 = L.InnerProduct(n.exp_emb_ans_tanh, num_output=2048, weight_filler=dict(type='xavier')) # Merge VQA answer and visual+textual feature n.exp_emb_resh = L.Reshape(n.exp_emb_ans2, reshape_param=dict(shape=dict(dim=[-1,2048,1,1]))) n.exp_emb_tiled_1 = L.Tile(n.exp_emb_resh, axis=2, tiles=14) n.exp_emb_tiled = L.Tile(n.exp_emb_tiled_1, axis=3, tiles=14) #n.exp_eltwise = L.Eltwise(n.eltwise_drop, n.exp_emb_tiled, eltwise_param={'operation': P.Eltwise.PROD}) n.eltwise_emb = L.Convolution(n.eltwise, kernel_size=1, stride=1, num_output=2048, pad=0, weight_filler=dict(type='xavier')) n.exp_eltwise = L.Eltwise(n.eltwise_emb, n.exp_emb_tiled, eltwise_param={'operation': P.Eltwise.PROD}) n.exp_eltwise_sqrt = L.SignedSqrt(n.exp_eltwise) n.exp_eltwise_l2 = L.L2Normalize(n.exp_eltwise_sqrt) n.exp_eltwise_drop = L.Dropout(n.exp_eltwise_l2, dropout_param={'dropout_ratio': 0.3}) # Attention for Explanation n.exp_att_conv1 = L.Convolution(n.exp_eltwise_drop, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.exp_att_conv1_relu = L.ReLU(n.exp_att_conv1) n.exp_att_conv2 = L.Convolution(n.exp_att_conv1_relu, kernel_size=1, stride=1, num_output=1, pad=0, weight_filler=dict(type='xavier')) n.exp_att_reshaped = L.Reshape(n.exp_att_conv2,reshape_param=dict(shape=dict(dim=[-1,1,14*14]))) n.exp_att_softmax = L.Softmax(n.exp_att_reshaped, axis=2) n.exp_att_map = L.Reshape(n.exp_att_softmax,reshape_param=dict(shape=dict(dim=[-1,1,14,14]))) exp_dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.exp_att_feature_prev = L.SoftAttention(n.img_feature, n.exp_att_map, exp_dummy) n.exp_att_feature_resh = L.Reshape(n.exp_att_feature_prev, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.exp_att_feature_embed = L.InnerProduct(n.exp_att_feature_resh, num_output=2048, weight_filler=dict(type='xavier')) n.exp_lstm12_embed = L.InnerProduct(n.lstm_12, num_output=2048, weight_filler=dict(type='xavier')) n.exp_eltwise2 = L.Eltwise(n.exp_lstm12_embed, n.exp_att_feature_embed, eltwise_param={'operation': P.Eltwise.PROD}) n.exp_att_feature = L.Eltwise(n.exp_emb_ans2, n.exp_eltwise2, eltwise_param={'operation': P.Eltwise.PROD}) n.silence_exp_att = L.Silence(n.exp_att_feature, ntop=0) return n.to_proto()
def qlstm(mode, batchsize, T, question_vocab_size): #prototxt 없이 network 생성시 사용 n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) #지정된 Python 모듈 형식 #https://stackoverflow.com/questions/41344168/what-is-a-python-layer-in-caffe #해당 클래스를 바탕으로 Layer를 생성하며 #리턴된 변수에 값을 채워넣으면 자동으로 Run된다. #여기서 만들어진 Class 내부에서 실질적인 databatch load가 이루어짐. #Glove = Global vectors for word representation #https://www.aclweb.org/anthology/D14-1162 #Pretrained 된 GloveVector를 Concat에 사용. #img_feature는 이미 Resnet512 통과후 L2를 적용한 Preprocessing이 끝난 상태의 Feature Vector. n.data, n.cont, n.img_feature, n.label, n.glove = L.Python(\ module='vqa_data_provider_layer', layer='VQADataProviderLayer', param_str=mode_str, ntop=5 ) #module = python 파일이름 #layer = layer형식이 맞춰진 python class #param_str = json으로 Data Load시 사용된 파라미터, 내부 class에 self.param_str = modestr 로 저장된다 #ntop = 각 setup , forward backward의 top 변수의 크기 #보통 textual Embed의 뜻은 => texture -> number #Embed 3000개의 Vector종류를 #300개로 compact하게 표현함 n.embed_ba = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='uniform',min=-0.08,max=0.08)) #Tanh 적용 n.embed = L.TanH(n.embed_ba) #Glove Data와 Concat concat_word_embed = [n.embed, n.glove] n.concat_embed = L.Concat(*concat_word_embed, concat_param={'axis': 2}) # T x N x 600 # LSTM1 n.lstm1 = L.LSTM(\ n.concat_embed, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops1 = L.Slice(n.lstm1, ntop=T, slice_param={'axis': 0}) for i in xrange(T - 1): n.__setattr__('slice_first' + str(i), tops1[int(i)]) n.__setattr__('silence_data_first' + str(i), L.Silence(tops1[int(i)], ntop=0)) n.lstm1_out = tops1[T - 1] n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm1_reshaped_droped = L.Dropout(n.lstm1_reshaped, dropout_param={'dropout_ratio': 0.3}) n.lstm1_droped = L.Dropout(n.lstm1, dropout_param={'dropout_ratio': 0.3}) # LSTM2 n.lstm2 = L.LSTM(\ n.lstm1_droped, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops2 = L.Slice(n.lstm2, ntop=T, slice_param={'axis': 0}) #https://www.programcreek.com/python/example/107865/caffe.NetSpec 참조. # give top2[~] the name specified by argument `slice_second` #변수 부여 기능 for i in xrange(T - 1): n.__setattr__('slice_second' + str(i), tops2[int(i)]) n.__setattr__('silence_data_second' + str(i), L.Silence(tops2[int(i)], ntop=0)) #마지막 LSTM output을 사용. n.lstm2_out = tops2[T - 1] n.lstm2_reshaped = L.Reshape(n.lstm2_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm2_reshaped_droped = L.Dropout(n.lstm2_reshaped, dropout_param={'dropout_ratio': 0.3}) concat_botom = [n.lstm1_reshaped_droped, n.lstm2_reshaped_droped] n.lstm_12 = L.Concat(*concat_botom) #lstm1의 output => 1024 reshape뒤 dropout #lstm2의 output => 1024 reshape뒤 dropout #concat n.q_emb_tanh_droped_resh = L.Reshape( n.lstm_12, reshape_param=dict(shape=dict(dim=[-1, 2048, 1, 1]))) #L.Tile 차원을 자동으로 안맞춰주므로 차원맞춤 함수. 2048,1 (tile=14, axis=1) =>2048,14 n.q_emb_tanh_droped_resh_tiled_1 = L.Tile(n.q_emb_tanh_droped_resh, axis=2, tiles=14) n.q_emb_tanh_droped_resh_tiled = L.Tile(n.q_emb_tanh_droped_resh_tiled_1, axis=3, tiles=14) n.i_emb_tanh_droped_resh = L.Reshape( n.img_feature, reshape_param=dict(shape=dict(dim=[-1, 2048, 14, 14]))) n.blcf = L.CompactBilinear(n.q_emb_tanh_droped_resh_tiled, n.i_emb_tanh_droped_resh, compact_bilinear_param=dict(num_output=16000, sum_pool=False)) n.blcf_sign_sqrt = L.SignedSqrt(n.blcf) n.blcf_sign_sqrt_l2 = L.L2Normalize(n.blcf_sign_sqrt) #논문 그림과 달리 Dropout 추가 n.blcf_droped = L.Dropout(n.blcf_sign_sqrt_l2, dropout_param={'dropout_ratio': 0.1}) # multi-channel attention n.att_conv1 = L.Convolution(n.blcf_droped, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.att_conv1_relu = L.ReLU(n.att_conv1) #논문 그림과 달리 output dim이 2 n.att_conv2 = L.Convolution(n.att_conv1_relu, kernel_size=1, stride=1, num_output=2, pad=0, weight_filler=dict(type='xavier')) n.att_reshaped = L.Reshape( n.att_conv2, reshape_param=dict(shape=dict(dim=[-1, 2, 14 * 14]))) n.att_softmax = L.Softmax(n.att_reshaped, axis=2) #softmax로 attentionmap 생성 #14x14 Softmax map이 2개 생성 n.att = L.Reshape(n.att_softmax, reshape_param=dict(shape=dict(dim=[-1, 2, 14, 14]))) #두가지 att_map을 각각 Slice att_maps = L.Slice(n.att, ntop=2, slice_param={'axis': 1}) n.att_map0 = att_maps[0] n.att_map1 = att_maps[1] dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.att_feature0 = L.SoftAttention(n.i_emb_tanh_droped_resh, n.att_map0, dummy) n.att_feature1 = L.SoftAttention(n.i_emb_tanh_droped_resh, n.att_map1, dummy) n.att_feature0_resh = L.Reshape( n.att_feature0, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.att_feature1_resh = L.Reshape( n.att_feature1, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.att_feature = L.Concat(n.att_feature0_resh, n.att_feature1_resh) #각각 ATT를 곱한값을 연산뒤 Concat한다. # merge attention and lstm with compact bilinear pooling n.att_feature_resh = L.Reshape( n.att_feature, reshape_param=dict(shape=dict(dim=[-1, 4096, 1, 1]))) #그뒤 4096으로 Reshape n.lstm_12_resh = L.Reshape( n.lstm_12, reshape_param=dict(shape=dict(dim=[-1, 2048, 1, 1]))) #논문과 달리 가로축 세로축 inputVector크기가 다름 #논문 2048 2048 #코드 4096 2048 n.bc_att_lstm = L.CompactBilinear(n.att_feature_resh, n.lstm_12_resh, compact_bilinear_param=dict( num_output=16000, sum_pool=False)) #SignedSqrt n.bc_sign_sqrt = L.SignedSqrt(n.bc_att_lstm) #L2_Normalize n.bc_sign_sqrt_l2 = L.L2Normalize(n.bc_sign_sqrt) #Dropout n.bc_dropped = L.Dropout(n.bc_sign_sqrt_l2, dropout_param={'dropout_ratio': 0.1}) n.bc_dropped_resh = L.Reshape( n.bc_dropped, reshape_param=dict(shape=dict(dim=[-1, 16000]))) #FullyConnected n.prediction = L.InnerProduct(n.bc_dropped_resh, num_output=3000, weight_filler=dict(type='xavier')) n.loss = L.SoftmaxWithLoss(n.prediction, n.label) return n.to_proto()
def mfh_baseline(mode, batchsize, T, question_vocab_size, folder): n = caffe.NetSpec() mode_str = json.dumps({'mode':mode, 'batchsize':batchsize,'folder':folder}) if mode == 'val': n.data, n.cont, n.img_feature, n.label = L.Python( \ module='vqa_data_layer', layer='VQADataProviderLayer', \ param_str=mode_str, ntop=4 ) else: n.data, n.cont, n.img_feature, n.label = L.Python(\ module='vqa_data_layer_kld', layer='VQADataProviderLayer', \ param_str=mode_str, ntop=4 ) n.embed = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='xavier')) n.embed_tanh = L.TanH(n.embed) # LSTM #n.lstm1 = L.LSTM(\ # n.embed_tanh, n.cont,\ # recurrent_param=dict(\ # num_output=config.LSTM_UNIT_NUM,\ # weight_filler=dict(type='xavier'))) #tops1 = L.Slice(n.lstm1, ntop=config.MAX_WORDS_IN_QUESTION, slice_param={'axis':0}) #for i in xrange(config.MAX_WORDS_IN_QUESTION-1): # n.__setattr__('slice_first'+str(i), tops1[int(i)]) # n.__setattr__('silence_data_first'+str(i), L.Silence(tops1[int(i)],ntop=0)) #n.lstm1_out = tops1[config.MAX_WORDS_IN_QUESTION-1] #n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ # reshape_param=dict(\ # shape=dict(dim=[-1,1024]))) #n.q_feat = L.Dropout(n.lstm1_reshaped,dropout_param={'dropout_ratio':config.LSTM_DROPOUT_RATIO}) n.lstm1 = L.LSTM(\ n.embed, n.cont,\ recurrent_param=dict(\ num_output=config.LSTM_UNIT_NUM,\ weight_filler=dict(type='xavier'))) tops1 = L.Slice(n.lstm1, ntop=config.MAX_WORDS_IN_QUESTION, slice_param={'axis':0}) for i in xrange(config.MAX_WORDS_IN_QUESTION-1): n.__setattr__('slice_first'+str(i), tops1[int(i)]) n.__setattr__('silence_data_first'+str(i), L.Silence(tops1[int(i)],ntop=0)) n.lstm1_out = tops1[config.MAX_WORDS_IN_QUESTION-1] n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm1_reshaped_droped = L.Dropout(n.lstm1_reshaped,dropout_param={'dropout_ratio':config.LSTM_DROPOUT_RATIO}) n.lstm1_droped = L.Dropout(n.lstm1,dropout_param={'dropout_ratio':config.LSTM_DROPOUT_RATIO}) # LSTM2 n.lstm2 = L.LSTM(\ n.lstm1_droped, n.cont,\ recurrent_param=dict(\ num_output=config.LSTM_UNIT_NUM, weight_filler=dict(type='xavier'))) tops2 = L.Slice(n.lstm2, ntop=config.MAX_WORDS_IN_QUESTION, slice_param={'axis':0}) for i in xrange(config.MAX_WORDS_IN_QUESTION-1): n.__setattr__('slice_second'+str(i), tops2[int(i)]) n.__setattr__('silence_data_second'+str(i), L.Silence(tops2[int(i)],ntop=0)) n.lstm2_out = tops2[config.MAX_WORDS_IN_QUESTION-1] n.lstm2_reshaped = L.Reshape(n.lstm2_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm2_reshaped_droped = L.Dropout(n.lstm2_reshaped,dropout_param={'dropout_ratio':config.LSTM_DROPOUT_RATIO}) concat_botom = [n.lstm1_reshaped_droped, n.lstm2_reshaped_droped] n.q_feat = L.Concat(*concat_botom) ''' Coarse Image-Question MFH fusion ''' n.mfb_q_o2_proj = L.InnerProduct(n.q_feat, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_i_o2_proj = L.InnerProduct(n.img_feature, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_iq_o2_eltwise = L.Eltwise(n.mfb_q_o2_proj, n.mfb_i_o2_proj, eltwise_param=dict(operation=0)) n.mfb_iq_o2_drop = L.Dropout(n.mfb_iq_o2_eltwise, dropout_param={'dropout_ratio':config.MFB_DROPOUT_RATIO}) n.mfb_iq_o2_resh = L.Reshape(n.mfb_iq_o2_drop, reshape_param=dict(shape=dict(dim=[-1,1,config.MFB_OUT_DIM,config.MFB_FACTOR_NUM]))) n.mfb_iq_o2_sumpool = L.Pooling(n.mfb_iq_o2_resh, pool=P.Pooling.SUM, \ pooling_param=dict(kernel_w=config.MFB_FACTOR_NUM, kernel_h=1)) n.mfb_o2_out = L.Reshape(n.mfb_iq_o2_sumpool,\ reshape_param=dict(shape=dict(dim=[-1,config.MFB_OUT_DIM]))) n.mfb_o2_sign_sqrt = L.SignedSqrt(n.mfb_o2_out) n.mfb_o2_l2 = L.L2Normalize(n.mfb_o2_sign_sqrt) n.mfb_q_o3_proj = L.InnerProduct(n.q_feat, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_i_o3_proj = L.InnerProduct(n.img_feature, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_iq_o3_eltwise = L.Eltwise(n.mfb_q_o3_proj, n.mfb_i_o3_proj,n.mfb_iq_o2_drop, eltwise_param=dict(operation=0)) n.mfb_iq_o3_drop = L.Dropout(n.mfb_iq_o3_eltwise, dropout_param={'dropout_ratio':config.MFB_DROPOUT_RATIO}) n.mfb_iq_o3_resh = L.Reshape(n.mfb_iq_o3_drop, reshape_param=dict(shape=dict(dim=[-1,1,config.MFB_OUT_DIM,config.MFB_FACTOR_NUM]))) n.mfb_iq_o3_sumpool = L.Pooling(n.mfb_iq_o3_resh, pool=P.Pooling.SUM, \ pooling_param=dict(kernel_w=config.MFB_FACTOR_NUM, kernel_h=1)) n.mfb_o3_out = L.Reshape(n.mfb_iq_o3_sumpool,\ reshape_param=dict(shape=dict(dim=[-1,config.MFB_OUT_DIM]))) n.mfb_o3_sign_sqrt = L.SignedSqrt(n.mfb_o3_out) n.mfb_o3_l2 = L.L2Normalize(n.mfb_o3_sign_sqrt) n.mfb_o23_l2 = L.Concat(n.mfb_o2_l2,n.mfb_o3_l2) n.prediction = L.InnerProduct(n.mfb_o23_l2, num_output=config.NUM_OUTPUT_UNITS, weight_filler=dict(type='xavier')) if mode == 'val': n.loss = L.SoftmaxWithLoss(n.prediction, n.label) else: n.loss = L.SoftmaxKLDLoss(n.prediction, n.label) return n.to_proto()
def creatNet(data_path, batch_size, DB_NAME_SAMPLES, DB_NAME_LABELS, DB_NAME_CLIP_MARKERS, DB_NAME_LOGICAL_LABELS, DB_NAME_SAMPLE_INDEX, image_height, image_width, channel_num, image_num_per_sequence): sequence_num_per_batch = batch_size / image_num_per_sequence # Current net structure # samples labels clip_markers # Convolution Reshape Reshape # ReLu # Pooling # InnerProduct # ReLu # DropOut # Reshape # Lstm # DEFINE THE NETWORK ARCHETECTURE net = caffe.NetSpec() # DATA LAYER # SAMPLE AND LABEL net.samples = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=DB_NAME_SAMPLES, transform_param = {'scale': 0.00390625}) net.labels = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=DB_NAME_LABELS) net.clip_markers = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=DB_NAME_CLIP_MARKERS) net.sample_indexes = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=DB_NAME_SAMPLE_INDEX) # Adding layers 20160809 # Batch Normalization net.batch_normalization_1 = L.BatchNorm(net.samples) net.scale_1 = L.Scale(net.batch_normalization_1, scale_param = {'bias_term': True}) net.relu_3 = L.ReLU(net.scale_1) # Adding layers 2016-08-08 # Convolution Later # Bottom:samples net.convolution_1 = L.Convolution(net.relu_3, param=[{'lr_mult': 1, 'decay_mult': 1}, {'lr_mult': 2, 'decay_mult': 0}], convolution_param={'num_output': 96, 'kernel_size': 7, 'stride': 2, 'weight_filler': {'type': 'gaussian', 'std': 0.01}, 'bias_filler': {'type': 'constant', 'value': 0.1}}) # Pooling net.pooling_1 = L.Pooling(net.convolution_1, pooling_param={'pool': P.Pooling.MAX, 'kernel_size': 3, 'stride': 2}) net.batch_normalization_2 = L.BatchNorm(net.pooling_1) net.scale_2 = L.Scale(net.batch_normalization_2, scale_param = {'bias_term': True}) # ReLu net.relu_1 = L.ReLU(net.scale_2) # Inner Product net.inner_product_1 = L.InnerProduct(net.relu_1, param=[{'lr_mult': 1, 'decay_mult': 1}, {'lr_mult': 2, 'decay_mult': 0}], inner_product_param={'num_output': 4096, 'weight_filler': {'type': 'gaussian', 'std': 0.01}, 'bias_filler': {'type': 'constant', 'value': 0.1}}) # ReLu net.relu_2 = L.ReLU(net.inner_product_1) net.dropout_1 = L.Dropout(net.relu_2, dropout_param={'dropout_ratio': 0.9}) net.reshape_sample_1 = L.Reshape(net.dropout_1, reshape_param={ 'shape': {'dim': [image_num_per_sequence, sequence_num_per_batch, 4096] } }) # Reshaple lable net.reshape_label_1 = L.Reshape(net.labels, reshape_param={ 'shape': {'dim': [image_num_per_sequence, sequence_num_per_batch]}}) # Reshape clip markers net.reshape_clip_markers_1 = L.Reshape(net.clip_markers, reshape_param={ 'shape': {'dim': [image_num_per_sequence, sequence_num_per_batch]}}) # LSTM network net.lstm_1 = L.LSTM(net.reshape_sample_1, net.reshape_clip_markers_1, recurrent_param={'num_output': 256, 'weight_filler': {'type': 'uniform', 'min':-0.01, 'max': 0.01}, 'bias_filler': {'type': 'constant', 'value': 0 }}) net.dropout_2 = L.Dropout(net.lstm_1, dropout_param={'dropout_ratio': 0.5}) # INNERPRODUCT LAYER net.inner_product_2 = L.InnerProduct(net.lstm_1, param=[{'lr_mult': 1, 'decay_mult': 1}, {'lr_mult': 2, 'decay_mult': 0}], inner_product_param={'num_output': 10, 'weight_filler': {'type': 'gaussian', 'std': 0.01}, 'bias_filler': {'type': 'constant', 'value': 0}, 'axis': -2}) net.slice_ip12_1 = L.Slice() # # net.inner_product_3 = L.InnerProduct(net.inner_product_2, # inner_product_param={'num_output': 1, # 'weight_filler': {'type': 'gaussian', 'std': 0.01}, # 'bias_filler': {'type': 'constant', 'value': 0}, # 'axis': 0}) # # # # # [net.slice_label_1, net.slice_label_2] = L.Slice(net.reshape_label_1, # ntop = 2, # slice_param = {'axis': 0, # 'slice_point': 1}) # # # net.reshape_slice_label_1 = L.Reshape(net.slice_label_1, # reshape_param={ 'shape': {'dim': [1, sequence_num_per_batch]}}) # LOSS LAYER #net.loss = L.SoftmaxWithLoss(net.inner_product_3, # net.reshape_slice_label_1) # Accuracy layer #net.accuracy = L.Accuracy(net.inner_product_3, # net.reshape_slice_label_1) # RESHAPE LAYER # WHY RESHAPE? # Data from database looks like: # sample (sequence = 1, time = 1) # sample (sequence = 1, time = 2) # . # . # . # sample (sequence = 1, time = T) # sample (sequence = 2, time = 1) # # Thus for feeding the LSTM, the data should like : # sample (s = 1, t = 1), sample (s = 2, t = 1), sample (s = 3 ,t = 1), ... sample (s = N, t = 1) # sample (s = 1, t = 2), sample (s = 2, t = 2), sample (s = 3, t = 2), ... sample (s = N, t = 2) # . # . # sample (s = 1, t = T), sample (s = 2, t = T), sample (s = 3, t = T), ... sample (s = N, t = T) . # RESHAPE SHOULE BE TWICE # WHY? # Because the caffe build-in reshape is line-prioritize filled # THE 1st RESHAPE # SAMPLES RESHAPE LAYER # input shape (raw sample shape): a blob of (T * N) * h * w # desired output shape: a blob of return net
def dis_net(mode, batchsize, T, exp_T, question_vocab_size, exp_vocab_size): n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) n.data, n.cont, n.img_feature, n.label, n.exp, n.exp_out, n.exp_cont_1, n.exp_cont_2, n.dis_label = L.Python( module='vqa_data_provider_layer', layer='VQADataProviderLayer', param_str=mode_str, ntop=9) n.vqa_exp_emb = L.Embed( n.exp, input_dim=exp_vocab_size, num_output=exp_vocab_size, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) #n.vqa_exp=batchxseqx exp_vocab_size n.vqa_exp = L.TanH(n.vqa_exp_emb) n.vqa_exp_reshape = L.Reshape( n.vqa_exp, reshape_param=dict(shape=dict(dim=[-1, exp_vocab_size]))) n.exp_embed_ba = L.InnerProduct(n.vqa_exp_reshape, num_output=300, weight_filler=dict(type='xavier')) n.exp_embed_ba_reshape = L.Reshape( n.exp_embed_ba, reshape_param=dict(shape=dict(dim=[-1, batchsize, 300]))) n.exp_embed = L.TanH(n.exp_embed_ba_reshape) # Embed VQA GT answer during training n.vqa_ans = L.Embed( n.label, input_dim=3000, num_output=3000, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) #n.vqa_ans=batchxseqx3000 n.exp_emb_ans = L.InnerProduct(n.vqa_ans, num_output=300, weight_filler=dict(type='xavier')) n.exp_emb_ans_tanh = L.TanH(n.exp_emb_ans) n.exp_emb_ans2 = L.InnerProduct(n.exp_emb_ans_tanh, num_output=2048, weight_filler=dict(type='xavier')) # Exp LSTM1 n.exp_lstm1 = L.LSTM(n.exp_embed, n.exp_cont_1, recurrent_param=dict(num_output=1024, weight_filler=dict( type='uniform', min=-0.08, max=0.08), bias_filler=dict(type='constant', value=0)), param=fixed_weights_lstm) exp_tops1 = L.Slice(n.exp_lstm1, ntop=exp_T, slice_param={'axis': 0}) for i in range(T - 1): n.__setattr__('slice_first' + str(i), exp_tops1[int(i)]) n.__setattr__('silence_data_first' + str(i), L.Silence(exp_tops1[int(i)], ntop=0)) n.exp_lstm1_out = exp_tops1[T - 1] n.exp_lstm1_reshaped = L.Reshape( n.exp_lstm1_out, reshape_param=dict(shape=dict(dim=[-1, 1024]))) n.exp_lstm1_reshaped_droped = L.Dropout( n.exp_lstm1_reshaped, dropout_param={'dropout_ratio': 0.3}) n.exp_lstm1_droped = L.Dropout(n.exp_lstm1, dropout_param={'dropout_ratio': 0.3}) # Exp LSTM2 n.exp_lstm2 = L.LSTM(n.exp_lstm1_droped, n.exp_cont_2, recurrent_param=dict(num_output=1024, weight_filler=dict( type='uniform', min=-0.08, max=0.08), bias_filler=dict(type='constant', value=0)), param=fixed_weights_lstm) exp_tops2 = L.Slice(n.exp_lstm2, ntop=exp_T, slice_param={'axis': 0}) for i in range(T - 1): n.__setattr__('slice_second' + str(i), exp_tops2[int(i)]) n.__setattr__('silence_data_second' + str(i), L.Silence(exp_tops2[int(i)], ntop=0)) n.exp_lstm2_out = exp_tops2[T - 1] n.exp_lstm2_reshaped = L.Reshape( n.exp_lstm2_out, reshape_param=dict(shape=dict(dim=[-1, 1024]))) n.exp_lstm2_reshaped_droped = L.Dropout( n.exp_lstm2_reshaped, dropout_param={'dropout_ratio': 0.3}) concat_botom = [n.exp_lstm1_reshaped_droped, n.exp_lstm2_reshaped_droped] n.exp_lstm_12 = L.Concat(*concat_botom) #To concat ans and exp concat_ans_exp = [n.exp_emb_ans2, n.exp_lstm_12] n.concat_ans_exp_layer = L.Concat(*concat_ans_exp) n.concat_ans_exp_layer_dis = L.InnerProduct( n.concat_ans_exp_layer, num_output=1, weight_filler=dict(type='xavier')) n.discr_loss = L.SoftmaxWithLoss(n.concat_ans_exp_layer_dis, n.dis_label, loss_param=dict(ignore_label=-1)) return n.to_proto()
def pj_x(mode, batchsize, exp_T, exp_vocab_size): n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) n.img_feature, n.label, n.exp, n.exp_out, n.exp_cont_1, n.exp_cont_2 = \ L.Python(module='activity_data_provider_layer', layer='ActivityDataProviderLayer', param_str=mode_str, ntop=6) # Attention n.att_conv1 = L.Convolution(n.img_feature, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.att_conv1_relu = L.ReLU(n.att_conv1) n.att_conv2 = L.Convolution(n.att_conv1_relu, kernel_size=1, stride=1, num_output=1, pad=0, weight_filler=dict(type='xavier')) n.att_reshaped = L.Reshape( n.att_conv2, reshape_param=dict(shape=dict(dim=[-1, 1, 14 * 14]))) n.att_softmax = L.Softmax(n.att_reshaped, axis=2) n.att_map = L.Reshape(n.att_softmax, reshape_param=dict(shape=dict(dim=[-1, 1, 14, 14]))) dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.att_feature = L.SoftAttention(n.img_feature, n.att_map, dummy) n.att_feature_resh = L.Reshape( n.att_feature, reshape_param=dict(shape=dict(dim=[-1, 2048]))) # Prediction n.prediction = L.InnerProduct(n.att_feature_resh, num_output=config.NUM_OUTPUT_UNITS, weight_filler=dict(type='xavier')) n.loss = L.SoftmaxWithLoss(n.prediction, n.label) n.accuracy = L.Accuracy(n.prediction, n.label) # Embed Activity GT answer during training n.exp_emb_ans = L.Embed(n.label, input_dim=config.NUM_OUTPUT_UNITS, num_output=300, \ weight_filler=dict(type='uniform', min=-0.08, max=0.08)) n.exp_emb_ans_tanh = L.TanH(n.exp_emb_ans) n.exp_emb_ans2 = L.InnerProduct(n.exp_emb_ans_tanh, num_output=2048, weight_filler=dict(type='xavier')) # merge activity answer and visual feature n.exp_emb_resh = L.Reshape( n.exp_emb_ans2, reshape_param=dict(shape=dict(dim=[-1, 2048, 1, 1]))) n.exp_emb_tiled_1 = L.Tile(n.exp_emb_resh, axis=2, tiles=14) n.exp_emb_tiled = L.Tile(n.exp_emb_tiled_1, axis=3, tiles=14) n.img_embed = L.Convolution(n.img_feature, kernel_size=1, stride=1, num_output=2048, pad=0, weight_filler=dict(type='xavier')) n.exp_eltwise = L.Eltwise(n.img_embed, n.exp_emb_tiled, eltwise_param={'operation': P.Eltwise.PROD}) n.exp_eltwise_sqrt = L.SignedSqrt(n.exp_eltwise) n.exp_eltwise_l2 = L.L2Normalize(n.exp_eltwise_sqrt) n.exp_eltwise_drop = L.Dropout(n.exp_eltwise_l2, dropout_param={'dropout_ratio': 0.3}) # Attention for Explanation n.exp_att_conv1 = L.Convolution(n.exp_eltwise_drop, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.exp_att_conv1_relu = L.ReLU(n.exp_att_conv1) n.exp_att_conv2 = L.Convolution(n.exp_att_conv1_relu, kernel_size=1, stride=1, num_output=1, pad=0, weight_filler=dict(type='xavier')) n.exp_att_reshaped = L.Reshape( n.exp_att_conv2, reshape_param=dict(shape=dict(dim=[-1, 1, 14 * 14]))) n.exp_att_softmax = L.Softmax(n.exp_att_reshaped, axis=2) n.exp_att_map = L.Reshape( n.exp_att_softmax, reshape_param=dict(shape=dict(dim=[-1, 1, 14, 14]))) exp_dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.exp_att_feature_prev = L.SoftAttention(n.img_feature, n.exp_att_map, exp_dummy) n.exp_att_feature_resh = L.Reshape( n.exp_att_feature_prev, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.exp_att_feature_embed = L.InnerProduct(n.exp_att_feature_resh, num_output=2048, weight_filler=dict(type='xavier')) n.exp_att_feature = L.Eltwise(n.exp_emb_ans2, n.exp_att_feature_embed, eltwise_param={'operation': P.Eltwise.PROD}) # Embed explanation n.exp_embed_ba = L.Embed(n.exp, input_dim=exp_vocab_size, num_output=300, \ weight_filler=dict(type='uniform', min=-0.08, max=0.08)) n.exp_embed = L.TanH(n.exp_embed_ba) # LSTM1 for Explanation n.exp_lstm1 = L.LSTM(\ n.exp_embed, n.exp_cont_1,\ recurrent_param=dict(\ num_output=2048,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) n.exp_lstm1_dropped = L.Dropout(n.exp_lstm1, dropout_param={'dropout_ratio': 0.3}) # merge with LSTM1 for explanation n.exp_att_resh = L.Reshape( n.exp_att_feature, reshape_param=dict(shape=dict(dim=[1, -1, 2048]))) n.exp_att_tiled = L.Tile(n.exp_att_resh, axis=0, tiles=exp_T) n.exp_eltwise_all = L.Eltwise(n.exp_lstm1_dropped, n.exp_att_tiled, eltwise_param={'operation': P.Eltwise.PROD}) n.exp_eltwise_all_l2 = L.L2Normalize(n.exp_eltwise_all) n.exp_eltwise_all_drop = L.Dropout(n.exp_eltwise_all_l2, dropout_param={'dropout_ratio': 0.3}) # LSTM2 for Explanation n.exp_lstm2 = L.LSTM(\ n.exp_eltwise_all_drop, n.exp_cont_2,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) n.exp_lstm2_dropped = L.Dropout(n.exp_lstm2, dropout_param={'dropout_ratio': 0.3}) n.exp_prediction = L.InnerProduct(n.exp_lstm2_dropped, num_output=exp_vocab_size, weight_filler=dict(type='xavier'), axis=2) n.exp_loss = L.SoftmaxWithLoss(n.exp_prediction, n.exp_out, loss_param=dict(ignore_label=-1), softmax_param=dict(axis=2)) n.exp_accuracy = L.Accuracy(n.exp_prediction, n.exp_out, axis=2, ignore_label=-1) return n.to_proto()
def unpack_item(self, layer, previous_image_size, layer_number, bottom, label=None): if layer.terminate == 1: # Softmax Accuracy/Loss # loss = cl.SoftmaxWithLoss(bottom, label) bottom = cl.InnerProduct(bottom, num_output=self.hp.NUM_CLASSES, weight_filler=dict(type='xavier')) return bottom if layer.layer_type == 'conv': out_depth = layer.filter_depth kernel_size = layer.filter_size stride = layer.stride pad = self.get_pad(kernel_size) bottom = cl.Convolution(bottom, kernel_size=kernel_size, num_output=out_depth, stride=stride, pad=pad, weight_filler=dict(type='xavier')) if self.ssp.batch_norm: bottom = self.add_batchnorm(bottom) return self.add_activate(bottom) if layer.layer_type == 'nin': out_depth = layer.filter_depth bottom = cl.Convolution(bottom, kernel_size=1, num_output=out_depth, weight_filler=dict(type='xavier')) bottom = self.add_activate(bottom) bottom = cl.Convolution(bottom, kernel_size=1, num_output=out_depth, weight_filler=dict(type='xavier')) bottom = self.add_activate(bottom) return bottom if layer.layer_type == 'gap': out_depth = self.hp.NUM_CLASSES bottom = cl.Convolution(bottom, kernel_size=1, num_output=out_depth, weight_filler=dict(type='xavier')) bottom = self.add_activate(bottom) bottom = cl.Pooling(bottom, kernel_size=previous_image_size, pool=P.Pooling.AVE) return bottom if layer.layer_type == 'fc': num_output = layer.fc_size bottom = cl.InnerProduct(bottom, num_output=num_output, weight_filler=dict(type='xavier')) bottom = self.add_activate(bottom) return bottom if layer.layer_type == 'dropout': dropout_ratio = 0.5 * float(layer.filter_depth) / layer.fc_size return cl.Dropout(bottom, dropout_ratio=dropout_ratio) if layer.layer_type == 'pool': kernel_size = layer.filter_size stride = layer.stride if self.ssp.batch_norm: bottom = self.add_batchnorm(bottom) return cl.Pooling(bottom, kernel_size=kernel_size, stride=stride, pool=P.Pooling.MAX) if layer.layer_type == 'lstm': bottom = cl.LSTM(bottom, weight_filler=dict(type='xavier'), num_output=layer.seq_lengh) return bottom if layer.layer_type == 'flatten': bottom = cl.Flatten(bottom) return bottom
def net(split, vocab_size, opts): n = caffe.NetSpec() param_str = json.dumps({'split': split, 'batchsize': cfg.BATCHSIZE}) n.qvec, n.cvec, n.img_feat, n.spt_feat, n.query_label, n.query_label_mask, n.query_bbox_targets, \ n.query_bbox_inside_weights, n.query_bbox_outside_weights = L.Python( \ name='data', module='networks.data_layer', layer='DataProviderLayer', param_str=param_str, ntop=9 ) n.embed_ba = L.Embed(n.qvec, input_dim=vocab_size, num_output=cfg.WORD_EMB_SIZE, \ weight_filler=dict(type='xavier')) n.embed = L.TanH(n.embed_ba) word_emb = n.embed # LSTM1 n.lstm1 = L.LSTM(\ word_emb, n.cvec,\ recurrent_param=dict(\ num_output=cfg.RNN_DIM,\ weight_filler=dict(type='xavier'))) tops1 = L.Slice(n.lstm1, ntop=cfg.QUERY_MAXLEN, slice_param={'axis': 0}) for i in xrange(cfg.QUERY_MAXLEN - 1): n.__setattr__('slice_first' + str(i), tops1[int(i)]) n.__setattr__('silence_data_first' + str(i), L.Silence(tops1[int(i)], ntop=0)) n.lstm1_out = tops1[cfg.QUERY_MAXLEN - 1] n.lstm1_reshaped = L.Reshape( n.lstm1_out, reshape_param=dict(shape=dict(dim=[-1, cfg.RNN_DIM]))) n.lstm1_droped = L.Dropout( n.lstm1_reshaped, dropout_param={'dropout_ratio': cfg.DROPOUT_RATIO}) n.lstm_l2norm = L.L2Normalize(n.lstm1_droped) n.q_emb = L.Reshape(n.lstm_l2norm, reshape_param=dict(shape=dict(dim=[0, -1]))) q_layer = n.q_emb # (N, 1024) v_layer = proc_img(n, n.img_feat, n.spt_feat) #out: (N, 100, 2053) out_layer = concat(n, q_layer, v_layer) # predict score n.query_score_fc = L.InnerProduct(out_layer, num_output=1, weight_filler=dict(type='xavier')) n.query_score_pred = L.Reshape( n.query_score_fc, reshape_param=dict(shape=dict(dim=[-1, cfg.RPN_TOPN]))) if cfg.USE_KLD: n.loss_query_score = L.SoftmaxKLDLoss(n.query_score_pred, n.query_label, n.query_label_mask, propagate_down=[1, 0, 0], loss_weight=1.0) else: n.loss_query_score = L.SoftmaxWithLoss(n.query_score_pred, n.query_label, n.query_label_mask, propagate_down=[1, 0, 0], loss_weight=1.0) # predict bbox n.query_bbox_pred = L.InnerProduct(out_layer, num_output=4, weight_filler=dict(type='xavier')) if cfg.USE_REG: n.loss_query_bbox = L.SmoothL1Loss( n.query_bbox_pred, n.query_bbox_targets, \ n.query_bbox_inside_weights, n.query_bbox_outside_weights, loss_weight=1.0) else: n.__setattr__('silence_query_bbox_pred', L.Silence(n.query_bbox_pred, ntop=0)) n.__setattr__('silence_query_bbox_targets', L.Silence(n.query_bbox_targets, ntop=0)) n.__setattr__('silence_query_bbox_inside_weights', L.Silence(n.query_bbox_inside_weights, ntop=0)) n.__setattr__('silence_query_bbox_outside_weights', L.Silence(n.query_bbox_outside_weights, ntop=0)) return n.to_proto()
def qlstm(mode, batchsize, max_words_in_question, question_vocab_size): n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) n.data, n.cont, n.img_feature, n.label, n.glove = L.Python( \ module='vqa_data_provider_layer', layer='VQADataProviderLayer', \ param_str=mode_str, ntop=5 ) n.embed_ba = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='uniform',min=-0.08,max=0.08)) # T x N -> T x N x 300 n.embed = L.TanH(n.embed_ba) concat_word_embed = [n.embed, n.glove] n.concat_embed = L.Concat(*concat_word_embed, concat_param={'axis': 2}) # T x N x 600 # LSTM1 n.lstm1 = L.LSTM(\ n.concat_embed, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops1 = L.Slice(n.lstm1, ntop=max_words_in_question, slice_param={'axis': 0}) for i in xrange(max_words_in_question - 1): n.__setattr__('slice_first' + str(i), tops1[int(i)]) n.__setattr__('silence_data_first' + str(i), L.Silence(tops1[int(i)], ntop=0)) n.lstm1_out = tops1[max_words_in_question - 1] n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm1_reshaped_drop = L.Dropout(n.lstm1_reshaped, dropout_param={'dropout_ratio': 0.3}) n.lstm1_drop = L.Dropout(n.lstm1, dropout_param={'dropout_ratio': 0.3}) # LSTM2 n.lstm2 = L.LSTM(\ n.lstm1_drop, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops2 = L.Slice(n.lstm2, ntop=max_words_in_question, slice_param={'axis': 0}) for i in xrange(max_words_in_question - 1): n.__setattr__('slice_second' + str(i), tops2[int(i)]) n.__setattr__('silence_data_second' + str(i), L.Silence(tops2[int(i)], ntop=0)) n.lstm2_out = tops2[max_words_in_question - 1] n.lstm2_reshaped = L.Reshape(n.lstm2_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm2_reshaped_drop = L.Dropout(n.lstm2_reshaped, dropout_param={'dropout_ratio': 0.3}) concat_lstms = [n.lstm1_reshaped_drop, n.lstm2_reshaped_drop] n.lstm_12 = L.Concat(*concat_lstms) n.q_emb_tanh_droped_resh = L.Reshape(n.lstm_12, \ reshape_param=dict(shape=dict(dim=[-1,2048,1,1]))) n.i_emb_tanh_droped_resh = L.Reshape(n.img_feature, \ reshape_param=dict(shape=dict(dim=[-1,2048,1,1]))) n.blcf = L.CompactBilinear(n.q_emb_tanh_droped_resh, n.i_emb_tanh_droped_resh, \ compact_bilinear_param=dict(num_output=16000,sum_pool=False)) n.blcf_sign_sqrt = L.SignedSqrt(n.blcf) n.blcf_sign_sqrt_l2 = L.L2Normalize(n.blcf_sign_sqrt) n.blcf_droped = L.Dropout(n.blcf_sign_sqrt_l2, dropout_param={'dropout_ratio': 0.1}) n.blcf_droped_resh = L.Reshape( n.blcf_droped, reshape_param=dict(shape=dict(dim=[-1, 16000]))) n.prediction = L.InnerProduct(n.blcf_droped_resh, num_output=config.NUM_OUTPUT_UNITS, \ weight_filler=dict(type='xavier')) n.loss = L.SoftmaxWithLoss(n.prediction, n.label) return n.to_proto()
def cnn_lstm(**args): n_rows = args.get('n_rows', 28) n_cols = args.get('n_cols', 28) kernel_size = args.get('kernel_size', 5) n_output = args.get('n_output', n_cols) batch_size = args.get('batch_size', 300) recur_steps = args.get('recur_steps', 4) n_classes = args.get('n_classes', 10) frame_shape = args.get('frame_shape', (1, n_rows / recur_steps, n_cols)) n = caffe.NetSpec() weight_param = dict(lr_mult=1, decay_mult=1) bias_param = dict(lr_mult=1, decay_mult=0) param = [weight_param, bias_param] N = batch_size T = recur_steps input_dim = list(frame_shape) input_dim.insert(0, N * T) n.data, n.clip, n.label = L.Input( shape=[dict(dim=input_dim), dict(dim=[T, N]), dict(dim=[N, 1])], ntop=3) n.conv1 = L.Convolution(n.data, kernel_size=[1, kernel_size], stride_h=1, stride_w=1, num_output=n_output, pad=0, param=param, weight_filler=dict(type='msra'), bias_filler=dict(type='constant')) n.relu1 = L.ReLU(n.conv1, in_place=True) n.conv2 = L.Convolution(n.relu1, kernel_size=[1, kernel_size], stride_h=1, stride_w=1, num_output=n_output, pad=0, param=param, weight_filler=dict(type='msra'), bias_filler=dict(type='constant')) n.relu2 = L.ReLU(n.conv2, in_place=True) n.pool1 = L.Pooling(n.relu2, kernel_h=1, kernel_w=2, stride_h=1, stride_w=2, pool=P.Pooling.MAX) n.fc1 = L.InnerProduct(n.pool1, num_output=n_output, param=param, weight_filler=dict(type='msra'), bias_filler=dict(type='constant')) n.fc1_relu = L.ReLU(n.fc1, in_place=True) n.fc1_reshape = L.Reshape(n.fc1_relu, shape=dict(dim=[T, N, n_output])) n.lstm = L.LSTM(n.fc1_reshape, n.clip, recurrent_param=dict(num_output=n_output)) n.lstm_last_step = L.Slice(n.lstm, slice_param=dict(axis=0, slice_point=T - 1), ntop=2)[-1] n.lstm_reshape = L.Reshape(n.lstm_last_step, shape=dict(dim=[N, n_output])) n.attrs = L.InnerProduct(n.lstm_reshape, num_output=n_classes, param=param, weight_filler=dict(type='msra'), bias_filler=dict(type='constant')) n.loss = L.SoftmaxWithLoss(n.attrs, n.label) n.class_prob = L.Softmax(n.attrs, in_place=False) return n.to_proto()
def mfb_coatt(mode, batchsize, T, question_vocab_size, folder): n = caffe.NetSpec() mode_str = json.dumps({'mode':mode, 'batchsize':batchsize,'folder':folder}) if mode == 'val': n.data, n.cont, n.img_feature, n.label, n.glove = L.Python( \ module='vqa_data_layer_hdf5', layer='VQADataProviderLayer', \ param_str=mode_str, ntop=5 ) else: n.data, n.cont, n.img_feature, n.label, n.glove = L.Python(\ module='vqa_data_layer_kld_hdf5', layer='VQADataProviderLayer', \ param_str=mode_str, ntop=5 ) n.embed = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='xavier')) n.embed_tanh = L.TanH(n.embed) concat_word_embed = [n.embed_tanh, n.glove] n.concat_embed = L.Concat(*concat_word_embed, concat_param={'axis': 2}) # T x N x 600 # LSTM n.lstm1 = L.LSTM(\ n.concat_embed, n.cont,\ recurrent_param=dict(\ num_output=config.LSTM_UNIT_NUM,\ weight_filler=dict(type='xavier'))) n.lstm1_droped = L.Dropout(n.lstm1,dropout_param={'dropout_ratio':config.LSTM_DROPOUT_RATIO}) n.lstm1_resh = L.Permute(n.lstm1_droped, permute_param=dict(order=[1,2,0])) n.lstm1_resh2 = L.Reshape(n.lstm1_resh, \ reshape_param=dict(shape=dict(dim=[0,0,0,1]))) ''' Question Attention ''' n.qatt_conv1 = L.Convolution(n.lstm1_resh2, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.qatt_relu = L.ReLU(n.qatt_conv1) n.qatt_conv2 = L.Convolution(n.qatt_relu, kernel_size=1, stride=1, num_output=config.NUM_QUESTION_GLIMPSE, pad=0, weight_filler=dict(type='xavier')) n.qatt_reshape = L.Reshape(n.qatt_conv2, reshape_param=dict(shape=dict(dim=[-1,config.NUM_QUESTION_GLIMPSE,config.MAX_WORDS_IN_QUESTION,1]))) # N*NUM_QUESTION_GLIMPSE*15 n.qatt_softmax = L.Softmax(n.qatt_reshape, axis=2) qatt_maps = L.Slice(n.qatt_softmax,ntop=config.NUM_QUESTION_GLIMPSE,slice_param={'axis':1}) dummy_lstm = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) qatt_feature_list = [] for i in xrange(config.NUM_QUESTION_GLIMPSE): if config.NUM_QUESTION_GLIMPSE == 1: n.__setattr__('qatt_feat%d'%i, L.SoftAttention(n.lstm1_resh2, qatt_maps, dummy_lstm)) else: n.__setattr__('qatt_feat%d'%i, L.SoftAttention(n.lstm1_resh2, qatt_maps[i], dummy_lstm)) qatt_feature_list.append(n.__getattr__('qatt_feat%d'%i)) n.qatt_feat_concat = L.Concat(*qatt_feature_list) ''' Image Attention with MFB ''' n.q_feat_resh = L.Reshape(n.qatt_feat_concat,reshape_param=dict(shape=dict(dim=[0,-1,1,1]))) n.i_feat_resh = L.Reshape(n.img_feature,reshape_param=dict(shape=dict(dim=[0,-1,config.IMG_FEAT_WIDTH,config.IMG_FEAT_WIDTH]))) n.iatt_q_proj = L.InnerProduct(n.q_feat_resh, num_output = config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.iatt_q_resh = L.Reshape(n.iatt_q_proj, reshape_param=dict(shape=dict(dim=[-1,config.JOINT_EMB_SIZE,1,1]))) n.iatt_q_tile1 = L.Tile(n.iatt_q_resh, axis=2, tiles=config.IMG_FEAT_WIDTH) n.iatt_q_tile2 = L.Tile(n.iatt_q_tile1, axis=3, tiles=config.IMG_FEAT_WIDTH) n.iatt_i_conv = L.Convolution(n.i_feat_resh, kernel_size=1, stride=1, num_output=config.JOINT_EMB_SIZE, pad=0, weight_filler=dict(type='xavier')) n.iatt_i_resh1 = L.Reshape(n.iatt_i_conv, reshape_param=dict(shape=dict(dim=[-1,config.JOINT_EMB_SIZE, config.IMG_FEAT_WIDTH,config.IMG_FEAT_WIDTH]))) n.iatt_iq_eltwise = L.Eltwise(n.iatt_q_tile2, n.iatt_i_resh1, eltwise_param=dict(operation=0)) n.iatt_iq_droped = L.Dropout(n.iatt_iq_eltwise, dropout_param={'dropout_ratio':config.MFB_DROPOUT_RATIO}) n.iatt_iq_resh2 = L.Reshape(n.iatt_iq_droped, reshape_param=dict(shape=dict(dim=[-1,config.JOINT_EMB_SIZE,config.IMG_FEAT_SIZE,1]))) n.iatt_iq_permute1 = L.Permute(n.iatt_iq_resh2, permute_param=dict(order=[0,2,1,3])) n.iatt_iq_resh2 = L.Reshape(n.iatt_iq_permute1, reshape_param=dict(shape=dict(dim=[-1,config.IMG_FEAT_SIZE, config.MFB_OUT_DIM,config.MFB_FACTOR_NUM]))) n.iatt_iq_sumpool = L.Pooling(n.iatt_iq_resh2, pool=P.Pooling.SUM, \ pooling_param=dict(kernel_w=config.MFB_FACTOR_NUM, kernel_h=1)) n.iatt_iq_permute2 = L.Permute(n.iatt_iq_sumpool, permute_param=dict(order=[0,2,1,3])) n.iatt_iq_sqrt = L.SignedSqrt(n.iatt_iq_permute2) n.iatt_iq_l2 = L.L2Normalize(n.iatt_iq_sqrt) ## 2 conv layers 1000 -> 512 -> 2 n.iatt_conv1 = L.Convolution(n.iatt_iq_l2, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.iatt_relu = L.ReLU(n.iatt_conv1) n.iatt_conv2 = L.Convolution(n.iatt_relu, kernel_size=1, stride=1, num_output=config.NUM_IMG_GLIMPSE, pad=0, weight_filler=dict(type='xavier')) n.iatt_resh = L.Reshape(n.iatt_conv2, reshape_param=dict(shape=dict(dim=[-1,config.NUM_IMG_GLIMPSE,config.IMG_FEAT_SIZE]))) n.iatt_softmax = L.Softmax(n.iatt_resh, axis=2) n.iatt_softmax_resh = L.Reshape(n.iatt_softmax,reshape_param=dict(shape=dict(dim=[-1,config.NUM_IMG_GLIMPSE,config.IMG_FEAT_WIDTH,config.IMG_FEAT_WIDTH]))) iatt_maps = L.Slice(n.iatt_softmax_resh, ntop=config.NUM_IMG_GLIMPSE,slice_param={'axis':1}) dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) iatt_feature_list = [] for i in xrange(config.NUM_IMG_GLIMPSE): if config.NUM_IMG_GLIMPSE == 1: n.__setattr__('iatt_feat%d'%i, L.SoftAttention(n.i_feat_resh, iatt_maps, dummy)) else: n.__setattr__('iatt_feat%d'%i, L.SoftAttention(n.i_feat_resh, iatt_maps[i], dummy)) n.__setattr__('iatt_feat%d_resh'%i, L.Reshape(n.__getattr__('iatt_feat%d'%i), \ reshape_param=dict(shape=dict(dim=[0,-1])))) iatt_feature_list.append(n.__getattr__('iatt_feat%d_resh'%i)) n.iatt_feat_concat = L.Concat(*iatt_feature_list) n.iatt_feat_concat_resh = L.Reshape(n.iatt_feat_concat, reshape_param=dict(shape=dict(dim=[0,-1,1,1]))) ''' Fine-grained Image-Question MFB fusion ''' n.mfb_q_proj = L.InnerProduct(n.q_feat_resh, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_i_proj = L.InnerProduct(n.iatt_feat_concat_resh, num_output=config.JOINT_EMB_SIZE, weight_filler=dict(type='xavier')) n.mfb_iq_eltwise = L.Eltwise(n.mfb_q_proj, n.mfb_i_proj, eltwise_param=dict(operation=0)) n.mfb_iq_drop = L.Dropout(n.mfb_iq_eltwise, dropout_param={'dropout_ratio':config.MFB_DROPOUT_RATIO}) n.mfb_iq_resh = L.Reshape(n.mfb_iq_drop, reshape_param=dict(shape=dict(dim=[-1,1,config.MFB_OUT_DIM,config.MFB_FACTOR_NUM]))) n.mfb_iq_sumpool = L.Pooling(n.mfb_iq_resh, pool=P.Pooling.SUM, \ pooling_param=dict(kernel_w=config.MFB_FACTOR_NUM, kernel_h=1)) n.mfb_out = L.Reshape(n.mfb_iq_sumpool,\ reshape_param=dict(shape=dict(dim=[-1,config.MFB_OUT_DIM]))) n.mfb_sign_sqrt = L.SignedSqrt(n.mfb_out) n.mfb_l2 = L.L2Normalize(n.mfb_sign_sqrt) n.prediction = L.InnerProduct(n.mfb_l2, num_output=config.NUM_OUTPUT_UNITS, weight_filler=dict(type='xavier')) if mode == 'val': n.loss = L.SoftmaxWithLoss(n.prediction, n.label) else: n.loss = L.SoftmaxKLDLoss(n.prediction, n.label) return n.to_proto()
def qlstm(mode, batchsize, T, question_vocab_size): n = caffe.NetSpec() mode_str = json.dumps({'mode': mode, 'batchsize': batchsize}) n.data, n.cont, n.img_feature, n.label, n.glove = L.Python(\ module='vqa_data_provider_layer', layer='VQADataProviderLayer', param_str=mode_str, ntop=5 ) n.embed_ba = L.Embed(n.data, input_dim=question_vocab_size, num_output=300, \ weight_filler=dict(type='uniform',min=-0.08,max=0.08)) n.embed = L.TanH(n.embed_ba) concat_word_embed = [n.embed, n.glove] n.concat_embed = L.Concat(*concat_word_embed, concat_param={'axis': 2}) # T x N x 600 # LSTM1 n.lstm1 = L.LSTM(\ n.concat_embed, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops1 = L.Slice(n.lstm1, ntop=T, slice_param={'axis': 0}) for i in xrange(T - 1): n.__setattr__('slice_first' + str(i), tops1[int(i)]) n.__setattr__('silence_data_first' + str(i), L.Silence(tops1[int(i)], ntop=0)) n.lstm1_out = tops1[T - 1] n.lstm1_reshaped = L.Reshape(n.lstm1_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm1_reshaped_droped = L.Dropout(n.lstm1_reshaped, dropout_param={'dropout_ratio': 0.3}) n.lstm1_droped = L.Dropout(n.lstm1, dropout_param={'dropout_ratio': 0.3}) # LSTM2 n.lstm2 = L.LSTM(\ n.lstm1_droped, n.cont,\ recurrent_param=dict(\ num_output=1024,\ weight_filler=dict(type='uniform',min=-0.08,max=0.08),\ bias_filler=dict(type='constant',value=0))) tops2 = L.Slice(n.lstm2, ntop=T, slice_param={'axis': 0}) for i in xrange(T - 1): n.__setattr__('slice_second' + str(i), tops2[int(i)]) n.__setattr__('silence_data_second' + str(i), L.Silence(tops2[int(i)], ntop=0)) n.lstm2_out = tops2[T - 1] n.lstm2_reshaped = L.Reshape(n.lstm2_out,\ reshape_param=dict(\ shape=dict(dim=[-1,1024]))) n.lstm2_reshaped_droped = L.Dropout(n.lstm2_reshaped, dropout_param={'dropout_ratio': 0.3}) concat_botom = [n.lstm1_reshaped_droped, n.lstm2_reshaped_droped] n.lstm_12 = L.Concat(*concat_botom) n.q_emb_tanh_droped_resh = L.Reshape( n.lstm_12, reshape_param=dict(shape=dict(dim=[-1, 2048, 1, 1]))) n.q_emb_tanh_droped_resh_tiled_1 = L.Tile(n.q_emb_tanh_droped_resh, axis=2, tiles=14) n.q_emb_tanh_droped_resh_tiled = L.Tile(n.q_emb_tanh_droped_resh_tiled_1, axis=3, tiles=14) n.i_emb_tanh_droped_resh = L.Reshape( n.img_feature, reshape_param=dict(shape=dict(dim=[-1, 2048, 14, 14]))) n.blcf = L.CompactBilinear(n.q_emb_tanh_droped_resh_tiled, n.i_emb_tanh_droped_resh, compact_bilinear_param=dict(num_output=16000, sum_pool=False)) n.blcf_sign_sqrt = L.SignedSqrt(n.blcf) n.blcf_sign_sqrt_l2 = L.L2Normalize(n.blcf_sign_sqrt) n.blcf_droped = L.Dropout(n.blcf_sign_sqrt_l2, dropout_param={'dropout_ratio': 0.1}) # multi-channel attention n.att_conv1 = L.Convolution(n.blcf_droped, kernel_size=1, stride=1, num_output=512, pad=0, weight_filler=dict(type='xavier')) n.att_conv1_relu = L.ReLU(n.att_conv1) n.att_conv2 = L.Convolution(n.att_conv1_relu, kernel_size=1, stride=1, num_output=2, pad=0, weight_filler=dict(type='xavier')) n.att_reshaped = L.Reshape( n.att_conv2, reshape_param=dict(shape=dict(dim=[-1, 2, 14 * 14]))) n.att_softmax = L.Softmax(n.att_reshaped, axis=2) n.att = L.Reshape(n.att_softmax, reshape_param=dict(shape=dict(dim=[-1, 2, 14, 14]))) att_maps = L.Slice(n.att, ntop=2, slice_param={'axis': 1}) n.att_map0 = att_maps[0] n.att_map1 = att_maps[1] dummy = L.DummyData(shape=dict(dim=[batchsize, 1]), data_filler=dict(type='constant', value=1), ntop=1) n.att_feature0 = L.SoftAttention(n.i_emb_tanh_droped_resh, n.att_map0, dummy) n.att_feature1 = L.SoftAttention(n.i_emb_tanh_droped_resh, n.att_map1, dummy) n.att_feature0_resh = L.Reshape( n.att_feature0, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.att_feature1_resh = L.Reshape( n.att_feature1, reshape_param=dict(shape=dict(dim=[-1, 2048]))) n.att_feature = L.Concat(n.att_feature0_resh, n.att_feature1_resh) # merge attention and lstm with compact bilinear pooling n.att_feature_resh = L.Reshape( n.att_feature, reshape_param=dict(shape=dict(dim=[-1, 4096, 1, 1]))) n.lstm_12_resh = L.Reshape( n.lstm_12, reshape_param=dict(shape=dict(dim=[-1, 2048, 1, 1]))) n.bc_att_lstm = L.CompactBilinear(n.att_feature_resh, n.lstm_12_resh, compact_bilinear_param=dict( num_output=16000, sum_pool=False)) n.bc_sign_sqrt = L.SignedSqrt(n.bc_att_lstm) n.bc_sign_sqrt_l2 = L.L2Normalize(n.bc_sign_sqrt) n.bc_dropped = L.Dropout(n.bc_sign_sqrt_l2, dropout_param={'dropout_ratio': 0.1}) n.bc_dropped_resh = L.Reshape( n.bc_dropped, reshape_param=dict(shape=dict(dim=[-1, 16000]))) n.prediction = L.InnerProduct(n.bc_dropped_resh, num_output=3000, weight_filler=dict(type='xavier')) n.loss = L.SoftmaxWithLoss(n.prediction, n.label) return n.to_proto()
def generate_model(split, config): n = caffe.NetSpec() batch_size = config.N mode_str = str(dict(split=split, batch_size=batch_size)) n.language, n.cont, n.image, n.spatial, n.label = L.Python(module=config.data_provider, layer=config.data_provider_layer, param_str=mode_str, ntop=5) # the base net (VGG-16) n.conv1_1, n.relu1_1 = conv_relu(n.image, 64, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv1_2, n.relu1_2 = conv_relu(n.relu1_1, 64, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.pool1 = max_pool(n.relu1_2) n.conv2_1, n.relu2_1 = conv_relu(n.pool1, 128, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv2_2, n.relu2_2 = conv_relu(n.relu2_1, 128, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.pool2 = max_pool(n.relu2_2) n.conv3_1, n.relu3_1 = conv_relu(n.pool2, 256, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv3_2, n.relu3_2 = conv_relu(n.relu3_1, 256, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv3_3, n.relu3_3 = conv_relu(n.relu3_2, 256, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.pool3 = max_pool(n.relu3_3) n.conv4_1, n.relu4_1 = conv_relu(n.pool3, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv4_2, n.relu4_2 = conv_relu(n.relu4_1, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv4_3, n.relu4_3 = conv_relu(n.relu4_2, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.pool4 = max_pool(n.relu4_3) n.conv5_1, n.relu5_1 = conv_relu(n.pool4, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv5_2, n.relu5_2 = conv_relu(n.relu5_1, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.conv5_3, n.relu5_3 = conv_relu(n.relu5_2, 512, fix_param=config.fix_vgg, finetune=(not config.fix_vgg)) n.pool5 = max_pool(n.relu5_3) # fully conv n.fcn_fc6, n.fcn_relu6 = conv_relu(n.pool5, 4096, ks=7, pad=3) if config.vgg_dropout: n.fcn_drop6 = L.Dropout(n.fcn_relu6, dropout_ratio=0.5, in_place=True) n.fcn_fc7, n.fcn_relu7 = conv_relu(n.fcn_drop6, 4096, ks=1, pad=0) n.fcn_drop7 = L.Dropout(n.fcn_relu7, dropout_ratio=0.5, in_place=True) n.fcn_fc8 = conv(n.fcn_drop7, 1000, ks=1, pad=0) else: n.fcn_fc7, n.fcn_relu7 = conv_relu(n.fcn_relu6, 4096, ks=1, pad=0) n.fcn_fc8 = conv(n.fcn_relu7, 1000, ks=1, pad=0) # embedding n.embed = L.Embed(n.language, input_dim=config.vocab_size, num_output=config.embed_dim, weight_filler=dict(type='uniform', min=-0.08, max=0.08)) # LSTM n.lstm = L.LSTM(n.embed, n.cont, recurrent_param=dict(num_output=config.lstm_dim, weight_filler=dict(type='uniform', min=-0.08, max=0.08), bias_filler=dict(type='constant', value=0))) tops = L.Slice(n.lstm, ntop=config.T, slice_param=dict(axis=0)) for i in range(config.T - 1): n.__setattr__('slice'+str(i), tops[i]) n.__setattr__('silence'+str(i), L.Silence(tops[i], ntop=0)) n.lstm_out = tops[-1] n.lstm_feat = L.Reshape(n.lstm_out, reshape_param=dict(shape=dict(dim=[-1, config.lstm_dim]))) # Tile LSTM feature n.lstm_resh = L.Reshape(n.lstm_feat, reshape_param=dict(shape=dict(dim=[-1, config.lstm_dim, 1, 1]))) n.lstm_tile_1 = L.Tile(n.lstm_resh, axis=2, tiles=config.featmap_H) n.lstm_tile_2 = L.Tile(n.lstm_tile_1, axis=3, tiles=config.featmap_W) # L2 Normalize image and language features n.img_l2norm = L.L2Normalize(n.fcn_fc8) n.lstm_l2norm = L.L2Normalize(n.lstm_tile_2) # Concatenate n.feat_all = L.Concat(n.lstm_l2norm, n.img_l2norm, n.spatial, concat_param=dict(axis=1)) # MLP Classifier over concatenated feature n.fcn_l1, n.fcn_relu1 = conv_relu(n.feat_all, config.mlp_hidden_dims, ks=1, pad=0) if config.mlp_dropout: n.fcn_drop1 = L.Dropout(n.fcn_relu1, dropout_ratio=0.5, in_place=True) n.fcn_scores = conv(n.fcn_drop1, 1, ks=1, pad=0) else: n.fcn_scores = conv(n.fcn_relu1, 1, ks=1, pad=0) # Loss Layer n.loss = L.SigmoidCrossEntropyLoss(n.fcn_scores, n.label) return n.to_proto()