def readout_bn(feature_map, gaze_map_size, drop_rate, gaussian=None, gaze_prior=None): #batch normalization x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv1')(feature_map) x = BatchNormalization()(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv2')(x) x = BatchNormalization()(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(2, (1, 1), activation='relu', name='readout_conv3')(x) x = BatchNormalization()(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(1, (1, 1), activation='linear', name='readout_conv4')(x) x = GaussianSmooth(kernel_size = GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth')(x) logits = tf.reshape(x, [-1, gaze_map_size[0]*gaze_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits gaze_prior = np.maximum(gaze_prior, EPSILON*np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul(tf.ones((tf.shape(pre_prior_logits)[0],1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape(log_prior_tensor, [-1, gaze_map_size[0]*gaze_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if gaze_prior is None: return logits else: return logits, pre_prior_logits
def big_conv_lstm_readout_net(feature_map_in_seqs, feature_map_size, drop_rate, gaze_prior=None): batch_size = tf.shape(feature_map_in_seqs)[0] n_step = tf.shape(feature_map_in_seqs)[1] n_channel = int(feature_map_in_seqs.get_shape()[4]) feature_map = tf.reshape(feature_map_in_seqs, [batch_size*n_step, feature_map_size[0], feature_map_size[1], n_channel]) x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv1')(feature_map) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv2')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(8, (1, 1), activation='relu', name='readout_conv3')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(1, (1, 1), activation='relu', name='readout_conv4')(x) x = layers.core.Dropout(drop_rate)(x) #x = layers.core.Reshape((-1,))(x) temp_shape = x.get_shape()[1:4] temp_shape = [int(s) for s in temp_shape] x = tf.reshape(x, [batch_size, n_step, temp_shape[0], temp_shape[1], temp_shape[2]]) x = layers.ConvLSTM2D(filters=1, kernel_size=(3,3), strides=(1,1), padding='same', dropout=drop_rate, recurrent_dropout=drop_rate, return_sequences=True)(x) x = wps.TimeDistributed(layers.Conv2D(1, (1, 1), activation='linear'))(x) x = tf.reshape(x, [batch_size*n_step, feature_map_size[0], feature_map_size[1], 1]) x = GaussianSmooth(kernel_size = GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth')(x) logits = tf.reshape(x, [-1, feature_map_size[0]*feature_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits gaze_prior = np.maximum(gaze_prior, EPSILON*np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul(tf.ones((tf.shape(pre_prior_logits)[0],1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape(log_prior_tensor, [-1, feature_map_size[0]*feature_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if gaze_prior is None: return logits else: return logits, pre_prior_logits
def lstm_readout_net_old(feature_map_in_seqs, gaze_map_size, drop_rate, gaze_prior=None): x = wps.TimeDistributed(\ layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv1'))(feature_map_in_seqs) x = wps.TimeDistributed(layers.core.Dropout(drop_rate))(x) x = wps.TimeDistributed(\ layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv2'))(x) x = wps.TimeDistributed(layers.core.Dropout(drop_rate))(x) x = wps.TimeDistributed(\ layers.Conv2D(2, (1, 1), activation='relu', name='readout_conv3'))(x) x = wps.TimeDistributed(layers.core.Dropout(drop_rate))(x) x = wps.TimeDistributed(layers.core.Reshape((-1, )))(x) x = layers.recurrent.LSTM(units=gaze_map_size[0] * gaze_map_size[1], dropout=drop_rate, recurrent_dropout=drop_rate, return_sequences=True)(x) x = wps.TimeDistributed( layers.core.Dense(gaze_map_size[0] * gaze_map_size[1]))(x) x = wps.TimeDistributed(layers.core.Reshape(gaze_map_size + (1, )))(x) x = wps.TimeDistributed(\ GaussianSmooth(kernel_size = GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth'))(x) logits = tf.reshape(x, [-1, gaze_map_size[0] * gaze_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits gaze_prior = np.maximum(gaze_prior, EPSILON * np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul( tf.ones((tf.shape(pre_prior_logits)[0], 1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape( log_prior_tensor, [-1, gaze_map_size[0] * gaze_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if gaze_prior is None: return logits else: return logits, pre_prior_logits
def readout_net_BDD(feature_map, gaze_map_size, drop_rate, gaussian=None, gaze_prior=None): x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv1')(feature_map) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv2')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(2, (1, 1), activation='relu', name='readout_conv3')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(1, (1, 1), activation='linear', name='readout_conv4')(x) x = layers.convolutional.UpSampling2D(size=(3, 3))(x) x = tf.pad(x, [[0, 0], [ 0, 0, ], [0, 4], [0, 0]], 'CONSTANT', constant_values=0) x = GaussianSmooth(kernel_size=GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth')(x) logits = tf.reshape(x, [-1, gaze_map_size[0] * gaze_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits ''' #add prior map with open(data_dir + 'gaze_prior.pickle', 'rb') as f: gaze_prior = pickle.load(f) if gaze_prior.shape != gaze_map_size: gaze_prior = misc.imresize(gaze_prior, gaze_map_size) gaze_prior = gaze_prior.astype(np.float32) gaze_prior /= np.sum(gaze_prior) ''' gaze_prior = np.maximum(gaze_prior, EPSILON * np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul( tf.ones((tf.shape(pre_prior_logits)[0], 1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape( log_prior_tensor, [-1, gaze_map_size[0] * gaze_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if gaze_prior is None: return logits else: return logits, pre_prior_logits