def __init__(self, axis=1, normalize=True, l1=0., l2=0., **kwargs): self.axis = axis self.normalize = normalize self.l1 = K.cast_to_floatx(l1) self.l2 = K.cast_to_floatx(l2) self.uses_learning_phase = normalize super(BatchLoss, self).__init__(**kwargs)
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [] with tf.control_dependencies([tf.compat.v1.assign_add(self.iterations, 1)]): t = tf.cast(self.iterations, backend.floatx()) # Due to the recommendations in [2], i.e. warming momentum schedule momentum_cache_t = self.beta_1 * ( 1. - 0.5 * (tf.pow(backend.cast_to_floatx(0.96), t * self.schedule_decay))) momentum_cache_t_1 = self.beta_1 * ( 1. - 0.5 * (tf.pow(backend.cast_to_floatx(0.96), (t + 1) * self.schedule_decay))) m_schedule_new = self.m_schedule * momentum_cache_t m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1 self.updates.append((self.m_schedule, m_schedule_new)) ms, vs = self._create_all_weights(params) for p, g, m, v in zip(params, grads, ms, vs): # the following equations given in [1] g_prime = g / (1. - m_schedule_new) m_t = self.beta_1 * m + (1. - self.beta_1) * g m_t_prime = m_t / (1. - m_schedule_next) v_t = self.beta_2 * v + (1. - self.beta_2) * tf.square(g) v_t_prime = v_t / (1. - tf.pow(self.beta_2, t)) m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime self.updates.append(tf.compat.v1.assign(m, m_t)) self.updates.append(tf.compat.v1.assign(v, v_t)) p_t = p - self.lr * m_t_bar / (backend.sqrt(v_t_prime) + self.epsilon) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(tf.compat.v1.assign(p, new_p)) return self.updates
def __init__(self, c1, c2, sigma, **kwargs): super(Brish, self).__init__(**kwargs) self.c1 = K.cast_to_floatx(c1) self.c2 = K.cast_to_floatx(c2) self.sigma = K.cast_to_floatx(sigma) self.cp = K.cast_to_floatx((self.c1 + self.c2) / 2) self.cm = K.cast_to_floatx((self.c1 - self.c2) / 2) self.cms = K.cast_to_floatx( (self.c1 - self.c2) / np.sqrt(2 * np.pi) * self.sigma) self.s1 = K.cast_to_floatx(np.sqrt(2) * self.sigma) self.s2 = K.cast_to_floatx(2 * self.sigma**2)
def Model(X, y, inputSize): X = K.cast_to_floatx(X) y = K.cast_to_floatx(y) # define the keras model model = Sequential() model.add(Dense(12, input_dim=inputSize, activation='relu')) model.add(Dense(8, activation='relu')) model.add(Dense(1, activation='sigmoid')) # compile the keras model model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # fit the keras model on the dataset model.fit(X, y, epochs=1, batch_size=10) # evaluate the keras model _, accuracy = model.evaluate(X, y) print('Accuracy: %.2f' % (accuracy * 100)) return model
def get_constants(self, inputs, training=None): constants = [] constants.append([K.cast_to_floatx(1.) for _ in range(3)]) if 0. < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = [ K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(3) ] constants.append(rec_dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) return constants
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] t = K.cast(self.iterations, K.floatx()) + 1 # Due to the recommendations in [2], i.e. warming momentum schedule momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay))) momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay))) m_schedule_new = self.m_schedule * momentum_cache_t m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1 self.updates.append((self.m_schedule, m_schedule_new)) shapes = [K.int_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): # the following equations given in [1] g_prime = g / (1. - m_schedule_new) m_t = self.beta_1 * m + (1. - self.beta_1) * g m_t_prime = m_t / (1. - m_schedule_next) if np.random.choice([1, -1]) == 1: v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g) else: v_t = K.maximum(self.beta_2 * v, K.abs(g)) v_t_prime = v_t / (1. - K.pow(self.beta_2, t)) m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def __init__(self, beta_init=0.1, weights=None, activity_regularizer=None, **kwargs): self.supports_masking = True self.beta_init = K.cast_to_floatx(beta_init) self.initial_weights = weights self.activity_regularizer = regularizers.get(activity_regularizer) super(ParametricSigmoid, self).__init__(**kwargs)
def __init__(self, theta=1.0, **kwargs): super(ThresholdedReLU, self).__init__(**kwargs) if theta is None: raise ValueError('Theta of a Thresholded ReLU layer cannot be ' 'None, requires a float. Got %s' % theta) if theta < 0: raise ValueError('The theta value of a Thresholded ReLU layer ' 'should be >=0, got %s' % theta) self.supports_masking = True self.theta = backend.cast_to_floatx(theta)
def load_tinyimagenet(tinyimagenet_path='./'): images = [plt.imread(fp) for fp in glob(os.path.join(tinyimagenet_path, '*.jpg'))] for i in range(len(images)): if len(images[i].shape) != 3: images[i] = np.stack([images[i], images[i], images[i]], axis=-1) images = np.stack(images) images = normalize_minus1_1(K.cast_to_floatx(images)) return images
def get_constants(self, x): constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.output_dim)) B_U = K.in_train_phase(K.dropout(ones, self.dropout_U), ones) constants.append(B_U) else: constants.append(K.cast_to_floatx(1.)) if self.consume_less == 'cpu' and 0 < self.dropout_W < 1: input_shape = self.input_spec[0].shape input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones) constants.append(B_W) else: constants.append(K.cast_to_floatx(1.)) return constants
def get_constants(self, x): constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * self.output_dim, 1) B_U = K.in_train_phase(K.dropout(ones, self.dropout_U), ones) constants.append(B_U) else: constants.append(K.cast_to_floatx(1.)) if self.consume_less == 'cpu' and 0 < self.dropout_W < 1: input_shape = self.input_spec[0].shape input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * input_dim, 1) B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones) constants.append(B_W) else: constants.append(K.cast_to_floatx(1.)) return constants
def get_constants(self, inputs, training=None): constants = [] input_dim = K.int_shape(inputs)[-1] if 0.0 < self.dropout < 1.0: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, input_dim)) def dropped_inputs(): return K.dropout(ones, self.dropout) dp_mask = K.in_train_phase(dropped_inputs, ones, training=training) constants.append(dp_mask) else: dp_mask = K.cast_to_floatx(1.) constants.append(dp_mask) if 0 < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = [ K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(self.depth - 1) ] constants.append(rec_dp_mask) else: rec_dp_mask = [ K.cast_to_floatx(1.) for _ in range(self.depth - 1) ] constants.append(rec_dp_mask) return constants
def get_constants(self, x): constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * self.output_dim, 1) B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)] constants.append(B_U) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) if 0 < self.dropout_W < 1: input_shape = self.input_spec[0].shape input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * input_dim, 1) B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)] constants.append(B_W) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) return constants
def bbalpha_softmax_cross_entropy_with_mc_logits(alpha): alpha = K.cast_to_floatx(alpha) def loss(y_true, mc_logits): # log(p_ij), p_ij = softmax(logit_ij) #assert mc_logits.ndim == 3 mc_log_softmax = mc_logits - K.max(mc_logits, axis=2, keepdims=True) mc_log_softmax = mc_log_softmax - K.log(K.sum(K.exp(mc_log_softmax), axis=2, keepdims=True)) mc_ll = K.sum(y_true * mc_log_softmax, -1) # N x K K_mc = mc_ll.get_shape().as_list()[1] # only for tensorflow return - 1. / alpha * (logsumexp(alpha * mc_ll, 1) + K.log(1.0 / K_mc)) return loss
def get_constants(self, x, training=None): constants = [] if 0 < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * self.units, 1) B_U = K.in_train_phase(K.dropout(ones, self.recurrent_dropout), ones) constants.append(B_U) else: constants.append(K.cast_to_floatx(1.)) return constants
def get_constants(self, x): constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.output_dim)) B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)] constants.append(B_U) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) if 0 < self.dropout_W < 1: input_shape = K.int_shape(x) input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)] constants.append(B_W) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) return constants
def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0): y_pred = K.constant(y_pred) if not K.is_tensor(y_pred) else y_pred y_true = K.cast(y_true, y_pred.dtype) if label_smoothing is not 0: smoothing = K.cast_to_floatx(label_smoothing) y_true = K.switch(K.greater(smoothing, 0), lambda: y_true * (1.0 - smoothing) + 0.5 * smoothing, lambda: y_true) return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1)
def __init__(self, k, b, wtype='random', randseed='none'): self.k = K.cast_to_floatx(k) self.uses_learning_phase = True self.wtype = wtype self.w = None self.p = None self.b = b if randseed == 'time': import time np.random.seed(int(time.time()))
def __init__(self, n_bases=10, spline_order=3, l2_smooth=0., l2=0.): """Regularizer for GAM's # Arguments n_bases: number of b-spline bases order: spline order (2 for quadratic, 3 for qubic splines) l2_smooth: float; Smoothness penalty (penalize w' * S * w) l2: float; L2 regularization factor - overall weights regularizer """ # convert S to numpy-array if it's a list self.n_bases = n_bases self.spline_order = spline_order self.l2_smooth = K.cast_to_floatx(l2_smooth) self.l2 = K.cast_to_floatx(l2) # convert to K.constant self.S = K.constant( K.cast_to_floatx(get_S(n_bases, spline_order, add_intercept=False)))
def get_constants(self, x): constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * self.output_dim, 1) B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(4)] constants.append(B_U) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) if 0 < self.dropout_W < 1: input_shape = self.input_spec[0].shape input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.concatenate([ones] * input_dim, 1) B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4)] constants.append(B_W) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) return constants
def call(self, inputs, memory=None): if memory is None: mem = K.zeros( (K.shape(inputs)[0], self.mem_size, K.shape(inputs)[-1])) else: mem = K.variable(K.cast_to_floatx(memory)) inputs = K.concatenate([mem, inputs], axis=1) ret = super(CausalConv, self).call(inputs) if self.return_memory: ret = ret, inputs[:, :self.mem_size, :] return ret
def __init__(self, theta=1.0, **kwargs): super(ThresholdedReLU, self).__init__(**kwargs) if theta is None: raise ValueError( 'Theta of a Thresholded ReLU layer cannot be None, expecting a float.' f' Received: {theta}') if theta < 0: raise ValueError('The theta value of a Thresholded ReLU layer ' f'should be >=0. Received: {theta}') self.supports_masking = True self.theta = backend.cast_to_floatx(theta)
def get_data_gen(): """ Return train and val generators that give data in batches, and data label names. """ # load data (X_train, y_train), (X_val, y_val) = cifar10.load_data() if K.image_data_format() == 'channels_first': X_train = X_train.transpose(0, 2, 3, 1) X_val = X_val.transpose(0, 2, 3, 1) X_train = K.cast_to_floatx(X_train) X_val = K.cast_to_floatx(X_val) y_train = to_categorical(y_train) y_val = to_categorical(y_val) X_train -= mean X_val -= mean X_train /= std X_val /= std # train generator train_transforms = [ partial(random_crop, new_size=32, padding=4), partial(horizontal_flip, f=0.5) ] train_generator = ArrayDataGenerator(X_train, y_train, transforms=train_transforms, shuffle=True, seed=28) # validation generator val_generator = ArrayDataGenerator(X_val, y_val, shuffle=False) return train_generator, val_generator
def reg_insidebatch(lastoutput,current): #function to be called by tf.scan, which will loop through the batch dimension regLoss=K.cast_to_floatx(0.) [height, width, channels]=K.int_shape(current) for i in range(channels): Wi=K.flatten(current[:,:,i]) for j in range(channels): Wj=K.flatten(current[:,:,j]) if j != i: tempLoss=K.sqrt(K.square(K.sum(Wi * Wj,axis=-1,keepdims=False))) regLoss+=tempLoss regLoss+=lastoutput # add the output from the current 3D tensor to the preceding batch loop return regLoss
def concrete_dropout(self, x): eps = K.cast_to_floatx(K.epsilon()) temp = 1.0 / 10.0 unif_noise = K.random_uniform(shape=K.shape(x)) drop_prob = (K.log(self.p + eps) - K.log(1. - self.p + eps) + K.log(unif_noise + eps) - K.log(1. - unif_noise + eps)) drop_prob = K.sigmoid(drop_prob / temp) random_tensor = 1. - drop_prob retain_prob = 1. - self.p x *= random_tensor x /= retain_prob return x
def weight_norm_regularizer(layer, weight): """Splits weight direction and norm to optimize them separately # Arguments l: Layer to apply w norm w: Float; Initial weights """ w_norm = K.cast_to_floatx(np.linalg.norm(K.get_value(weight), axis=0)) g = layer.add_weight(name="{}_{}_g".format(layer.name, weight.name.split(':')[-1]), shape=w_norm.shape, initializer=keras.initializers.Constant(w_norm)) normed_weight = weight * (g / l2norm(weight)) return normed_weight
def lossfn(self,y_true, y_pred): K.set_floatx('float32') Ploc_true = (y_true[0]) mu_true = (y_true[1]) Ploc_pred = (y_pred[0]) mu_pred = (y_pred[1]) print(y_pred[1]) sigma = K.cast_to_floatx(np.diag(np.full((2*self.F,), 0.005))) cost = (-tf.log(Ploc_pred) + (0.5) * tf.log(tf.matrix_determinant(sigma)) + (0.5) * K.dot(tf.transpose(mu_true - mu_pred), K.dot(tf.matrix_inverse(sigma), (mu_true - mu_pred))) + tf.log(2 * m.pi)) return cost
def get_constants(self, x): print("begin get_constants(self, x)") constants = [] if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.controller_output_dim)) B_U = [ K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(4) ] constants.append(B_U) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) if 0 < self.dropout_W < 1: input_shape = self.input_spec[0].shape input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) B_W = [ K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4) ] constants.append(B_W) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) # if 0 < self.dropout_R < 1: # input_shape = self.input_spec[0].shape # input_dim = input_shape[-1] # ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) # ones = K.tile(ones, (1, int(input_dim))) # B_R = [K.in_train_phase(K.dropout(ones, self.dropout_R), ones) for _ in range(4)] # constants.append(B_R) # else: # constants.append([K.cast_to_floatx(1.) for _ in range(4)]) print("end get_constants(self, x)") return constants
def Emu(para_array): if len(para_array.shape) == 1: para_array_rescaled = scale01(lhdmin, lhdmax, para_array) W_predArray, _ = GPy_predict( np.expand_dims(para_array_rescaled, axis=0)) W_predArray = W_predArray.reshape(latent_dim, -1).T # W_predArray = K.cast_to_floatx(W_predArray) # W_predArray = K.cast(W_predArray,K.floatx()) # W_predArray = tf.convert_to_tensor(W_predArray, dtype=tf.float32) W_predArray = K.cast_to_floatx(W_predArray) x_decoded = decoder.predict(W_predArray) x_decoded_scaled = (normFactor * x_decoded) + meanFactor return np.squeeze(x_decoded_scaled) #[0]
def categorical_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0): y_pred = K.constant(y_pred) if not K.is_tensor(y_pred) else y_pred y_true = K.cast(y_true, y_pred.dtype) if label_smoothing is not 0: smoothing = K.cast_to_floatx(label_smoothing) def _smooth_labels(): num_classes = K.cast(K.shape(y_true)[1], y_pred.dtype) return y_true * (1.0 - smoothing) + (smoothing / num_classes) y_true = K.switch(K.greater(smoothing, 0), _smooth_labels, lambda: y_true) return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
def set_layer_sharpness(model, values): """Sets the sharpness values of all spiking layers. # Arguments model: Keras model with one or more Spiking layers. values: A list of sharpness values (between 0.0 and 1.0 inclusive) for each spiking layer in the same order as their indices. """ assert type(values) is list and all( [type(i) is float and i >= 0.0 and i <= 1.0 for i in values]) for i, v in enumerate(values): layer = model.layers[get_spiking_layer_indices(model=model)[i]] K.set_value(layer.sharpness, K.cast_to_floatx(v))
def get_data_gen(): """ Return train and val generators that give data in batches, and data label names. """ # load data (X_train, y_train), (X_val, y_val) = cifar10.load_data() if K.image_data_format() == 'channels_first': X_train = X_train.transpose(0, 2, 3, 1) X_val = X_val.transpose(0, 2, 3, 1) X_train = K.cast_to_floatx(X_train) X_val = K.cast_to_floatx(X_val) y_train = to_categorical(y_train) y_val = to_categorical(y_val) X_train -= mean X_val -= mean X_train /= std X_val /= std # train generator train_transforms = [ partial(random_crop, new_size=32, padding=4), partial(horizontal_flip, f=0.5) ] train_generator = ArrayDataGenerator(X_train, y_train, transforms=train_transforms, shuffle=True, seed=28) # validation generator val_generator = ArrayDataGenerator(X_val, y_val, shuffle=False) return train_generator, val_generator
def __init__(self, l2_reg=0., s_ll_reg=0., S_ll=None, orth_reg=0., embedding_dim=0, reshape=None, mask_value=None): """ Custom regularizer used for the last layer of a SimEc s_ll_reg enforces that W^TW approximates S, orth_reg enforces that WW^T approximates lambda*I, i.e. that the vectors are orthogonal (but not necessarily length 1) """ self.l2_reg = K.cast_to_floatx(l2_reg) self.s_ll_reg = K.cast_to_floatx(s_ll_reg) if s_ll_reg > 0.: assert (S_ll is not None), "need to give S_ll" self.S_ll = S_ll else: self.S_ll = None self.orth_reg = K.cast_to_floatx(orth_reg) if orth_reg > 0.: assert (embedding_dim > 0), "need to give shape of embedding layer, i.e. x.shape[0]" self.embedding_dim = embedding_dim self.reshape = reshape if mask_value is None: self.errfun = mean_squared_error else: self.errfun = masked_mse(mask_value)
def load_g10(): X_train, y_train = galaxy10.load_data() actual_X_train = np.zeros((X_train.shape[0], X_train.shape[1] + 1, X_train.shape[2] + 1, X_train.shape[3])) for idx, image in enumerate(X_train): padded = np.pad(image, 1, mode='edge') actual_X_train[idx] = padded[1:, 1:, 1:-1] X_train = normalize_minus1_1(cast_to_floatx(actual_X_train)) # X_test = normalize_minus1_1(cast_to_floatx(X_test)) return (X_train[:1000], y_train[:1000]), (X_train[:1000], y_train[:1000])
def get_constants(self, inputs, training=None): constants = [] if 0. < self.recurrent_dropout < 1.: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(3)] constants.append(rec_dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) return constants
def get_test_gen(dtype='val'): if dtype != 'val': raise ValueError("CIFAR10 only has validation data.") _, (X_val, y_val) = cifar10.load_data() if K.image_data_format() == 'channels_first': X_val = X_val.transpose(0, 2, 3, 1) X_val = K.cast_to_floatx(X_val) y_val = to_categorical(y_val) X_val -= mean X_val /= std val_generator = ArrayDataGenerator(X_val, y_val, shuffle=False) return val_generator
def concrete_dropout(self, x): ''' Concrete dropout - used at training time (gradients can be propagated) :param x: input :return: approx. dropped out input ''' eps = K.cast_to_floatx(K.epsilon()) temp = 0.1 unif_noise = K.random_uniform(shape=K.shape(x)) drop_prob = ( K.log(self.p + eps) - K.log(1. - self.p + eps) + K.log(unif_noise + eps) - K.log(1. - unif_noise + eps) ) drop_prob = K.sigmoid(drop_prob / temp) random_tensor = 1. - drop_prob retain_prob = 1. - self.p x *= random_tensor x /= retain_prob return x
def __init__(self, gamma=1., lam=10., axis='last'): self.gamma = K.cast_to_floatx(gamma) self.lam = K.cast_to_floatx(lam) self.axis = axis
def __init__(self, TV=0., TV2=0., axes=[0, 1]): self.TV = K.cast_to_floatx(TV) self.TV2 = K.cast_to_floatx(TV2) self.axes = list(axes)
def __init__(self, gamma=0., axis=1, division_idx=None): self.gamma = K.cast_to_floatx(gamma) self.axis = [] self.axis.append(axis) self.division_idx = division_idx
def __init__(self, l1=0., l2=0., axis=0): self.l1 = K.cast_to_floatx(l1) self.l2 = K.cast_to_floatx(l2) self.axis = [] self.axis.append(axis)
def __init__(self, l1=0., l2=0., axis=0): self.l1 = K.cast_to_floatx(l1) self.l2 = K.cast_to_floatx(l2) self.axis = axis
def __init__(self, l=0., p=0.1): self.l = K.cast_to_floatx(l) self.p = K.cast_to_floatx(p) self.uses_learning_phase = True
def __init__(self, l1=0., l2=0.): self.l1 = K.cast_to_floatx(l1) self.l2 = K.cast_to_floatx(l2) self.uses_learning_phase = False self.layer = None
def __init__(self, gamma=1., lam=10., division_idx=None): self.gamma = K.cast_to_floatx(gamma) self.lam = K.cast_to_floatx(lam) self.division_idx = division_idx
def __init__(self, beta_init=0.1, weights=None, activity_regularizer=None, **kwargs): self.supports_masking = True self.beta_init = K.cast_to_floatx(beta_init) self.initial_weights = weights self.activity_regularizer = regularizers.get(activity_regularizer) super(ParametricSigmoid, self).__init__(**kwargs)
def __init__(self, m=0., **kwargs): self.m = K.cast_to_floatx(m) super(ManifoldWeightRegularizer, self).__init__(**kwargs)
def __init__(self, l1=0., l2=0.,p_mask=None): self.l1 = K.cast_to_floatx(l1) self.l2 = K.cast_to_floatx(l2) self.p_mask = K.cast_to_floatx(p_mask) self.uses_learning_phase = True
def get_constants(self, x): constants = [] constants.append([K.cast_to_floatx(1.) for _ in range(4)]) constants.append([K.cast_to_floatx(1.) for _ in range(4)]) return constants