def build(self, input_shape): assert isinstance(input_shape, list) self.W1 = self.add_weight(name='kernel', shape=(1,), initializer='uniform', trainable=True, constraint=min_max_norm(min_value=0.78, max_value=4)) self.W2 = self.add_weight(name='kernel', shape=(1,), initializer='uniform', trainable=True, constraint=min_max_norm(min_value=0.78, max_value=4)) super(RotationThetaWeightLayer, self).build(input_shape)
def Heap_Block(Samples): Model_input = Input((1500,)) X = Dense( 1500, kernel_constraint=min_max_norm( 0, 0.6), bias_constraint=min_max_norm( 0, 0.6))(Model_input) # Initial layer X = Activation('relu')(X) X = Id_Block(X) # Starting the task of heaping blocks (3 times) X = Id_Block(X) X = Id_Block(X) X = Dense(5, activation='softmax')(X) # Classification layer model = Model(inputs=Model_input, outputs=X) return model
def Id_Block(X): X_copy = X # make a copy of X X = Dense( 1500, use_bias=True, kernel_constraint=min_max_norm( 0, 0), bias_constraint=min_max_norm( 0, 0.6))(X) X = Activation('relu')(X) X = Dense( 1500, use_bias=True, kernel_constraint=min_max_norm( 0, 0.6), bias_constraint=min_max_norm( 0, 0.6))(X) X = Add()([X, X_copy]) # skip these layers X = Activation('relu')(X) return X
def build(self, input_shape): initializer_uniform = RandomUniform(minval=0, maxval=1) constraint_min_max = min_max_norm(min_value=0.0, max_value=1.0) self.b = self.add_weight(name='b', shape=(input_shape[-1], ), initializer=initializer_uniform, constraint=constraint_min_max, trainable=True) super(ANDNoisy, self).build(input_shape)
def build(self, input_shape): ''' input_shape describes the number of the junctions. ''' assert isinstance(input_shape, list) self.W1 = self.add_weight(name='junction_weight_first_element', shape=(1, ), initializer='uniform', trainable=True, constraint=min_max_norm(min_value=0, max_value=1)) self.W2 = self.add_weight(name='junction_weight_second_element', shape=(1, ), initializer='uniform', trainable=True, constraint=min_max_norm(min_value=0, max_value=1)) super(JunctionWeightLayer, self).build(input_shape)
def test_min_max_norm(): array = get_example_array() for m in get_test_values(): norm_instance = constraints.min_max_norm(min_value=m, max_value=m * 2) normed = norm_instance(K.variable(array)) value = K.eval(normed) l2 = np.sqrt(np.sum(np.square(value), axis=0)) assert l2[l2 < m].size == 0 assert l2[l2 > m * 2 + 1e-5].size == 0
def test_min_max_norm(): array = get_example_array() for m in get_test_values(): norm_instance = constraints.min_max_norm(min_value=m, max_value=m * 2) normed = norm_instance(K.variable(array)) value = K.eval(normed) l2 = np.sqrt(np.sum(np.square(value), axis=0)) assert l2[l2 < m].size == 0 assert l2[l2 > m * 2 + 1e-5].size == 0
def create_dfn(input_shape): inputs = Input(shape=(input_shape, )) # first sub net hidden_1 = Dense( units=200, activation='relu', # 100 activity_regularizer=regularizers.l1(1e-10))(inputs) #drop1 = Dropout(.1)(hidden_1) hidden_2 = Dense( units=200, activation='relu', # 100 activity_regularizer=regularizers.l1(1e-10))(hidden_1) hidden_3 = Dense( units=100, activation='relu', # 60 activity_regularizer=regularizers.l1(1e-10))(hidden_2) theta_layer = Dense(1, activation='linear')(hidden_3) theta_outputs = Dense(1, activation='linear', kernel_constraint=min_max_norm(max_value=1, min_value=1), bias_constraint=min_max_norm(max_value=1, min_value=-1), kernel_initializer='ones', bias_initializer='zeros')(theta_layer) #bias_initializer= initializers.Constant(value= -0.05 ) response_outputs = Dense( input_shape, kernel_constraint=min_max_norm(max_value=3, min_value=1), bias_constraint=min_max_norm(max_value=3, min_value=-3), kernel_initializer=initializers.Constant(value=1.5), bias_initializer=initializers.Constant(value=0), activation='sigmoid')(theta_outputs) deepFowardFeedNets = Model(inputs, output=[theta_outputs, response_outputs]) projectTheta = Model(inputs, theta_layer) return deepFowardFeedNets, projectTheta
def Decoder(X): X = Id_Block(X, 0.05) X = Id_Block(X, 0.05) X = Id_Block(X, 0.05) X = Conv1D(kernel_size=3, kernel_constraint=min_max_norm(0, 0.05), activation='relu', padding='same', filters=1)(X) return X
def Id_Block(X, limit=0.09): X_copy = X # the ResNet algorithm X = Conv1D(kernel_size=3, kernel_constraint=min_max_norm(0, limit), activation='relu', padding='same', filters=32)(X) X = Conv1D(kernel_size=3, kernel_constraint=min_max_norm(0, limit), activation='relu', padding='same', filters=32)(X) X = Conv1D(kernel_size=3, kernel_constraint=min_max_norm(0, limit), padding='same', filters=32)(X) X = Add()([X, X_copy]) X = Activation('relu')(X) return X
def build(self, input_shapes): X_shape = input_shapes[0] # number of atom props * output width kernel_shape = (X_shape[-1], self.width) # atom (self) weights self.kernel_dense = self.add_weight( shape=kernel_shape, initializer=self.kernel_initializer, name="dense_kernel", regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias is not None: self.bias = self.add_weight(shape=(self.width, ), initializer=self.bias_initializer, name="bias", regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None constraint = min_max_norm(min_value=0.0, max_value=1.0, rate=1.0, axis=0) if self.conv_wts == "single": self.kernel_neigh = self.add_weight( shape=[1], initializer=self.kernel_initializer, name="kernel_neigh", regularizer=self.kernel_regularizer, constraint=constraint) self.kernel_self = self.add_weight( shape=[1], initializer=self.kernel_initializer, name="kernel_self", regularizer=self.kernel_regularizer, constraint=constraint) elif self.conv_wts == "all": self.kernel_neigh = self.add_weight( shape=(self.width, ), initializer=self.kernel_initializer, name="kernel_neigh", regularizer=self.kernel_regularizer, constraint=constraint) self.kernel_self = self.add_weight( shape=(self.width, ), initializer=self.kernel_initializer, name="kernel_neigh", regularizer=self.kernel_regularizer, constraint=constraint) self.built = True
def build(self, input_shape): self.f = self.add_weight(name='f', shape=(1,), # initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=0.0, max_value=1.0, rate=0.9), trainable=True) self.smax = self.add_weight(name='smax', shape=(1,), initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=1 / 15, max_value=1.0, rate=0.9), trainable=True) self.qmax = self.add_weight(name='qmax', shape=(1,), initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=0.2, max_value=1.0, rate=0.9), trainable=True) self.ddf = self.add_weight(name='ddf', shape=(1,), initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=0.0, max_value=1.0, rate=0.9), trainable=True) self.tmin = self.add_weight(name='tmin', shape=(1,), initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=0.0, max_value=1.0, rate=0.9), trainable=True) self.tmax = self.add_weight(name='tmax', shape=(1,), initializer=initializers.Constant(value=0.5), constraint=constraints.min_max_norm(min_value=0.0, max_value=1.0, rate=0.9), trainable=True) super(PRNNLayer, self).build(input_shape)
def Discriminator(X): X = Id_Block(X) X = Id_Block(X) X = Id_Block(X) X = Conv1D(kernel_size=3, kernel_constraint=min_max_norm(0, 0.09), activation='relu', padding='same', filters=1)(X) X = keras.layers.Reshape((900, ))(X) X = Dense(80, activation='relu')(X) X = Dense(30, activation='relu')(X) X = Dense(1, activation='sigmoid')(X) return X
def get_decoder(decoder, N, K): if decoder == 'SC': return SCDecoder(N, K), True else: try: data_path = os.environ['DEEPRAN_DATA_PATH'] except KeyError: data_path = '/home/iwodiany/Projects/DeepRAN/data/' model_name = '%s-%s-%s.h5' % (decoder, N, K) model_path = data_path + 'models/' + model_name nn_decoder = NNDecoder(N, K) if os.path.isfile(model_path): nn_decoder.decoder = load_model(model_path, custom_objects={'errors': errors}) if decoder == 'NN': if nn_decoder.decoder is None: nn_decoder.compose([512, 256, 128]) nn_decoder.train(train_llrs=True) nn_decoder.decoder.save(model_path) return nn_decoder, True elif decoder == 'NN_LLR': if nn_decoder.decoder is None: nn_decoder.compose([512, 256, 128], final_activation='hard_sigmoid', constraint=min_max_norm(-1.0, 1.0), use_bias=False) nn_decoder.train(train_llrs=True) nn_decoder.decoder.save(model_path) return nn_decoder, True else: raise ValueError( 'Unsupported decoder type! Supported decoders: SC, NN, NN_LLR!' )
y_train = keras.utils.to_categorical(y_train, num_classes) Y_test = keras.utils.to_categorical(Y_test, num_classes) model = Sequential() model.add( Dense(512, activation='relu', input_shape=(784, ), kernel_constraint=max_norm(2.))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu', bias_constraint=non_neg())) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu', kernel_constraint=unit_norm())) model.add(Dropout(0.2)) model.add( Dense(10, activation='softmax', bias_constraint=min_max_norm(0.0, 1.0))) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(X_test, Y_test)) score = model.evaluate(X_test, Y_test, verbose=0) print('Test loss:', score[0])
import sys gl = glob.glob(os.path.join('train', '*.[sf]')) poscount, locidx = rfutils.count_locpos(gl) embedding_dim = 4 filter_sizes = (3, 8) num_filters = 1 dropout_prob = (0.01, 0.01) hidden_dims = 10 K.set_floatx('float64') in_vals = Input((poscount, 1), name='vals', dtype='float64') normd = BatchNormalization(axis=1, gamma_constraint=min_max_norm(), beta_constraint=min_max_norm())(in_vals) in_locs = Input((poscount, ), name='locs', dtype='uint64') embed_locs = Embedding(locidx.watermark, embedding_dim, input_length=poscount)(in_locs) merged = concatenate([embed_locs, normd]) drop = Dropout(dropout_prob[0])(merged) conv_list = [] for filtsz in filter_sizes: tmp = Conv1D(num_filters, filtsz, activation='relu')(drop) tmp = Flatten()(MaxPooling1D()(tmp)) conv_list.append(tmp) out = Dense(1, activation='sigmoid')(Dense(hidden_dims, activation='relu')( Dropout(dropout_prob[1])(concatenate(conv_list)))) ml = Model(inputs=[in_locs, in_vals], outputs=out) ml.compile(Adam(lr=0.01), metrics=['acc'], loss=binary_crossentropy)
import rfutils import sys gl = glob.glob(os.path.join('train', '*.[sf]')) poscount, locidx = rfutils.count_locpos(gl) embedding_dim = 4 dropout_prob = 0.4 dense_count = int(sys.argv[3]) if len(sys.argv) > 3 else 50 optr = Adam(lr=0.03) K.set_floatx('float64') in_vals = Input((poscount, 1), name='vals', dtype='float64') normd = BatchNormalization( axis=1, gamma_constraint=min_max_norm(), beta_constraint=min_max_norm())(in_vals) in_locs = Input((poscount, ), name='locs', dtype='uint64') embed_locs = Embedding( locidx.watermark, embedding_dim, input_length=poscount)(in_locs) merged = concatenate([embed_locs, normd]) dense_list = [] for i in range(dense_count): dense_list.append( Dropout(dropout_prob)(Dense(1, activation='sigmoid')(Flatten()( merged)))) mult = multiply(dense_list) ml = Model(inputs=[in_locs, in_vals], outputs=mult) ml.compile(optr, metrics=['acc'], loss=mse) locs, vals, labels = rfutils.read_data(gl, poscount, locidx)
def embedding_model_lstm(self, words, embedding_weights_a=None, embedding_weights_b=None, trainable=False, skip_embed=False, return_sequences_b=False): lstm_unit_a = units lstm_unit_b = units * 2 embed_unit = int(hparams['embed_size']) x_shape = (tokens_per_sentence, ) decoder_dim = units * 2 # (tokens_per_sentence, units *2) if hparams['dense_activation'] is None or hparams[ 'dense_activation'] == 'none': decoder_dim = embed_unit valid_word_a = Input(shape=x_shape) #valid_word_b = Input(shape=x_shape) embeddings_a = Embedding(words, embed_unit, weights=[embedding_weights_a], input_length=tokens_per_sentence, trainable=trainable) embed_a = embeddings_a(valid_word_a) ### encoder for training ### lstm_a = Bidirectional( LSTM( units=lstm_unit_a, return_sequences=True, dropout=0.5 #return_state=True, #recurrent_dropout=0.2, #input_shape=(None,) ), merge_mode='concat', trainable=True) recurrent_a = lstm_a(embed_a) ############# #conv1d_b = Conv1D(tokens_per_sentence,lstm_unit_b)(recurrent_a) lstm_b = AttentionDecoder( units=lstm_unit_b, output_dim=decoder_dim, kernel_constraint=min_max_norm(), dropout=0.5 #return_sequences=return_sequences_b, #return_state=True ) recurrent_b = lstm_b(recurrent_a) if hparams['dense_activation'] is not None and hparams[ 'dense_activation'] != 'none': dense_b = Dense( embed_unit, input_shape=(tokens_per_sentence, ), activation=hparams['dense_activation'] #softmax, tanh, or relu #name='dense_layer_b', ) decoder_b = dense_b(recurrent_b) # recurrent_b dropout_b = Dropout(0.5)(decoder_b) model = Model([valid_word_a], dropout_b) # decoder_b else: model = Model([valid_word_a], recurrent_b) ### boilerplate ### adam = optimizers.Adam(lr=learning_rate) # loss try 'categorical_crossentropy', 'mse', 'binary_crossentropy' # optimizer try 'rmsprop' model.compile(optimizer=adam, loss='mse', metrics=['acc']) return model, None, None #, None, model_inference
input_bits = Input(dtype='float32', shape=(M, ), name='input') inner_layer = Dense(M, activation='relu')(input_bits) inner_layer = Dense(n_channel, activation='linear')(inner_layer) to_channel = BatchNormalization( axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=min_max_norm(min_value=0, max_value=0), gamma_constraint=unit_norm(axis=-1))(inner_layer) encoder = Model(input_bits, to_channel) # We need an extra identity in tensorflow for the input placeholder #output = Lambda(tf.identity,output_shape=(n_channel,), arguments=None, name = 'lambda_output')(to_channel) tf.identity(to_channel, name='output') print(encoder.summary()) # Export directory export_dir = os.path.dirname(os.path.realpath(__file__)) # Load weights from Keras model encoder.load_weights(export_dir + '/export/keras_weights_encoder.h5') # Now save as tensorflow model sess = K.get_session()