def OptimTypeFunc(x, OpLr, OpMom): # Function to determine to get desiered optimezer return { 0: Nadam(lr=OpLr), 1: Adamax(lr=OpLr), 2: SGD(lr=OpLr, momentum=OpMom), 3: Adam(lr=OpLr, clipnorm=1.0), 4: Nadam(lr=OpLr) }[x]
def __init__( self, input_shape, number_of_classes, filtres=16, tailleBlock={ 'A': 10, 'B': 3, 'C': 3 }, optimiseur='Nadam', activation='elu', beta=1.1, initializer='he_normal', metrics=['accuracy'], learningR=None, #0.0005, nb_gpu=2): get_custom_objects()['swish'] = swish get_custom_objects()['e_swish'] = e_swish self.input_shape = input_shape self.number_of_classes = number_of_classes self.filtres = filtres self.tailleBlock = tailleBlock #if learningR is not None : self.optimiseur = optimiseur if learningR is not None: self.optimiseur = { 'SGD': SGD(learning_rate=learningR), 'RMSprop': RMSprop(learning_rate=learningR), 'Adagrad': Adagrad(learning_rate=learningR), 'Adadelta': Adadelta(learning_rate=learningR), 'Adam': Adam(learning_rate=learningR), 'Adamax': Adamax(learning_rate=learningR), 'Nadam': Nadam(learning_rate=learningR), }[optimiseur] else: self.optimiseur = { 'SGD': SGD(), 'RMSprop': RMSprop(), 'Adagrad': Adagrad(), 'Adadelta': Adadelta(), 'Adam': Adam(), 'Adamax': Adamax(), 'Nadam': Nadam(), }[optimiseur] self.activation = activation self.initializer = initializer self.nb_gpu = nb_gpu self.metrics = metrics # la valeur 3 indique que les canaux des couleurs sont à la fin # autrement -1 (je n'utilise pas cette syntaxe ) self.channel_axis = 3
def __init__(self, update_after='each', epsilon=0.85, EPS_DECAY=0.998, OBSERVATION_SPACE_SIZE=(5, 5, 3), ACTION_SPACE_SIZE=4, ep_number=0, RENDER_EVERY=200, DISCOUNT=0.50, BATCH_SIZE=350): #Update after: All = all subtasks; Each = each subtask self.update_after = update_after self.OBSERVATION_SPACE_SIZE = OBSERVATION_SPACE_SIZE self.ACTION_SPACE_SIZE = ACTION_SPACE_SIZE self.ep_number = ep_number self.RENDER_EVERY = RENDER_EVERY # render every 1000 episodes self.DISCOUNT = DISCOUNT self.BATCH_SIZE = BATCH_SIZE self.epsilon = epsilon self.EPSILON_DECAY = EPS_DECAY self.MIN_EPSILON = 0.05 self.logdir = "logs" self.REPLAY_BUFFER_SIZE = 10 * self.BATCH_SIZE self.epoch_history = deque(maxlen=self.REPLAY_BUFFER_SIZE) self.REWARDS = [] self.MEAN_REWARDS = [] self.WINDOW_SIZE = 100 self.model = StackCNNDenseModel( input_dim=self.OBSERVATION_SPACE_SIZE, output_dim=self.ACTION_SPACE_SIZE, hidden_dim=4, num_hidden_layers=2, activation='linear', initializer=tf.keras.initializers.glorot_normal() ) # self.create_model_simple() # tf.keras.models.load_model('model') self.model.compile(loss="mse", optimizer=Nadam(lr=0.005), metrics=['mae']) self.target_model = StackCNNDenseModel( input_dim=self.OBSERVATION_SPACE_SIZE, output_dim=self.ACTION_SPACE_SIZE, hidden_dim=4, num_hidden_layers=2, activation='linear', initializer=tf.keras.initializers.glorot_normal()) self.target_model.compile(loss="mse", optimizer=Nadam(lr=0.005), metrics=['mae'])
def train_model(df, days_ahead, num_iter): print("Training {}...".format(days_ahead)) for target in targets: df["target_{}".format(target)] = df.groupby("loc")[target].shift(1-days_ahead) train_df = df[df["target_{}".format(targets[0])].notnull()].copy() X_hist_train = np.zeros((train_df.shape[0], NUM_SHIFT, len(targets))) for i in range(len(targets)): X_hist_train[:, :, i] = train_df[ts_features[targets[i]]].values epochs = [10, 4] if days_ahead >= 20: epochs = [6, 2] for it in range(num_iter): print("Iteration:", it) K.clear_session() model = get_model() for loss, lr, epoch in [(get_custom_loss([0.15, 0.50, 0.85]), 0.0001, epochs[0]), (get_custom_loss([0.05, 0.50, 0.95]), 0.00005, epochs[1])]: model.compile(loss=loss, optimizer=SWA(Nadam(lr=lr), average_period=2)) model.fit(X_hist_train, get_output(train_df), sample_weight=get_weight(train_df), shuffle=True, batch_size=128, epochs=epoch, verbose=0) model.save_weights("weights/model_{d}_{it}.h5".format(d=days_ahead, it=it))
def construct_model(): """ Construct the CNN model. *** Please add your model implementation here, and don't forget compile the model E.g., model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) NOTE, You must include 'accuracy' in as one of your metrics, which will be used for marking later. *** :return: model: the initial CNN model """ model = Sequential([ Conv2D(16, kernel_size=3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)), MaxPooling2D(2, 2), Conv2D(32, kernel_size=3, padding='same', activation='relu'), MaxPooling2D(2, 2), Conv2D(64, 3, padding='same', activation='relu'), MaxPooling2D(2, 2), Conv2D(128, 3, padding='same', activation='relu'), Flatten(), Dense(64, 'relu'), Dropout(0.5), Dense(3, activation='softmax') ]) model.compile(loss=categorical_crossentropy, optimizer= Nadam(), metrics=['accuracy']) return model
def speed_model(): inputs = Input(shape=IMG_SHAPE) inputs1 = Lambda(lambda x: x/ 127.5 - 1, input_shape = IMG_SHAPE)(inputs) conv1 = Conv2D(24, (5, 5), padding="valid")(inputs1) act1 = Activation(ELU())(conv1) conv2 = Conv2D(36, (5, 5), strides=(2, 2), padding="valid")(act1) act2 = Activation(ELU())(conv2) drop1 = Dropout(0.5)(act2) conv3 = Conv2D(48, (5, 5), strides=(2, 2), padding="valid")(drop1) act3 = Activation(ELU())(conv3) conv4 = Conv2D(64, (3, 3), strides=(1, 1), padding="valid")(act3) act4 = Activation(ELU())(conv4) conv5 = Conv2D(64, (3, 3), padding="valid")(act4) flat1 = Flatten()(conv5) act4 = Activation(ELU())(flat1) dense1 = Dense(100)(act4) act5 = Activation(ELU())(dense1) dense2 = Dense(50)(act5) act6 = Activation(ELU())(dense2) dense4 = Dense(10)(act6) act8 = Activation(ELU())(dense4) output = Dense(1)(act8) model = Model(inputs, output) adam = Nadam() model.compile(optimizer=adam, loss='mse') print(model.summary()) return model
def _build_model(self) -> Sequential: model = Sequential() for state in self.sequence: if isinstance(state, Input): model.add(layers.Input(shape=(state.units,))) elif isinstance(state, Hidden): model.add(layers.Dense(units=state.units, activation=state.activation)) elif isinstance(state, Dropout): model.add(layers.Dropout(rate=state.rate)) elif isinstance(state, Output): model.add(layers.Dense(units=state.units, activation=state.activation)) optimizer = OPTIMIZER if optimizer == 'adam': optimizer = Adam(LEARNING_RATE) elif optimizer == 'nadam': optimizer = Nadam(LEARNING_RATE) elif optimizer == 'rmsprop': optimizer = RMSprop(LEARNING_RATE) metrics = [] for metric in METRICS: if metric == 'r_square': metric = RSquare(name=metric, y_shape=(1,)) elif metric == 'rmse': metric = RootMeanSquaredError(metric) metrics.append(metric) model.compile( optimizer=optimizer, loss=LOSS, metrics=metrics ) return model
def evaluate(self, device: str, strategy_name, pretrained): # init strategy if strategy_name == 'tpu': resolver = tf.distribute.cluster_resolver.TPUClusterResolver() tf.config.experimental_connect_to_cluster(resolver) tf.tpu.experimental.initialize_tpu_system(resolver) strategy = tf.distribute.experimental.TPUStrategy(resolver) print('TPU strategy selected.') else: strategy = tf.distribute.OneDeviceStrategy(device=device) print('GPU strategy selected.') dl = DataLoader(self.record_paths) self.test_ds = dl.get_dataset( 'test', batch_size=self.config['train_batch_size']) # infer steps for fitting and data structure test_steps = get_dataset_size(self.test_ds) data_structure = get_dataset_structure(self.test_ds) with strategy.scope(): model = get_model(input_shape=data_structure, name="VGG16_tf") model.load_weights( os.path.join(MODEL_PATH, pretrained, 'checkpoint')) optimizer = Nadam() model.compile(loss=tf.keras.losses.CategoricalCrossentropy(), optimizer=optimizer, metrics=[ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.Recall(), tf.keras.metrics.Precision() ]) model.summary() model.evaluate(self.test_ds, steps=test_steps)
def create(self, input_shape, segment_length, step, optimizer, learning_rate, verbose): identifier = f'Conv1D_pred' super().__init__(input_shape, segment_length, step, n_output_classes=1, loss_function='mean_squared_error', metrics=['mse'], identifier=identifier, verbose=verbose) self.add(Reshape((segment_length, 1), input_shape=(input_shape,))) self.add(Conv1D(128, 2, activation='relu', input_shape=(segment_length, 1))) self.add(MaxPooling1D(pool_size=2, strides=1)) self.add(Conv1D(64, 2, activation='relu')) self.add(GlobalAveragePooling1D()) self.add(Flatten()) self.add(Dense(10, activation='relu')) self.add(Dense(1, activation='linear')) if optimizer == 'sgd': if learning_rate: self.optimizer = SGD(lr=learning_rate, nesterov=True) else: self.optimizer = SGD(nesterov=True) elif optimizer == 'nadam' and learning_rate: self.optimizer = Nadam(lr=learning_rate) self.compile()
def read_model(file): model = get_unet0() model.compile(optimizer=Nadam(lr=1e-3), loss=jaccard_coef_loss, metrics=['binary_crossentropy', jaccard_coef_int]) model.load_weights(file) return model
def deep_lstm_model(image_shape, sequence_length, classes): (h, w, c) = image_shape sequence_shape = (sequence_length, h, w, c) video = Input(shape=sequence_shape) # cnn_base = VGG16(input_shape=image_shape, # weights="imagenet", # include_top=False) extractor_base = Xception( include_top=False, weights='imagenet', input_shape=image_shape, pooling='avg', ) # cnn_out = GlobalAveragePooling2D()(cnn_base.output) extractor = Model(extractor_base.input, extractor_base.output) extractor.trainable = False encoded_frames = TimeDistributed(extractor)(video) encoded_sequences = LSTM(64, return_sequences=True)(encoded_frames) encoded_sequence = LSTM(64)(encoded_sequences) hidden_layer = Dense(256, activation="relu")(encoded_sequence) outputs = Dense(classes, activation="softmax")(hidden_layer) model = Model(video, outputs) optimizer = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) model.compile(loss="sparse_categorical_crossentropy", optimizer=optimizer, metrics=["categorical_accuracy"]) return model
def build(self): """Builds the network using Keras. """ word_inputs = Input(shape=(None, MAX_WORD_LENGTH + 2), dtype="int32") inputs = [word_inputs] word_outputs = self._build_word_cnn(word_inputs) if len(self.word_vectorizers) > 0: additional_word_inputs = [ Input(shape=(None, input_dim), dtype="float32") for input_dim, dense_dim in self.word_vectorizers ] inputs.extend(additional_word_inputs) additional_word_embeddings = [ Dense(dense_dim)(additional_word_inputs[i]) for i, (_, dense_dim) in enumerate(self.word_vectorizers) ] word_outputs = Concatenate()([word_outputs] + additional_word_embeddings) outputs, lstm_outputs = self._build_basic_network(word_outputs) compile_args = { "optimizer": Nadam(lr=0.002, clipnorm=5.0), "loss": "categorical_crossentropy", "metrics": ["accuracy"] } self.model_ = Model(inputs, outputs) self.model_.compile(**compile_args) if self.verbose > 0: self.model_.summary(print_fn=log.info) return self
def fit_fnn(x_train, y_train, x_val): tf.keras.backend.clear_session() es = EarlyStopping(monitor='val_loss', mode='min', verbose=2, patience=2) X_train_array = x_train.values.reshape( (x_train.shape[0], x_train.shape[1], 1)) y_val = x_val[['X(t)']] X_val_array = x_val.drop(['X(t)'], axis=1).values.reshape( (x_val.shape[0], x_val.drop(['X(t)'], axis=1).shape[1], 1)) model = Sequential() model.add( Dense(X_train_array.shape[1], activation='relu', input_shape=(X_train_array.shape[1], X_train_array.shape[2]))) model.add(Dense(X_train_array.shape[1], activation='relu')) model.add(Dense(X_train_array.shape[1], activation='relu')) model.add(Dense(X_train_array.shape[1], activation='relu')) model.add(Dense(X_train_array.shape[1], activation='relu')) model.add(Flatten()) model.add(Dense(X_train_array.shape[1] // 2, activation='relu')) model.add(Dense(1)) nadam = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999) model.compile(optimizer=nadam, loss='mae', metrics=['mse']) # fit network model.fit(X_train_array, y_train, epochs=100, batch_size=128, validation_data=(X_val_array, y_val), shuffle=False, verbose=2, callbacks=[es]) return model
def learn_model(log, attributes, epochs, early_stop): num_activities = len(log.values[log.activity]) + 1 # Input + Embedding layer for every attribute input_layers = [] embedding_layers = [] for attr in attributes: if attr not in log.ignoreHistoryAttributes and attr != log.time and attr != log.trace: for k in range(log.k): i = Input(shape=(1, ), name=attr.replace(" ", "_").replace("(", "").replace( ")", "").replace(":", "_") + "_Prev%i" % k) input_layers.append(i) # e = Embedding(len(log.values[attr]) + 1, 32, embeddings_initializer="zeros")(i) e = Embedding(len(log.values[attr]) + 1, len(log.values[attr]) + 1, embeddings_initializer="zeros")(i) embedding_layers.append(e) concat = Concatenate()(embedding_layers) drop = Dropout(0.2)(concat) dense2 = Dense(num_activities)(drop) flat = Flatten()(dense2) output = Softmax(name="output")(flat) model = Model(inputs=input_layers, outputs=[output]) opt = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004, clipvalue=3) model.compile(loss={'output': 'categorical_crossentropy'}, optimizer=opt) model.summary() early_stopping = EarlyStopping(monitor='val_loss', patience=early_stop) outfile = 'tmp/model_{epoch:03d}-{val_loss:.2f}.h5' model_checkpoint = ModelCheckpoint(outfile, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto') x, y, vals = transform_data( log, [a for a in attributes if a != log.time and a != log.trace]) if len(y) < 10: split = 0 else: split = 0.2 model.fit(x=x, y=y, validation_split=split, verbose=2, callbacks=[early_stopping], batch_size=32, epochs=epochs) return model
def create_model(): strategy = tf.distribute.MirroredStrategy() with strategy.scope(): shape = (48, 96, 1) filters = 32 depth = 8 stacks = 1 input_img = Input(shape) model = MaskedConvolution2D(filters=filters, kernel_size=(7, 7), padding='same', mask='A')(input_img) # model = InitialBlock(filters)(input_img) model = ResidualBlockList(filters, depth, stacks)(model) for _ in range(2): model = Convolution2D(filters, (1, 1), padding='valid')(model) model = ReLU()(model) outs = Convolution2D(1, (1, 1), padding='valid')(model) outs = Activation('sigmoid')(outs) model = Model(input_img, outs) model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy']) model.summary() return model
def newCNN_model_new(num_variables, optimizer, nClasses, nl, dp): model = Sequential() model.add( Dense(36, input_dim=num_variables, kernel_initializer='glorot_normal', activation='relu')) model.add(Reshape((1, 6, 6), input_shape=(36, ))) #model.add(Conv2D(256, (3, 3), padding='same',activation='relu')) model.add(Conv2D(64, (3, 3), padding='same', activation='relu')) model.add(Conv2D(32, (3, 3), padding='same', activation='relu')) model.add(Conv2D(16, (3, 3), padding='same', activation='relu')) model.add(Flatten()) model.add(Dense(100, activation='relu')) model.add(Dropout(0.1)) model.add(Dense(25, activation='relu')) model.add(Dense(nClasses, activation='softmax')) if optimizer == 'Adam': model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01), metrics=['acc']) if optimizer == 'Nadam': model.compile(loss='categorical_crossentropy', optimizer=Nadam(lr=0.01), metrics=['acc']) if optimizer == 'RMSprop': model.compile(loss='categorical_crossentropy', optimizer=RMSprop(learning_rate=0.001), metrics=['acc']) return model
def buildModel(): print("Building Convolutional Neural Network") model = Sequential() model.add(Conv2D(32, (8, 8), padding="same", strides=(4, 4), input_shape=( img_cols, img_rows, img_channels))) # First layer of 80*80*4 with 32 filters model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Activation("relu")) # Second layer of 40*40*4 with 64 filters model.add(Conv2D(64, (4, 4), strides=(2, 2), padding='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Activation("relu")) # Third layer of 30*30*4 with 64 filters model.add(Conv2D(64, (3, 3), strides=(1, 1), padding='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Activation("relu")) model.add(Flatten()) model.add(Dense(512)) model.add(Activation("relu")) model.add(Dense(ACTIONS)) #adam = Adam(lr=LEARNING_RATE) nadam = Nadam(lr=LEARNING_RATE) model.compile(loss="mse", optimizer=nadam) # Creating model file if not present if not os.path.isfile(LOSS_FILE_PATH): model.save_weights("model.h5") print("Finished building the Convolutional Neural Network") return model
def init_model(self): self.model = Sequential() self.model.add( Dense(self.hidden_units, input_dim=self.input_units, activation=self.activation)) self.model.add( Dropout(self.dropout, noise_shape=self.noise_shape, seed=self.seed)) self.model.add( Dense(self.output_units, activation=self.activation_last)) if self.optimizer == 'RMSprop': opt = RMSprop(learning_rate=self.learning_rate) elif self.optimizer == 'Adadelta': opt = kAdadelta(learning_rate=self.learning_rate) elif self.optimizer == 'SGD': opt = SGD(learning_rate=self.learning_rate) elif self.optimizer == 'Adagrad': opt = Adagrad(learning_rate=self.learning_rate) elif self.optimizer == 'Adamax': opt = Adamax(learning_rate=self.learning_rate) elif self.optimizer == 'Nadam': opt = Nadam(learning_rate=self.learning_rate) else: opt = Adam(learning_rate=self.learning_rate) self.model.compile(optimizer=opt, loss=self.loss, metrics=[self.metrics])
def create_model(): # import Inception v3 model trained on imagenet base_model = tf.keras.applications.InceptionV3( input_shape=(96, 96, 3), include_top=False, weights="imagenet") base_model.trainable = False # initialize top layers model = Sequential([ base_model, GlobalAveragePooling2D(), Dropout(0.3), Dense(4, activation='softmax') ]) # compile model model.compile( optimizer=Nadam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07), loss="categorical_crossentropy", metrics=["accuracy"]) return model
def fit(self, X_train, y_train): # X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train) input_size = X_train.shape[1] if self.type == 'custom': self.make_custom_model(input_size) elif self.type == 'larger': self.larger_model(input_size) elif self.type == 'wider': self.wider_model(input_size) print(self.model.summary()) # model_json = self.model.to_json() # with open("DL_model.json", "w") as json_file: # json_file.write(model_json) self.model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4)) hist = self.model.fit(X_train, y_train, batch_size=5, epochs=500, verbose=0, shuffle=True) # KerasRegressor(build_fn=larger_model, epochs=50, batch_size=5, verbose=0)) return self.model
def nvidia_model(): model = Sequential() # normalization # perform custom normalization before lambda layer in network model.add(Lambda(lambda x: x/ 127.5 - 1, input_shape = IMG_SHAPE)) model.add(Conv2D(24, (5, 5), strides=(2,2), padding = 'valid', kernel_initializer = 'he_normal', name = 'conv1')) model.add(ELU()) model.add(Conv2D(36, (5, 5), strides=(2,2), padding = 'valid', kernel_initializer = 'he_normal', name = 'conv2')) model.add(ELU()) model.add(Conv2D(48, (5, 5), strides=(2,2), padding = 'valid', kernel_initializer = 'he_normal', name = 'conv3')) model.add(ELU()) model.add(Dropout(0.5)) model.add(Conv2D(64, (3, 3), strides = (1,1), padding = 'valid', kernel_initializer = 'he_normal', name = 'conv4')) model.add(ELU()) model.add(Conv2D(64, (3, 3), strides= (1,1), padding = 'valid', kernel_initializer = 'he_normal', name = 'conv5')) model.add(Flatten(name = 'flatten')) model.add(ELU()) model.add(Dense(100, kernel_initializer = 'he_normal', name = 'fc1')) model.add(ELU()) model.add(Dense(50, kernel_initializer = 'he_normal', name = 'fc2')) model.add(ELU()) model.add(Dense(10, kernel_initializer = 'he_normal', name = 'fc3')) model.add(ELU()) # do not put activation at the end because we want to exact output, not a class identifier model.add(Dense(1, name = 'output', kernel_initializer = 'he_normal')) adam = Nadam() model.compile(optimizer = adam, loss = 'mse') return model
def build(self, hidden_layers=[32], n_filters=[8, 8], activations=[None], dropout=0.8, learning_rate=0.1, l2_norm=5e-4, use_bias=False, k=8): with self.device: x = Input(batch_shape=[None, self.n_features], dtype=tf.float32, name='features') adj = Input(batch_shape=[None, None], dtype=tf.float32, sparse=False, name='adj_matrix') mask = Input(batch_shape=[None], dtype=tf.bool, name='mask') h = x for hid, activation in zip(hidden_layers, activations): h = Dropout(rate=dropout)(h) h = DenseGraphConv( hid, use_bias=use_bias, activation=activation, kernel_regularizer=regularizers.l2(l2_norm))([h, adj]) for n_filter in n_filters: top_k_h = Top_k_features(k=k)([h, adj]) cur_h = LGConvolution( n_filter, k, use_bias=use_bias, dropout=dropout, activation=None, kernel_regularizer=regularizers.l2(l2_norm))(top_k_h) cur_h = BatchNormalization()(cur_h) h = Concatenate()([h, cur_h]) h = Dropout(rate=dropout)(h) h = DenseGraphConv( self.n_classes, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))([h, adj]) h = tf.boolean_mask(h, mask) output = Softmax()(h) model = Model(inputs=[x, adj, mask], outputs=output) model.compile(loss='sparse_categorical_crossentropy', optimizer=Nadam(lr=learning_rate), metrics=['accuracy']) self.k = k self.model = model self.built = True
def get_Nadam(): if self.config.Adam_eps == "default": Adam_eps = 1e-7 elif type(self.config.Adam_eps) == float: Adam_eps = self.config.Adam_eps else: raise Exception('Unsupported Adam_eps!! --> is not default/float') return Nadam(learning_rate=self.config.learning_rate, epsilon=Adam_eps)
def build(self, hiddens=[32], n_filters=[8, 8], activations=[None, None], dropout=0.8, l2_norm=5e-4, lr=0.1, use_bias=False, k=8): with tf.device(self.device): x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=self.floatx, sparse=False, name='adj_matrix') mask = Input(batch_shape=[None], dtype=tf.bool, name='node_mask') h = x for idx, hidden in enumerate(hiddens): h = Dropout(rate=dropout)(h) h = DenseConvolution( hidden, use_bias=use_bias, activation=activations[idx], kernel_regularizer=regularizers.l2(l2_norm))([h, adj]) for idx, n_filter in enumerate(n_filters): top_k_h = Top_k_features(k=k)([h, adj]) cur_h = LGConvolution( n_filter, kernel_size=k, use_bias=use_bias, dropout=dropout, activation=activations[idx], kernel_regularizer=regularizers.l2(l2_norm))(top_k_h) cur_h = BatchNormalization()(cur_h) h = Concatenate()([h, cur_h]) h = Dropout(rate=dropout)(h) h = DenseConvolution( self.graph.n_classes, use_bias=use_bias, activation=activations[-1], kernel_regularizer=regularizers.l2(l2_norm))([h, adj]) h = Mask()([h, mask]) model = Model(inputs=[x, adj, mask], outputs=h) model.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Nadam(lr=lr), metrics=['accuracy']) self.k = k self.model = model
def main(): # Safety checks if not os.path.exists(opt.ckptDir): os.makedirs(opt.ckptDir) if not os.path.exists(opt.logs): os.makedirs(opt.logs) logger.info('Building model...') model = WDSRConv3D(scale=3, numFilters=32, kernelSize=(3, 3, 3), numResBlocks=8, expRate=8, decayRate=0.8, numImgLR=9, patchSizeLR=32, isGrayScale=True) if opt.optimizer == 'adam': optimizer = Adam(learning_rate=5e-4) elif opt.optimizer == 'nadam': # http://cs229.stanford.edu/proj2015/054_report.pdf optimizer = Nadam(learning_rate=5e-4) else: optimizer = SGD(learning_rate=5e-4) checkpoint = tf.train.Checkpoint(step=tf.Variable(0), psnr=tf.Variable(1.0), optimizer=optimizer, model=model) checkpointManager = tf.train.CheckpointManager(checkpoint=checkpoint, directory=opt.ckptDir, max_to_keep=5) # Load Data logger.info('Loading data...') patchLR = np.load(opt.patchLR, allow_pickle=True) patchHR = np.load(opt.patchHR, allow_pickle=True) X_train, X_val, y_train, y_val = train_test_split(patchLR, patchHR, test_size=opt.split, random_state=17) valData = [X_val, y_val] # Initialize metrics trainLoss = Mean(name='trainLoss') trainPSNR = Mean(name='trainPSNR') testLoss = Mean(name='testLoss') testPSNR = Mean(name='testPSNR') fitTrainData(model, optimizer, [trainLoss, trainPSNR, testLoss, testPSNR], shiftCompensatedL1Loss, shiftCompensatedcPSNR, X_train, y_train, opt.batchSize, opt.epochs, opt.dataBufferSize, valData, opt.valSteps, checkpoint, checkpointManager, opt.logDir, opt.ckptDir, opt.saveBestOnly)
def create_model(self): """Creates the regressor network. Returns: A Keras model for the regressor network architecture. """ # Convolutional Encoder input_img = Input(shape=(*self.dims, self.channels)) conv_1 = Conv2D(32, (3, 3), activation='relu')(input_img) pool_1 = MaxPooling2D((2, 2))(conv_1) conv_2 = Conv2D(64, (3, 3), activation='relu')(pool_1) pool_2 = MaxPooling2D((2, 2), strides=(2, 2))(conv_2) conv_3 = Conv2D(32, (3, 3), activation='relu')(pool_2) pool_3 = MaxPooling2D((2, 2))(conv_3) conv_4 = Conv2D(16, (3, 3), activation='relu')(pool_3) pool_4 = MaxPooling2D((2, 2))(conv_4) flatten = Flatten()(pool_4) # Deep feed-forward network dense_1_d = Dense(units=300, activation='relu', kernel_initializer='he_normal')(flatten) dropout_1_d = Dropout(self.dropout)(dense_1_d) dense_2_d = Dense(units=192, activation='relu', kernel_initializer='he_normal')(dropout_1_d) dropout_2_d = Dropout(self.dropout)(dense_2_d) dense_3_d = Dense(units=123, activation='relu', kernel_initializer='he_normal')(dropout_2_d) dropout_3_d = Dropout(self.dropout)(dense_3_d) dense_4_d = Dense(units=79, activation='relu', kernel_initializer='he_normal')(dropout_3_d) dropout_4_d = Dropout(self.dropout)(dense_4_d) dense_5_d = Dense(units=50, activation='relu', kernel_initializer='he_normal')(dropout_4_d) dropout_5_d = Dropout(self.dropout)(dense_5_d) depth_linear = Dense(units=self.outputs, activation='linear', name='depth')(dropout_5_d) sld_linear = Dense(units=self.outputs, activation='linear', name='sld')(dropout_5_d) model = Model(inputs=input_img, outputs=[depth_linear, sld_linear]) model.compile( loss={ 'depth': 'mse', 'sld': 'mse' }, #msle was tried here for depth. optimizer=Nadam(self.learning_rate), metrics={ 'depth': 'mae', 'sld': 'mae' }) return model
def _build_vae(self): ''' builds variational autoencoder network ''' self.kl_anneal = Input(batch_shape=(self.batch_size, ), name='kl_anneal') # build VAE if np.all(np.array([self.alpha, self.beta, self.lamb]) == 0): self.x_output = self.decoder(self.encoder(self.enc_x_input)) self.vae = Model(inputs=[self.enc_x_input], outputs=[self.x_output], name='variational_autoencoder') elif self.alpha == self.beta == self.lamb: self.x_output = self.decoder(self.encoder(self.enc_x_input)[2]) self.vae = Model(inputs=[self.enc_x_input, self.kl_anneal], outputs=[self.x_output], name='variational_autoencoder') tc_loss = self.kl_anneal * self.kullback_leibler_divergence_loss() self.vae.add_loss(tc_loss) self.vae.add_metric(tc_loss, name='tc_loss', aggregation='mean') elif np.any(np.array([self.alpha, self.beta, self.lamb]) > 0): self.x_output = self.decoder(self.encoder(self.enc_x_input)[2]) self.vae = Model(inputs=[self.enc_x_input, self.kl_anneal], outputs=[self.x_output], name='variational_autoencoder') tc_loss = self.kl_anneal * self.total_correlation_loss() self.vae.add_loss(tc_loss) self.vae.add_metric(tc_loss, name='tc_loss', aggregation='mean') # define VAE optimizer if self.vae_opt_n == 'sgd': self.vae_opt = SGD(learning_rate=self.lr) elif self.vae_opt_n == 'sgdm': self.vae_opt = SGD(learning_rate=self.lr, momentum=0.5) elif self.vae_opt_n == 'nsgd': self.vae_opt = SGD(learning_rate=self.lr, momentum=0.5, nesterov=True) elif self.vae_opt_n == 'rmsprop': self.vae_opt = RMSprop(learning_rate=self.lr) elif self.vae_opt_n == 'rmsprop_cent': self.vae_opt = RMSprop(learning_rate=self.lr, centered=True) elif self.vae_opt_n == 'adam': self.vae_opt = Adam(learning_rate=self.lr, beta_1=0.5) elif self.vae_opt_n == 'adam_ams': self.vae_opt = Adam(learning_rate=self.lr, beta_1=0.5, amsgrad=True) elif self.vae_opt_n == 'adamax': self.vae_opt = Adamax(learning_rate=self.lr, beta_1=0.5) elif self.vae_opt_n == 'adamax_ams': self.vae_opt = Adamax(learning_rate=self.lr, beta_1=0.5, amsgrad=True) elif self.vae_opt_n == 'nadam': self.vae_opt = Nadam(learning_rate=self.lr, beta_1=0.5) # compile VAE rc_loss = self.reconstruction_loss() self.vae.add_loss(rc_loss) self.vae.add_metric(rc_loss, name='rc_loss', aggregation='mean') self.vae.compile(optimizer=self.vae_opt)
def build_network(self): model = Sequential() model.add(Dense(24, input_shape=self.observation_space().shape, activation='relu')) model.add(Dense(24, activation='relu')) model.add(Dense(self.action_space().n, activation='linear')) opt = Nadam(learning_rate=self.alpha) model.compile(loss='mse', optimizer=opt) return model
def _init_model(self, hidden_layer_units): input_ = Input(shape=(self.config.input_shape, )) y = Dense(units=hidden_layer_units, activation='relu')(input_) y = Dense(units=self.config.output_shape, activation='sigmoid')(y) model = Model(input_, y) model.compile(optimizer=Nadam(), loss='categorical_crossentropy', metrics=['categorical_accuracy']) return model
def main(): list_all_midi = get_list_midi() sampled_200_midi = list_all_midi[0:100] # changed to 100 batch = 1 start_index = 0 note_tokenizer = NoteTokenizer() seq_len = 50 EPOCHS = 4 BATCH_SONG = 16 BATCH_NNET_SIZE = 96 TOTAL_SONGS = len(sampled_200_midi) FRAME_PER_SECOND = 5 for i in tqdm(range(len(sampled_200_midi))): dict_time_notes = generate_dict_time_notes(sampled_200_midi, batch_song=1, start_index=i, use_tqdm=False, fs=5) full_notes = process_notes_in_song(dict_time_notes) for note in full_notes: note_tokenizer.partial_fit(list(note.values())) note_tokenizer.add_new_note('e') # Add empty notes unique_notes = note_tokenizer.unique_word # print(unique_notes) model = create_model(seq_len, unique_notes) print(model.summary()) tf.keras.utils.plot_model(model, 'this_model.png', show_shapes=True) # TRAIN optimizer = Nadam() checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model) checkpoint_dir = '../../training_checkpoints' checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") loss_fn = sparse_categorical_crossentropy # TRAIN HERE (1 epoch = 1hour) train_class = TrainModel(EPOCHS, note_tokenizer, sampled_200_midi, FRAME_PER_SECOND, BATCH_NNET_SIZE, BATCH_SONG, optimizer, checkpoint, loss_fn, checkpoint_prefix, TOTAL_SONGS, model) train_class.train() # save mpdel as h5 and the tokenizer model.save('data/model/epochs4-long-model_ep4.h5') pickle.dump(note_tokenizer, open("epochs4-long-tokenizer.p", "wb"))