((trainX, trainY), (testX, testY)) = cifar10.load_data() # scale the data to range [0, 1] trainX = trainX.astype("float") / 255.0 testX = testX.astype("float") / 255.0 # convert labels from integers to vectors lb = LabelBinarizer() trainY = lb.fit_transform(trainY) testY = lb.fit_transform(testY) # init the optimizer print("[INFO] compiling model") # define callbacks to be passed to model callbacks = [LearningRateScheduler(step_decay)] # SGD optimizer with Nesterov momentum sgd = SGD(lr=0.01, momentum=0.9, nesterov=True) # sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) # simple SGD optimizer with learning rate of 0.01 # sgd = SGD(lr=0.01) # compile the network model = MiniVGGNet.build(width=32, height=32, depth=3, classes=10) model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["accuracy"]) # train the network print("[INFO] training network") epochs = 200 batch_size = 64 H = model.fit(x=trainX, y=trainY, validation_data=(testX, testY), callbacks=callbacks, batch_size=batch_size, epochs=epochs, verbose=1)
def get_optimizer(config): lr = config['optimizer']['initial_lr'] optimizer = SGD(lr=lr) # Using Adam instead of SGD to speed up training return optimizer
x_train = sc.fit_transform(X) x_test = sc.fit_transform(XT) from keras.models import Sequential from keras.layers import Dense from keras.layers import Dense, Dropout, Activation from keras.optimizers import SGD model = Sequential() model.add(Dense(64, activation='relu', input_dim=559)) model.add(Dropout(0.5)) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(6, activation='softmax')) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) model.fit(x_train, y_train, epochs=20, batch_size=128) score = model.evaluate(x_test, y_test, batch_size=128) y_pred = model.predict(x_test) y_pred = (y_pred > 0.5)
s2 = l6(s2) s2 = GlobalAveragePooling1D()(s2) merge_text = multiply([s1, s2]) x = Dense(100, activation='linear')(merge_text) x = keras.layers.LeakyReLU(alpha=0.3)(x) x = Dense(int((hidden_dim + 7) / 2), activation='linear')(x) x = keras.layers.LeakyReLU(alpha=0.3)(x) main_output = Dense(1, activation='sigmoid')(x) merge_model = Model(inputs=[seq_input1, seq_input2], outputs=[main_output]) return merge_model hidden_dim = 25 n_epochs = 50 batch_size = 256 sgd = SGD(lr=0.001, momentum=0.0, nesterov=False) #class label train, validate, test split class_lbl = binary.iloc[:, 9].as_matrix() pos = np.where(class_lbl == 1) neg = np.where(class_lbl == 0) import random random.seed(4) np.random.shuffle(pos[0]) np.random.shuffle(neg[0]) #train, test indeces pos = np.array((pos[0][0:125000], pos[0][125000:]))
) model.add(Dense(1024, W_regularizer=l2(L2), activity_regularizer=activity_l2(L2) ) ) model.add(Dense(10)) model.add(Activation('softmax')) # Print the model summary to terminal model.summary() # Compile the model # Need to be categorical because this is not a binary classification problem # Compile stochastic gradient descent model with step decreasing learning rate sgd = SGD(lr=0.001, momentum=MOMENTUM, decay=0.0, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # Add a learning rate schedule # References # --------- # https://goo.gl/4vQhdj # https://goo.gl/VrrciJ def step_decay(epoch): """ Define a step decay, which takes the form lr = lr0 * drop^floor(epoch/epocs_drop) """
def build_models(self): lr = 4e-4 clip = 1.0 decay = 1e-8 # Initialize architectures self.disc = self.init_discriminator() self.gen = self.init_generator() if self.weights_path is not None: self.gen.load_weights(self.weights_path + 'generator_weights.h5') self.disc.load_weights(self.weights_path + 'discriminator_weights.h5') # Create the model if self.num_gpus > 1: self.disc_model = multi_gpu_model(self.disc, gpus=self.num_gpus) else: self.disc_model = self.disc # Compile Discriminator Model # doptimizer = RMSprop(lr=lr, decay=decay, clipvalue=clip) self.disc_model.compile(loss='mse', optimizer=SGD(lr=lr, nesterov=True, clipvalue=clip), metrics=['accuracy']) #self.disc_model = Sequential() #self.disc_model.add(self.disc) #self.disc_model.compile(loss='binary_crossentropy', # optimizer = doptimizer, # metrics=['accuracy']) # Compile Adversarial Model goptimizer = RMSprop(lr=lr / 2, decay=decay, clipvalue=clip) self.disc.trainable = False im_lr = Input(shape=(self.imlr_w, self.imlr_h, self.im_c)) im_hr = Input(shape=(self.imhr_w, self.imhr_h, self.im_c)) # Generated HR Images gen_hr = self.gen(im_lr) # Discriminator on Generator Output disc_gen_hr = self.disc(gen_hr) self.adv_model = Model(im_lr, disc_gen_hr) # Create the model if self.num_gpus > 1: self.adv_model = multi_gpu_model(Model(im_lr, disc_gen_hr), gpus=self.num_gpus) else: self.adv_model = Model(im_lr, disc_gen_hr) self.adv_model.compile( loss=['binary_crossentropy'], #loss_weights=[1e-3,1], optimizer=Adam(clipvalue=clip), metrics=['accuracy'])
def main(): #o_path = "/Users/ra-mit/development/fabric/uns/sim/" o_path = "/data/eval/wiki/" from utils import prepare_sqa_data #data = prepare_sqa_data.get_sqa(filter_stopwords=True) if not fb and not wiki: spos = prepare_sqa_data.get_spo_from_rel(filter_stopwords=True) uns_spos, loc_dic = prepare_sqa_data.get_spo_from_uns() spos = spos + uns_spos if wiki: # structured_path = "/Users/ra-mit/data/fabric/dbpedia/triples_structured/all.csv" structured_path = "/data/smalldatasets/wiki/all.csv" # unstructured_path = "/Users/ra-mit/data/fabric/dbpedia/triples_unstructured/" unstructured_path = "/data/smalldatasets/wiki/triples_unstructured/" spos = [] df = pd.read_csv(structured_path, encoding='latin1') ss = list(df.iloc[:, 0]) ps = df.iloc[:, 1] os = df.iloc[:, 2] for s, p, o in zip(ss, ps, os): spos.append((s, p, o)) print("Total structured spos: " + str(len(spos))) # uns_files = csv_access.list_files_in_directory(unstructured_path) uns_spos, loc_dic = prepare_sqa_data.get_spo_from_uns(path=unstructured_path) # uns_spos = [] # for f in uns_files: # df = pd.read_csv(f, encoding='latin1') # ss = list(df.iloc[:, 0]) # ps = df.iloc[:, 1] # os = df.iloc[:, 2] # for s, p, o in zip(ss, ps, os): # uns_spos.append((s, p, o)) print("Total unstructured spos: " + str(len(uns_spos))) spos += uns_spos print("Total: " + str(len(spos))) true_pairs = [] S = [] P = [] O = [] # positive pairs for s, p, o in spos: s = str(s) p = str(p) o = str(o) true_pairs.append((s, p, 0)) true_pairs.append((s, o, 0)) true_pairs.append((p, o, 0)) S.append(s) P.append(p) O.append(o) if not fb and not wiki: with open(o_path + "true_pairs.pkl", "wb") as f: pickle.dump(true_pairs, f) print("True pairs: " + str(len(true_pairs))) # set to avoid negative samples that collide with positive ones pos = set() for e1, e2, label in true_pairs: pos.add(e1 + e2) print("Unique true pairs: " + str(len(pos))) # negative pairs random_permutation = np.random.permutation(len(S)) S = np.asarray(S) S = S[random_permutation] random_permutation = np.random.permutation(len(O)) O = np.asarray(O) O = O[random_permutation] false_pairs = [] for s, p, o in zip(list(S), P, list(O)): if s + p in pos or s + o in pos or p + o in pos: continue # this is probably colliding with pos, so we do not include false_pairs.append((s, p, 1)) false_pairs.append((s, o, 1)) false_pairs.append((p, o, 1)) print("Negative pairs 1: " + str(len(false_pairs))) random_permutation = np.random.permutation(len(S)) S = np.asarray(S) S = S[random_permutation] random_permutation = np.random.permutation(len(O)) O = np.asarray(O) O = O[random_permutation] false_pairs2 = [] for s, p, o in zip(list(S), P, list(O)): if s + p in pos or s + o in pos or p + o in pos: continue # this is probably colliding with pos, so we do not include false_pairs2.append((s, p, 1)) false_pairs2.append((s, o, 1)) false_pairs2.append((p, o, 1)) print("Negative pairs 2: " + str(len(false_pairs2))) all_data = true_pairs + false_pairs + false_pairs2 sparsity_code_size = 48 if fb: sparsity_code_size = 4 # 1 word per clause o_path = "/data/eval/fb/" all_data, true_pairs = process_fb.extract_data() # start counting vals #_test = all_data[:2000] # test #total = 0 #for s, p, label in _test: # total += label #print("total: " + str(total/len(all_data))) # end counting vals random_permutation = np.random.permutation(len(all_data)) all_data = np.asarray(all_data) all_data = all_data[random_permutation] with open(o_path + "true_pairs.pkl", "wb") as f: pickle.dump(true_pairs, f) #all_data = all_data[:2000] # test #total = 0 #for s, p, label in all_data: # total += label #print("total: " + str(total/len(all_data))) if wiki: sparsity_code_size = 48 o_path = "/data/eval/wiki/" random_permutation = np.random.permutation(len(all_data)) all_data = np.asarray(all_data) all_data = all_data[random_permutation] with open(o_path + "true_pairs.pkl", "wb") as f: pickle.dump(true_pairs, f) vocab = dict() if not fb: idx_vectorizer = IndexVectorizer(vocab_index=vocab, sparsity_code_size=sparsity_code_size, tokenizer_sep=" ") else: idx_vectorizer = FlatIndexVectorizer(vocab_index=vocab, sparsity_code_size=sparsity_code_size) vectorizer = tp.CustomVectorizer(idx_vectorizer) st = time.time() print("start vectorizing...") # vectorization happens here X1 = [] X2 = [] Y = [] for e1, e2, label in all_data: ve1 = vectorizer.get_vector_for_tuple(e1) ve1 = ve1.toarray()[0] ve2 = vectorizer.get_vector_for_tuple(e2) ve2 = ve2.toarray()[0] X1.append(ve1) X2.append(ve2) Y.append(label) X1 = np.asarray(X1) X2 = np.asarray(X2) Y = np.asarray(Y) et = time.time() print("finish vectorizing...") print("took: " + str(et-st)) vocab, inv_vocab = vectorizer.get_vocab_dictionaries() print("vocab size: " + str(len(vocab))) # def model1(): input_dim = sparsity_code_size * 32 # declare network i1 = Input(shape=(input_dim,), name="i1") i2 = Input(shape=(input_dim,), name="i2") base = Sequential() base.add(Dense(1024, input_shape=(input_dim,), activation='relu')) #base.add(Dense(2056, input_shape=(input_dim,), activation='relu')) #base.add(Dense(512, input_shape=(input_dim,), activation='relu')) #base.add(Dense(2056, activation='relu')) #base.add(Dense(768, activation='relu')) base.add(Dense(512, activation='relu')) #base.add(Dense(1024, activation='relu')) base.add(Dense(256, activation='relu')) base.add(Dense(128, activation='relu')) #base.add(Dense(64, activation='relu')) emb_1 = base(i1) emb_2 = base(i2) def euclidean_distance(vects): x, y = vects return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon())) def eucl_dist_output_shape(shapes): shape1, shape2 = shapes return shape1[0], 1 def contrastive_loss(y_true, y_pred): margin = 1 # Y=0 means similar and Y=1 means dissimilar. Think of it as distance return K.mean((1 - y_true) * K.square(y_pred) + y_true * K.square(K.maximum(margin - y_pred, 0))) distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([emb_1, emb_2]) fullmodel = Model(input=[i1, i2], output=distance) opt = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True) callbacks = [] callback_best_model = keras.callbacks.ModelCheckpoint(o_path + "epoch-{epoch}.h5", monitor='val_loss', save_best_only=False) callbacks.append(callback_best_model) fullmodel.compile(optimizer=opt, loss=contrastive_loss, metrics=['accuracy']) fullmodel.summary() def size(model): # Compute number of params in a model (the actual number of floats) return sum([np.prod(K.get_value(w).shape) for w in model.trainable_weights]) print("trainable params: " + str(size(fullmodel))) fullmodel.fit([X1, X2], Y, epochs=300, shuffle=True, batch_size=80, callbacks=callbacks) encoder = Model(input=i1, output=emb_1) fullmodel.save(o_path + "/sim.h5") encoder.save(o_path + "/sim_encoder.h5") with open(o_path + "tf_dictionary.pkl", "wb") as f: pickle.dump(vocab, f)
y = DenseTarget2D(x, growth_rate=growth_rate, include_target='true', l2=l2) x = Concatenate(axis=-1)([x, y]) l2 *= l2_buildup y = DenseTarget2D(x, growth_rate=growth_rate, include_target='true', l2=l2) x = Concatenate(axis=-1)([x, y]) x = BatchNormalization()(x) x = Activation('relu')(x) x = GlobalAveragePooling2D()(x) out = Dense(num_classes, activation='softmax')(x) model = Model(input, out) # Optimizer #adam = Adam(lr= 0.001, beta_1= 0.9, beta_2= 0.999, epsilon= 1e-08, decay= 0.0) sgd = SGD(lr=0.1, momentum=0.9, decay=0.0001, nesterov=True) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=sgd, metrics=['accuracy']) # check the model model.summary() lr_decay = LearningRateScheduler(scheduler) csv_logger = CSVLogger('MNIST_train.log') checkpoint = ModelCheckpoint(filepath='MNIST_weights.hdf5', monitor='val_acc', verbose=0, save_best_only=True,
json_file.close() model = model_from_json(loaded_model_json) print('[INFO] >> Model Loaded JSON File from the Disk') # Populate Weights into the JSON Model: model.load_weights( '/home/hassan/usb_drive/gideon/Activity_Accidents_16m_04032020_weights_ucf_accidents_16mp.h5' ) print('[INFO] >> Model Loaded Successfully from the Disk') # Model Summary: model.summary() # Model Compilation: le = 0.001 opt = SGD(lr=le, momentum=0.009, decay=le) model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy']) print('[INFO] >> Model Compiled Successfully !\n\n') #print('********************** Model Weights (Format 1)**********************') #model_weights = model.get_weights() #print(model_weights) print('********************** Model Configuration **********************') total_layers = 0 for layer in model.layers: layer_weights = layer.get_weights() total_layers += 1 print('Layer Name:', layer.name) print('Layer Configuration:', layer.get_config()) print('Layer Input Shape:', layer.input_shape)
color_mode='rgb', batch_size=32, shuffle=False, class_mode='categorical') # inicializar e otimizar modelo print("[INFO] Inicializando e otimizando a CNN...") start = time.time() early_stopping_monitor = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15) model = Convolucao.build(64, 64, 3, CLASS) model.compile(optimizer=SGD(0.01), loss="categorical_crossentropy", metrics=["accuracy"]) # treinar a CNN print("[INFO] Treinando a CNN...") classifier = model.fit_generator( training_set, steps_per_epoch=(training_set.n // training_set.batch_size), epochs=EPOCHS, validation_data=test_set, validation_steps=(test_set.n // test_set.batch_size), shuffle=False, verbose=2, callbacks=[early_stopping_monitor])
# pool1 # model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # conv2 model.add(Conv2D(50, kernel_size=(5, 5), padding="same", activation='relu')) # pool2 # model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # model.add(Dropout(0.25)) model.add(Flatten()) # ip1 model.add(Dense(500, activation='relu', name='fc1')) # model.add(Dropout(0.5)) # ip2 model.add(Dense(num_classes, activation='softmax', name='predictions')) # optimizer = keras.optimizers.Adadelta() optimizer = SGD(lr=1e-4, decay=1e-6, momentum=0.5, nesterov=True) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) from data_mnist import DataSet def get_generators(): data = DataSet( seq_length=seq_length, class_limit=class_limit, image_shape=image_shape, )
def train(): data_out_dir = '/media/tk/EE44DA8044DA4B4B/cataract_phase_img' height = 224 width = 224 skip_rate = 10 batch = 32 nb_classes = 11 nb_epoch = 100 current_batch_count = 1 out_dir_name = 'ResNet50Pretrain_phase' ## CHECK THIS!!!!!!!!! activation = "relu" ## CHECK THIS!!!!!!!!! momentum = 0.9 lr = 0.01 optimizer = SGD(lr=lr, momentum=momentum, decay=0.0, nesterov=True) ## CHECK THIS!!!!!!!!! loss = 'categorical_crossentropy' model = Models.resnet(nb_classes) model.compile(optimizer, loss=loss, metrics=['accuracy']) X = np.zeros((batch,height,width,3)) Y = np.zeros((batch,nb_classes)) for e in range(0,nb_epoch): ACC = 0. LOSS = 0. N = 0 for vid_num in sample_lengths.keys(): lmdb_env_x = lmdb.open(os.path.join(data_out_dir,vid_num+"X")) lmdb_txn_x = lmdb_env_x.begin() lmdb_cursor_x = lmdb_txn_x.cursor() lmdb_env_y = lmdb.open(os.path.join(data_out_dir,vid_num+"y")) lmdb_txn_y = lmdb_env_y.begin() lmdb_cursor_y = lmdb_txn_y.cursor() indices = list(range(0,int(sample_lengths[vid_num]/skip_rate))) np.random.shuffle(indices) label = np.frombuffer(lmdb_cursor_y.get('{:0>8d}'.format(0).encode()),dtype=np.dtype(np.int64)) for index in indices: real_frame_ind = index*skip_rate try: value = np.frombuffer(lmdb_cursor_x.get('{:0>8d}'.format(index).encode()),dtype=np.dtype(np.uint8)) except: continue #pdb.set_trace() x = value.reshape((height,width,3)) x.setflags(write=1) x = x.astype(np.float) x -= 128 x /= 128.0 y = label[real_frame_ind] X[current_batch_count] = x Y[current_batch_count,y] = 1 current_batch_count += 1 if (current_batch_count % batch) == 0: losses = model.train_on_batch(X, Y) ACC += losses[1] # current accuracy distinguishing real-vs-fake LOSS += losses[0] N += 1 print("epoch: {:03d} | loss: {:.03f} | acc: {:.03f} \r".format(e,LOSS/N,ACC/N), end='\r') ## TRAIN() X = np.zeros((batch,height,width,3)) Y = np.zeros((batch,nb_classes)) current_batch_count = 0 print("Finished with epoch:", e,"\n") model_file = './weights/'+ out_dir_name + '_ep:%03d_acc:%0.3f_loss:%0.3f.h5' % (e+1,(LOSS/N),(ACC/N)) model.save_weights(model_file, overwrite=True)
def __init__(self): # Variables to hold the description of the experiment self.config_description = "This is the template config file." # System dependent variable self._workers = 5 self._multiprocessing = True # Variables for comet.ml self._project_name = "jpeg_deep" self._workspace = "ssd" # Network variables self._weights = "/dlocal/home/2017018/bdegue01/weights/jpeg_deep/classification_dct/resnet_deconv/classification_dct_jpeg-deep_GYftBmXMdjdxoMksyI3e9VqB5IriBC9T/checkpoints/epoch-87_loss-0.7459_val_loss-1.5599.h5" self._network = SSD300_resnet( backbone="deconv_rfa", dct=True, image_shape=(38, 38)) # Training variables self._epochs = 240 self._batch_size = 32 self._steps_per_epoch = 1000 self.optimizer_parameters = { "lr": 0.001, "momentum": 0.9} self._optimizer = SGD(**self.optimizer_parameters) self._loss = SSDLoss(neg_pos_ratio=3, alpha=1.0).compute_loss self._metrics = None dataset_path = environ["DATASET_PATH"] images_2007_path = join(dataset_path, "VOC2007/JPEGImages") self.train_sets = [(images_2007_path, join( dataset_path, "VOC2007/ImageSets/Main/train.txt"))] self.validation_sets = [(images_2007_path, join( dataset_path, "VOC2007/ImageSets/Main/val.txt"))] self.test_sets = [(images_2007_path, join( dataset_path, "VOC2007/ImageSets/Main/test.txt"))] # Keras stuff self.model_checkpoint = None self.reduce_lr_on_plateau = ReduceLROnPlateau(patience=5, verbose=1) self.terminate_on_nan = TerminateOnNaN() self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=15) self._callbacks = [self.reduce_lr_on_plateau, self.early_stopping, self.terminate_on_nan] self.input_encoder = SSDInputEncoder() self.train_transformations = [SSDDataAugmentation()] self.validation_transformations = [ ConvertTo3Channels(), Resize(height=300, width=300)] self.test_transformations = [ConvertTo3Channels(), Resize( height=300, width=300)] self._train_generator = None self._validation_generator = None self._test_generator = None self._horovod = None self._displayer = DisplayerObjects()
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ['CUDA_VISIBLE_DEVICES'] = '3' os.environ["TF_CPP_MIN_LOG_LEVEL"] = '3' import tensorflow as tf import tensorflow_datasets as tfds import numpy as np from keras import layers from keras.applications import VGG16 from keras.optimizers import SGD from keras import Sequential from keras import layers from keras.models import load_model from keras.backend import clear_session #Optimizer sgd_optimizer = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True) feature_map_per_packet = 8 #Loading Datas datas = np.load("../datasets/testing_features.npy") label = np.load("../datasets/testing_label.npy") print("Test data is ready") #Loading Original Model original_back_layers = load_model( "../original_models/original_back_layers_model.h5") original_back_layers.compile(optimizer=sgd_optimizer, loss='categorical_crossentropy', metrics=['accuracy']) print("Original Model is ready")
def run_cifar10(batch_size, nb_epoch, depth, nb_dense_block, nb_filter, growth_rate, dropout_rate, learning_rate, weight_decay, logfile, plot_architecture): """ Run CIFAR10 experiments :param batch_size: int -- batch size :param nb_epoch: int -- number of training epochs :param depth: int -- network depth :param nb_dense_block: int -- number of dense blocks :param nb_filter: int -- initial number of conv filter :param growth_rate: int -- number of new filters added by conv layers :param dropout_rate: float -- dropout rate :param learning_rate: float -- learning rate :param weight_decay: float -- weight decay :param plot_architecture: bool -- whether to plot network architecture """ ################### # Data processing # ################### # the data, shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = cifar10.load_data() nb_classes = len(np.unique(y_train)) img_dim = X_train.shape[1:] if K.image_data_format() == "channels_first": n_channels = X_train.shape[1] else: n_channels = X_train.shape[-1] # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) X_train = X_train.astype('float32') X_test = X_test.astype('float32') # Normalisation X = np.vstack((X_train, X_test)) # 2 cases depending on the image ordering if K.image_data_format() == "channels_first": for i in range(n_channels): mean = np.mean(X[:, i, :, :]) std = np.std(X[:, i, :, :]) X_train[:, i, :, :] = (X_train[:, i, :, :] - mean) / std X_test[:, i, :, :] = (X_test[:, i, :, :] - mean) / std elif K.image_data_format() == "channels_last": for i in range(n_channels): mean = np.mean(X[:, :, :, i]) std = np.std(X[:, :, :, i]) X_train[:, :, :, i] = (X_train[:, :, :, i] - mean) / std X_test[:, :, :, i] = (X_test[:, :, :, i] - mean) / std ################### # Construct model # ################### model = densenet.DenseNet(nb_classes, img_dim, depth, nb_dense_block, growth_rate, nb_filter, dropout_rate=dropout_rate, weight_decay=weight_decay) # Model output model.summary() # Build optimizer # opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) opt = SGD(lr=learning_rate, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=["accuracy"]) if plot_architecture: from keras.utils.visualize_util import plot plot(model, to_file='./figures/densenet_archi.png', show_shapes=True) #################### # Network training # #################### print("Training") list_train_loss = [] list_test_loss = [] list_learning_rate = [] datagen = ImageDataGenerator() for e in range(nb_epoch): if e == int(0.5 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.)) if e == int(0.75 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.)) l_train_loss = [] start = time.time() model.fit_generator(datagen.flow(X_train, Y_train, batch_size), epochs=1) test_logloss, test_acc = model.evaluate(X_test, Y_test, verbose=1, batch_size=64) list_test_loss.append([test_logloss, test_acc]) list_learning_rate.append(float(K.get_value(model.optimizer.lr))) # to convert numpy array to json serializable print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start)) d_log = {} d_log["batch_size"] = batch_size d_log["nb_epoch"] = nb_epoch d_log["optimizer"] = opt.get_config() # d_log["train_loss"] = list_train_loss d_log["test_loss"] = list_test_loss d_log["learning_rate"] = list_learning_rate json_file = os.path.join('./log', logfile) with open(json_file, 'w') as fp: json.dump(d_log, fp, indent=4, sort_keys=True)
X2 = np.random.randn(n, M) + np.array([5, 5]) X3 = np.random.randn(n, M) + np.array([10, 0]) Y1 = np.array([[1, 0, 0] for i in range(n)]) Y2 = np.array([[0, 1, 0] for i in range(n)]) Y3 = np.array([[0, 0, 1] for i in range(n)]) X = np.concatenate((X1, X2, X3), axis=0) Y = np.concatenate((Y1, Y2, Y3), axis=0) """ モデル定義(rogistic:sigmoid->multiclass:softmax) """ model = Sequential() model.add(Dense(input_dim=M, units=K)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.1)) """ モデル学習 """ minibatch_size = 50 model.fit(X, Y, epochs=20, batch_size=minibatch_size) X_, Y_ = shuffle(X, Y) classes = model.predict_classes(X_[0:10], batch_size=minibatch_size) prob = model.predict_proba(X_[0:10], batch_size=1) print('classified:') print(np.argmax(model.predict(X_[0:10]), axis=1) == classes) print() print('output probability:')
# Architecture arch = config['architecture']['architecture'] kernel_num = config['architecture'].getint('kernel_num') create_network = load_network(arch) # Optimizer opt_name = config['optimizer']['optimizer'] learning_rate = config['optimizer'].getfloat('learning_rate') clipnorm = config['optimizer'].getfloat('clipnorm') momentum = config['optimizer'].getfloat('momentum') if opt_name == 'adam': from keras.optimizers import Adam opt = Adam(lr=learning_rate, clipnorm=clipnorm) elif opt_name == 'sgd': from keras.optimizers import SGD opt = SGD(lr=learning_rate, clipnorm=clipnorm, momentum=momentum) else: raise ValueError('{} is not a valid optimizer'.format(opt_name)) # Regularizer L1_REGULARIZER = config['regularizer'].getfloat('l1_penalty') L2_REGULARIZER = config['regularizer'].getfloat('l2_penalty') DROPOUT_RATE = config['regularizer'].getfloat('dropout_rate') # Callbacks callbacks = [] if config['callbacks'].getboolean('csv_logger'): csv_fname = config['callbacks']['csv_fname'] csv_logger = CSVLogger(csv_fname) callbacks.append(csv_logger)
def main(): # make sure soft-placement is off tf_config = tf.ConfigProto(allow_soft_placement=False) tf_config.gpu_options.allow_growth = True s = tf.Session(config=tf_config) K.set_session(s) # parameters model_name = "models/inception_kaufland.h5" num_classes = 12 size = (299, 299) in_size = 3600 bs = 12 steps_per_epoch = in_size // bs val_size = 1200 steps_val = val_size // bs cores = 8 pool = multiprocessing.Pool(processes=cores, initializer=init_worker) train_datagen = T.ImageDataGenerator( rescale=1./255, rotation_range=180, width_shift_range=.2, height_shift_range=.2, #brightness_range=(0.5, 1), shear_range=0.2, zoom_range=0.2, fill_mode="nearest", pool=pool) train_generator = train_datagen.flow_from_directory( 'data/train_set_xs', target_size=size, batch_size=bs, class_mode='categorical', #interpolation="bicubic" ) test_datagen = T.ImageDataGenerator( rescale=1./255, pool=pool ) test_generator = test_datagen.flow_from_directory( 'data/test_set', target_size=size, batch_size=bs, class_mode='categorical') # create the base pre-trained model base_model = InceptionV3(weights='imagenet', include_top=False) # add a global spatial average pooling layer x = base_model.output x = GlobalAveragePooling2D()(x) # let's add a fully-connected layer x = Dense(1024, activation='relu')(x) # and a logistic layer -- let's say we have `num_classes` classes predictions = Dense(num_classes, activation='softmax')(x) # this is the model we will train model = Model(inputs=base_model.input, outputs=predictions) # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional InceptionV3 layers for layer in base_model.layers: layer.trainable = False # compile the model (should be done *after* setting layers to non-trainable) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc']) # train the model on the new data for a few epochs model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=10, validation_data=test_generator, validation_steps=steps_val) # at this point, the top layers are well trained and we can start fine-tuning # convolutional layers from inception V3. We will freeze the bottom N layers # and train the remaining top layers. # let's visualize layer names and layer indices to see how many layers # we should freeze: #for i, layer in enumerate(base_model.layers): # print(i, layer.name) # we chose to train the top 2 inception blocks, i.e. we will freeze # the first 249 layers and unfreeze the rest: for layer in model.layers[:249]: layer.trainable = False for layer in model.layers[249:]: layer.trainable = True print("Fine tuning...") # we need to recompile the model for these modifications to take effect # we use SGD with a low learning rate from keras.optimizers import SGD model.compile( optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['acc'] ) # we train our model again (this time fine-tuning the top 2 inception blocks # alongside the top Dense layers model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=10, validation_data=test_generator, validation_steps=steps_val) model.save(os.path.join(os.getcwd(), model_name))
from __future__ import print_function import numpy as np from keras.datasets import mnist from keras.models import Sequential from keras.layers.core import Dense, Activation, Dropout from keras.optimizers import SGD from keras.utils import np_utils np.random.seed(1671) # for reproducibility # network and training NB_EPOCH = 20 BATCH_SIZE = 128 VERBOSE = 1 NB_CLASSES = 10 # number of outputs = number of digits OPTIMIZER = SGD() # SGD optimizer, explained later in this chapter N_HIDDEN = 128 VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION # data: shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() #X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 RESHAPED = 784 X_train = X_train.reshape(60000, RESHAPED) X_test = X_test.reshape(10000, RESHAPED) X_train = X_train.astype('float32') X_test = X_test.astype('float32')
model.add(Conv2D(1256, (3, 3), padding='same', activation='relu')) model.add(Conv2D(1256, (1, 1), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid')) #2x2x1256 model.add(AveragePooling2D(pool_size=(2, 2), strides=(1, 1), padding='valid')) #1x1x1256 model.add(Flatten()) #model.add(Dense(4096, activation='relu', use_bias=True)) model.add(Dense(628, activation='relu', use_bias=True)) model.add(Dense(314, activation='relu', use_bias=True)) model.add(Dense(5, activation='softmax', use_bias=True)) print("Model created!!") model.summary() print("\nCompiling model...") #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ sgd = SGD(momentum=0.9, nesterov=True, lr=0.003) #callbacks = [LearningRateScheduler(8, verbose=1)] model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy']) print("Successfully compiled model!!") model.save("WorkingModels/convnet224x224x3_untrained.h5") print("Model saved!!")
data = data.astype("float") / 255.0 # partition the data into training and testing splits using 75% of # the data for training and the remaining 25% for testing (trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.25, random_state=42) # convert the labels from integers to vectors trainY = LabelBinarizer().fit_transform(trainY) testY = LabelBinarizer().fit_transform(testY) # initialize the optimizer and model print("[INFO] compiling model...") opt = SGD(lr=0.005) model = ShallowNet.build(width=32, height=32, depth=3, classes=3) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) # train the network print("[INFO] training network...") H = model.fit(trainX, trainY, validation_data=(testX, testY), batch_size=32, epochs=100, verbose=1) # evaluate the network
from keras.preprocessing.image import ImageDataGenerator from keras.datasets import cifar10 import numpy as np from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation, Flatten from keras.layers.convolutional import Conv2D, MaxPooling2D from keras.optimizers import SGD, Adam, RMSprop from sklearn.preprocessing import LabelBinarizer BATCH_SIZE = 128 NB_EPOCH = 40 VERBOSE = 1 # OPTIM = RMSprop() OPTIM = SGD(lr=1e-1) # load dataset (X_train, y_train), (X_test, y_test) = cifar10.load_data() X_train = X_train.astype("float") X_test = X_test.astype("float") X_train /= 255 X_test /= 255 # convert the labels from integers to vectors lb = LabelBinarizer() y_train = lb.fit_transform(y_train) y_test = lb.transform(y_test) # augumenting print("Augmenting training set images...") datagen = ImageDataGenerator( rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, zoom_range=0.2,
# partition the data into training and testing splits, using 75% # of the data for training and the remaining 25% for testing print("[INFO] constructing training/testing split...") (trainData, testData, trainLabels, testLabels) = train_test_split(data, labels, test_size=0.25, random_state=42) # define the architecture of the network model = Sequential() model.add(Dense(768, input_dim=3072, init="uniform", activation="relu")) model.add(Dense(384, activation="relu", kernel_initializer="uniform")) model.add(Dense(2)) model.add(Activation("softmax")) # train the model using SGD print("[INFO] compiling model...") sgd = SGD(lr=0.01) model.compile(loss="binary_crossentropy", optimizer=sgd, metrics=["accuracy"]) model.fit(trainData, trainLabels, epochs=50, batch_size=128, verbose=1) # show the accuracy on the testing set print("[INFO] evaluating on testing set...") (loss, accuracy) = model.evaluate(testData, testLabels, batch_size=128, verbose=1) print("[INFO] loss={:.4f}, accuracy: {:.4f}%".format(loss, accuracy * 100)) # dump the network architecture and weights to file print("[INFO] dumping architecture and weights to file...") model.save(args["model"])
temp_img = image.load_img(test_path + test['image_id'][i] + '.png', target_size=(256, 256)) temp_img = image.img_to_array(temp_img) test_img.append(temp_img) test_img = np.array(test_img) test_img = preprocess_input(test_img) # we chose to train the top few layers of resnet50 for layer in model.layers[:10]: layer.trainable = False for layer in model.layers[10:]: layer.trainable = True sgd = SGD(lr=0.0001, momentum=0.9) model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(X_train, Y_train, batch_size=batch_size, epochs=6, shuffle=True, verbose=1) predictions_valid = model.predict(test_img, batch_size=batch_size, verbose=1) y_pred = np.argmax(predictions_valid, axis=1) from sklearn.metrics import f1_score y_true = pd.read_csv(r'actual_categories.csv')['label']
from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation print 'Building model using softplus as activation function' ''' Use softplus as our activation function ''' model_sp = Sequential() model_sp.add(Dense(128, input_dim=200)) model_sp.add(Activation('softplus')) model_sp.add(Dense(256)) model_sp.add(Activation('softplus')) model_sp.add(Dense(5)) model_sp.add(Activation('softmax')) ''' Use SGD(lr=0.01) as the optimizer ''' ''' lr set to 0.01 according to 02_learningRateSelection.py ''' from keras.optimizers import SGD, Adam, RMSprop, Adagrad sgd = SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False) model_sp.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) history_sp = model_sp.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=True, validation_split=0.1) loss_sp = history_sp.history.get('loss') acc_sp = history_sp.history.get('acc')
def compile(): single = Sequential() single.add( Conv2D(64, (7, 7), strides=(1, 1), padding='valid', activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), input_shape=(33, 33, 4))) single.add(BatchNormalization()) single.add(Dropout(0.5)) single.add( Conv2D(128, (5, 5), strides=(1, 1), padding='valid', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), activation='relu')) single.add(BatchNormalization()) single.add(Dropout(0.5)) single.add( Conv2D(128, (5, 5), strides=(1, 1), padding='valid', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), activation='relu')) single.add(BatchNormalization()) single.add(Dropout(0.5)) single.add( Conv2D(128, (3, 3), strides=(1, 1), padding='valid', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), activation='relu')) single.add(BatchNormalization()) single.add(Dropout(0.25)) single.add( Conv2D(128, (3, 3), strides=(1, 1), padding='valid', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), activation='relu')) single.add(Dropout(0.25)) single.add( Conv2D(128, (3, 3), strides=(1, 1), padding='valid', kernel_regularizer=l1_l2(l1=0.01, l2=0.01), activation='relu')) single.add(BatchNormalization()) single.add(Dropout(0.25)) single.add(Flatten()) single.add(Dense(5, activation='softmax')) sgd = SGD(lr=0.001, decay=0.01, momentum=0.9) single.compile(loss='categorical_crossentropy', optimizer='sgd') return single
generator = define_generator(ndense=4, nhid=100, act='tanh', input_shape=input_shape) discriminator = define_discriminator(ndense=2, nhid=20, act='tanh', lr=lr * 10, mom=momdisc, decay=decay) #, clipnorm=clipnorm) discriminator.trainable = False gan = Sequential(layers=[generator, discriminator]) gan.compile(loss='binary_crossentropy', optimizer=SGD(lr=lr, momentum=momgen, decay=decay, clipnorm=clipnorm)) history = defaultdict(list) for epoch in tqdm(range(num_epochs)): # Generate images. noisebatch = np.random.uniform(-1, 1, size=[batchsize, input_shape]) generated = generator.predict(noisebatch) # Update discriminator. databatch = X_train[ np.random.randint(0, X_train.shape[0], size=batchsize), :] X = np.concatenate([databatch, generated]) T = np.concatenate( [np.ones((databatch.shape[0], )),
from keras.layers.core import Dense, Activation from keras.optimizers import SGD, RMSprop, Adam from keras.utils import np_utils import pandas as pandas import os import matplotlib.pyplot as plt from sklearn.preprocessing import StandardScaler np.random.seed(1617) # training NB_EPOCH = 20 BATCH_SIZE = 128 VERBOSE = 1 NB_CLASSES = 10 # outputs OPTIMIZER = SGD() N_HIDDEN = 128 VALIDATION_SPLIT = 0.2 # how much train is reserved for validation OPTIMIZER = RMSprop() # optimizer OPTIMIZER = Adam() # data suffled between training and testing (X_train, y_train), (X_test, y_test) = mnist.load_data() # X_train is 60000 rows of 28x28 vales, reshaped into 60000xRESHAPED RESHAPED = 784 # 28x28 X_train = X_train.reshape(60000, RESHAPED) X_test = X_test.reshape(10000, RESHAPED) X_train = X_train.astype('float32') X_test = X_test.astype('float32') # normalize X_test between 0 and 1
def fit(self, x_train, y_train, x_val, y_val, sigma_noise=None, **kwargs): r""" Train the QRNN on given training data. The training uses an internal validation set to monitor training progress. This can be either split at random from the training data (see `training_fraction` argument) or provided explicitly using the `x_val` and `y_val` parameters Training of the QRNN is performed using stochastic gradient descent (SGD). The learning rate is adaptively reduced during training when the loss on the internal validation set has not been reduced for a certain number of epochs. Two data augmentation techniques can be used to improve the calibration of the QRNNs predictions. The first one adds Gaussian noise to training batches before they are passed to the network. The noise statistics are defined using the `sigma_noise` argument. The second one is adversarial training. If adversarial training is used, half of each training batch consists of adversarial samples generated using the fast gradient sign method. The strength of the perturbation is controlled using the `delta_at` parameter. During one epoch, each training sample is passed once to the network. Their order is randomzied between epochs. Arguments: x_train(np.array): Array of shape `(n, m)` containing n training samples of dimension m. y_train(np.array): Array of shape `(n, )` containing the training output corresponding to the training data in `x_train`. sigma_noise(None, float, np.array): If not `None` this value is used to multiply the Gaussian noise that is added to each training batch. If None no noise is added. x_val(np.array): Array of shape :code:`(n', m)` containing n' validation inputs that will be used to monitor training loss. Must be provided in unison with :code:`y_val` or otherwise will be ignored. y_val(np.array): Array of shape :code:`(n')` containing n' validation outputs corresponding to the inputs in :code:`x_val`. Must be provided in unison with :code:`x_val` or otherwise will be ignored. adversarial_training(Bool): Whether or not to use adversarial training. `False` by default. delta_at(flaot): Perturbation factor for the fast gradient sign method determining the strength of the adversarial training perturbation. `0.01` by default. batch_size(float): The batch size to use during training. Defaults to `512`. convergence_epochs(int): The number of epochs without decrease in validation loss before the learning rate is reduced. Defaults to `10`. initial_learning_rate(float): The inital value for the learning rate. learning_rate_decay(float): The factor by which to reduce the learning rate after no improvement on the internal validation set was observed for `convergence_epochs` epochs. Defaults to `2.0`. learning_rate_minimum(float): The minimum learning rate at which the training is terminated. Defaults to `1e-6`. maximum_epochs(int): The maximum number of epochs to perform if training does not terminate before. training_split(float): The ratio `0 < ts < 1.0` of the samples in to be used as internal validation set. Defaults to 0.9. """ #if not (x_train[0,:].shape == self.input_dim): # raise Exception("Training input must have the same extent along dimension 1 as input_dim (" + str(self.input_dim)+ ")") #if not (y_train.shape[1] == 1): # raise Exception("Currently only scalar retrieval targets are supported.") x_mean = np.mean(x_train, axis=0, keepdims=True) x_sigma = np.std(x_train, axis=0, keepdims=True) self.x_mean = x_mean self.x_sigma = x_sigma # Handle parameters # at: adversarial training # bs: batch size # ce: convergence epochs # ilr: initial learning rate # lrd: learning rate decay # lrm: learning rate minimum # me: maximum number of epochs # ts: split ratio of training set at, dat, bs, ce, ilr, lrd, lrm, me, ts, kwargs = self.__fit_params__( kwargs) # Split training and validation set if x_val or y_val # are not provided. #n = x_train.shape[0] #n_train = n #loss = QuantileLoss(self.quantiles, self.model_name) loss = QuantileLoss(self.quantiles) self.custom_objects = {loss.__name__: loss} for model in self.models: optimizer = SGD(lr=ilr) #optimizer = Adagrad() #optimizer = Adam() #optimizer = RMSprop(learning_rate=0.001) model.compile(loss=loss, optimizer=optimizer) training_generator = DataGenerator(x_train, self.x_mean, self.x_sigma, y_train, sigma_noise, bs, True) validation_generator = DataGenerator(x_val, self.x_mean, self.x_sigma, y_val, sigma_noise, bs, False) lr_callback = LRDecay(model, lrd, lrm, ce) ''' model.fit_generator(training_generator, steps_per_epoch=n_train // bs, epochs=me, validation_data=validation_generator, validation_steps=1, callbacks=[lr_callback]) ''' model.fit_generator(training_generator, epochs=me, validation_data=validation_generator, callbacks=[lr_callback])
batch_size = 32 test_dir = 'resized_val' file_name = 'vgg16_wallclassification_fine' display_dir = 'display' label = [ '石張り', '吹付タイル', '押出成形セメント板', 'タイル張り', 'スレート波板張り', "スパンドレル", "コンクリート打ち放し", "コンクリートブロック", "ガラスブロック", "ガラスカーテンウォール", "ALC板" ] #load model and weights # json_string=open(file_name+'.json').read() # model=model_from_json(json_string) model = load_model(file_name + '.h5') model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy']) #data generate test_datagen = ImageDataGenerator(rescale=1.0 / 255) test_generator = test_datagen.flow_from_directory(test_dir, target_size=(224, 224), batch_size=batch_size, class_mode='categorical', shuffle=True) #evaluate model score = model.evaluate_generator(test_generator) print('\n test loss:', score[0])