def VGGish(load_weights=True, weights='audioset', input_tensor=None, input_shape=None, out_dim=None, include_top=True, pooling='avg'): if weights not in {'audioset', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `audioset` ' '(pre-training on audioset).') if out_dim is None: out_dim = 128 # input shape if input_shape is None: input_shape = (496, 64, 1) if input_tensor is None: aud_input = Input(shape=input_shape, name='input_1') else: if not K.is_keras_tensor(input_tensor): aud_input = Input(tensor=input_tensor, shape=input_shape, name='input_1') else: aud_input = input_tensor # Block 1 x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv1')(aud_input) x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool1')(x) # Block 2 x = Conv2D(128, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool2')(x) # Block 3 x = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_1')(x) x = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool3')(x) # Block 4 x = Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_1')(x) x = Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool4')(x) if include_top: # FC block x = Flatten(name='flatten_')(x) x = Dense(4096, activation='relu', name='vggish_fc1/fc1_1')(x) x = Dense(4096, activation='relu', name='vggish_fc1/fc1_2')(x) x = Dense(out_dim, activation='relu', name='vggish_fc2')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = aud_input # Create model. model = Model(inputs, x, name='VGGish') # load weights if load_weights: if weights == 'audioset': if include_top: model.load_weights(WEIGHTS_PATH_TOP) else: print("POGGERS") model.load_weights(WEIGHTS_PATH) else: print("failed to load weights") return model
imgs += Path(args.dir).files('*.jpeg') elif args.img[0] is not None: print("Loading images from path(s) : ", args.img) imgs = args.img else: raise RuntimeError('Either -dir or -img arguments must be passed as argument') with tf.device('/CPU:0'): base_model = MobileNet((None, None, 3), alpha=1, include_top=False, pooling='avg', weights=None) x = Dropout(0.75)(base_model.output) x = Dense(10, activation='softmax')(x) model = Model(base_model.input, x) model.load_weights('weights/mobilenet_weights.h5') score_list = [] NIMA_scores = [] for img_path in tqdm(imgs): img = load_img(img_path, target_size=target_size) x = img_to_array(img) x = np.expand_dims(x, axis=0) x = preprocess_input(x) scores = model.predict(x, batch_size=1, verbose=0)[0] mean = mean_score(scores) std = std_score(scores)
save_best_only=True, verbose=1) model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=0.0005), metrics=['acc']) learning_history = model.fit_generator(train_generator, epochs=300, steps_per_epoch=len(x_train) / 32, validation_data=valid_generator, callbacks=[early_stopping, lr, mc]) model.save('../data/lotte/h5/b4_weight_model.h5') # predict model.load_weights('../data/lotte/mc/lotte_b4.h5') result = model.predict(x_pred, verbose=True) # tta_steps = 30 # # tta : 증강된 이미지를 여러번 보여준 다음 각각의 단계에 대해서 prediction을 평균하고 이 결과를 최종값으로 사용하는 것 # predictions = [] # for i in tqdm(range(tta_steps)): # # generator 초기화 # test_generator.reset() # preds = model.predict_generator(generator = test_generator, verbose = 1) # predictions.append(preds) # 평균을 통한 final prediction # pred = np.mean(predictions, axis=0)
class AEOshea1hot(object): def __init__(self, in_dim=None, latent_dim=None, h_dim=None, train_snr_dB=10.0): self.in_dim = in_dim self.latent_dim = latent_dim self.train_snr_dB = train_snr_dB self.h_dim = h_dim self.train_noisepow = 10.0**(-self.train_snr_dB/10.0) if self.in_dim and self.latent_dim: self.make_model() def make_model(self): # Input layer self.inputs = Input(shape=(self.in_dim,), name="enc_in") # Hidden Layers x = self.inputs if self.h_dim is not None: for (i,d) in enumerate(self.h_dim): x = Dense( d, activation='relu', name="enc_l{}".format(i))(x) # Mean and Variance x = Dense(self.latent_dim)(x) self.z_mean = BatchNormalization(center=False, scale=False)(x) # Channel self.z = Lambda(self.channel, output_shape=(self.latent_dim,), name="z")(self.z_mean) # Encoder model self.encoder = Model(self.inputs, [self.z_mean,self.z], name="encoder") # Decoder self.latent_inputs = Input(shape=(self.latent_dim,), name="z_sample") # Hidden layers x = self.latent_inputs if self.h_dim is not None: for (i,d) in enumerate(self.h_dim[::-1]): x = Dense( d, activation='relu', name="dec_l{}".format(i))(x) self.dec_outputs = Dense( self.in_dim, activation='softmax', name="decoder_out")(x) # Decoder model self.decoder = Model(self.latent_inputs, self.dec_outputs, name="decoder") # VAE self.outputs = self.decoder(self.encoder(self.inputs)[1]) self.model = Model(self.inputs, self.outputs, name="VAE") # Losses self.recon_loss = categorical_crossentropy(self.inputs, self.outputs) self.model.add_loss(self.recon_loss) self.model.compile(optimizer='adam') # self.model.compile( optimizer=tf.train.AdamOptimizer(learning_rate=0.01)) def channel( self, zMean ): batch = K.shape( zMean )[0] epsilon = K.random_normal( shape = (batch,self.latent_dim) ) # Because BatchNormalization produces z vector with signal power 'latentDim', # we should not scale the noise power here. return zMean + np.sqrt(self.train_noisepow)*epsilon def fit(self, x_train, epochs=10, batch_size=128, validation_data=None, verbose=0, callbacks=None): train_log = self.model.fit(x_train, epochs=epochs, batch_size=batch_size, validation_data=validation_data, verbose=verbose, callbacks=callbacks) return train_log.history def encode( self, data ): return self.encoder.predict(data) def decode( self, data ): return self.decoder.predict(data) def analysis( self ): xTest = np.eye(self.in_dim) enc_mu, enc_z = self.encode(xTest) dec_mu = self.decode(enc_mu) dec_z = self.decode(enc_z) chDim = self.latent_dim//2 f = plt.figure(figsize=(5*chDim,9)) for i in range(chDim): ax1 = plt.subplot(2,chDim,i+1) ax1.scatter(enc_mu[:,i],enc_mu[:,i+chDim],c=np.arange(self.in_dim)) for j in range(self.in_dim): ax1.annotate( j, (enc_mu[j,i],enc_mu[j,i+chDim]) ) ax1.set_title( "TX Symbols n = {}".format(i+1) ) def save_model(self, fileprefix): with open(fileprefix+".dil","wb") as obj: dill.dump( {'in_dim': self.in_dim, 'latent_dim': self.latent_dim, 'train_noisepow': self.train_noisepow, 'train_snr_dB': self.train_snr_dB, 'h_dim': self.h_dim}, obj ) self.model.save_weights(fileprefix+".h5") def load_model(self, fileprefix): with open(fileprefix+".dil","rb") as obj: config = dill.load(obj) self.in_dim = config['in_dim'] self.latent_dim = config['latent_dim'] self.train_snr_dB = config['train_snr_dB'] self.train_noisepow = config['train_noisepow'] self.h_dim = config['h_dim'] self.make_model() self.model.load_weights(fileprefix+".h5")
x = Conv2D(512, (3, 3), activation='relu', name='conv5_2', padding='same')(x) x = Conv2D(512, (3, 3), activation='relu', name='conv5_3', padding='same')(x) x = pooling_func(x) model = Model(ip, x) if model == "vgg19": weights = get_file('vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5', TF_19_WEIGHTS_PATH_NO_TOP, cache_subdir='models') else: weights = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', TF_WEIGHTS_PATH_NO_TOP, cache_subdir='models') model.load_weights(weights) if K.backend() == 'tensorflow' and K.image_data_format() == "channels_first": convert_all_kernels_in_model(model) print('Model loaded.') outputs_dict = dict([(layer.name, layer.output) for layer in model.layers]) shape_dict = dict([(layer.name, layer.output_shape) for layer in model.layers]) def gram_matrix(input_tensor): assert K.ndim(input_tensor) == 3 features = K.batch_flatten(K.permute_dimensions(input_tensor, (2, 0, 1))) gram = K.dot(features, K.transpose(features)) return gram
gan = Model(inputs=[random_vec_input, labels_input], outputs=decision) gan.compile(loss="binary_crossentropy", optimizer=tf.keras.optimizers.Adam(lr=1e-4)) if verbose: print("Model is built.") print(gan.summary()) # Training # ------------------------- if verbose: print("Training GAN model...") if path_weights is not None and os.path.exists(path_weights): gan.load_weights(path_weights) # Adversarial ground truths real = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) iterations = len(x_train) // batch_size for e in range(epochs): if verbose: print("Epoch {}/{} :".format(e, epochs), end="") for i in range(iterations): # Training discriminator # -------------------------
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath="/content/drive/My Drive/weights{epoch:02d}.h5", save_weights_only=True, save_freq="epoch") N = X_train.shape[0] X_L, Y_L = X_train[:N // 2], Y_train[:N // 2] U_hat = X_train[N // 2:] history = model.fit(generate_data(X_L, Y_L, U_hat, X_val, Y_val, batch_size), epochs=30, steps_per_epoch=X_L.shape[0] // batch_size, callbacks=model_checkpoint_callback) data = np.load('/content/drive/My Drive/Dataset/splitted_mfccs_order0_new.npz') X_test = data['X_test'] Y_test = data['Y_test'] model_U.load_weights("/content/drive/My Drive/weights11.h5") model_U.evaluate(X_test, Y_test) inp = Input(shape=(129, 20)) x = LSTM(128, return_sequences=True)(inp) x = LSTM(32)(x) y = Dense(10, activation='softmax', kernel_regularizer=tf.keras.regularizers.l2(1e-4))(x) model_base = Model(inputs=inp, outputs=y) print(model_base.summary()) model_base.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy']) model_base.fit(X_train, Y_train, epochs=5, validation_data=(X_val, Y_val))
metrics=['accuracy']) with tf.Session() as session: K.set_session(session) session.run(tf.global_variables_initializer()) session.run(tf.tables_initializer()) history = model.fit(x_train, y_train, epochs=10, batch_size=32) model.save_weights('./elmo-model-9.h5') model.summary() with tf.Session() as session: K.set_session(session) session.run(tf.global_variables_initializer()) session.run(tf.tables_initializer()) model.load_weights('./elmo-model-12.h5') predicts = model.predict(x_test, batch_size=10) y_test = decode(le, y_test) y_preds = decode(le, predicts) #for item in y_test: # print(item.strip()) #print(y_test) print('-----') for item in y_preds: print(item.strip()) from sklearn import metrics print(metrics.confusion_matrix(y_test, y_preds))
classifier = nn.classifier(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: print('loading weights from {}'.format(C.base_net_weights)) model_rpn.load_weights(C.base_net_weights, by_name=True) model_classifier.load_weights(C.base_net_weights, by_name=True) except: print( 'Could not load pretrained model weights. Weights can be found in the keras application folder \ https://github.com/fchollet/keras/tree/master/keras/applications') optimizer = Adam(lr=1e-5) optimizer_classifier = Adam(lr=1e-5) model_rpn.compile( optimizer=optimizer, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)]) model_classifier.compile( optimizer=optimizer_classifier, loss=[
class HLRGDVGG(object): def __init__(self, input_dimension, output_dimension, vggModelLocation, number_of_classes=2, optimizer=None, loss_threshold=defaultLossThreshold, patience=defaultPatience, dropout_rate=defaultDropoutRate, max_relu_bound=None, adv_penalty=0.01, unprotected=False, verbose=False): self.buildModel(input_dimension=input_dimension, output_dimension=output_dimension , vggModelLocation=vggModelLocation, optimizer=optimizer, number_of_classes=number_of_classes, loss_threshold=loss_threshold, patience=patience, dropout_rate=dropout_rate, max_relu_bound=max_relu_bound, adv_penalty=adv_penalty, unprotected=unprotected, verbose=verbose) def buildModel(self, input_dimension, output_dimension, vggModelLocation, number_of_classes=2, optimizer=None, loss_threshold=defaultLossThreshold, patience=defaultPatience, dropout_rate=defaultDropoutRate, max_relu_bound=None, adv_penalty=0.01, unprotected=False, verbose=False): #instantiate the vgg instance self.ourVGG = HLVGG(input_dimension=input_dimension, output_dimension=output_dimension, dual_outputs=True, number_of_classes=number_of_classes, loss_threshold=loss_threshold, patience=patience, dropout_rate=0, freezeWeights=True, optimizer=optimizer, max_relu_bound=max_relu_bound, adv_penalty=adv_penalty, unprotected=True, verbose=verbose) #set the vgg weights self.ourVGG.model.load_weights(vggModelLocation) #set protected flag self.unprotected = unprotected #instantiate our CAE with residual connections self.input_dimension, self.output_dimension = input_dimension, np.copy(output_dimension) self.loss_threshold, self.number_of_classes = np.copy(loss_threshold), np.copy(number_of_classes) self.dropoutRate, self.max_relu_bound = dropout_rate, np.copy(max_relu_bound) self.patience = np.copy(patience) # self.learning_rate, self.learning_rate_drop = np.copy(learning_rate), np.copy(learning_rate_drop) self.image_size = 32 self.num_channels = 3 self.num_labels = number_of_classes self.penaltyCoeff = adv_penalty if (verbose): print("input dimension: %s"%str(self.input_dimension)) # define input layer self.inputLayer = layers.Input(shape=self.input_dimension) self.advInputLayer = layers.Input(shape=self.input_dimension) previousLayer = self.advInputLayer #following the architecture shown in figure 3 of the hlrgd paper, [ https://openaccess.thecvf.com/content_cvpr_2018/papers/Liao_Defense_Against_Adversarial_CVPR_2018_paper.pdf ] # self.hiddenEncoderLayers = dict() #first encoder block previousLayer = layers.Conv2D(numberOfFilters[0], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[0], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) self.hiddenEncoderLayers[0] = previousLayer #second encoder block previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) self.hiddenEncoderLayers[1] = previousLayer #third encoder block previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) self.hiddenEncoderLayers[2] = previousLayer #fourth encoder block previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) self.hiddenEncoderLayers[3] = previousLayer #fifth encoder block previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) self.hiddenEncoderLayers[4] = previousLayer #first decoding block #fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer) previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[3]]) #perform convolutions previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) #second decoding block #fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer # previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer) previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[2]]) #perform convolutions previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) #third decoding block #fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer # previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer) previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[1]]) #perform convolutions previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) #fourth decoding block #fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer # previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer) previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[0]]) #perform convolutions previousLayer = layers.Conv2DTranspose(numberOfFilters[8], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) previousLayer = layers.Conv2DTranspose(numberOfFilters[8], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer) previousLayer = layers.BatchNormalization()(previousLayer) if (self.dropoutRate > 0): previousLayer = Dropout(self.dropoutRate)(previousLayer) #1x1 convolutional layer previousLayer = layers.Conv2DTranspose(1, kernel_size=(1,1), strides=(1, 1), padding='same', activation='relu')(previousLayer) #compute the output #\hat{x} = x^{\asterisk} - d\hat{x} duOutput = self.inputLayer - previousLayer #compute vgg penultimate layer post-activation outputs when the vgg subnet is stimulated with duOutput and the benign sample # inputList = [self.inputLayer, self.inputLayer] # duList = [duOutput, duOutput] vggOutputBenign, vggPenultimate = self.ourVGG.model([self.inputLayer, self.inputLayer]) vggOutputDU, vggPenultimateDU = self.ourVGG.model([duOutput, duOutput]) #assign output layer as the evaluation of vgg that depends on the (potentially) adv. input self.outputLayer = vggOutputDU #formulate our loss function #define our custom loss function #this is L1 norm of the difference between the vggOutput def customLossWrapper(vggPenultimateDU, vggPenultimate, penaltyCoeff = self.penaltyCoeff): def customLoss(y_true, y_pred): return K.sum(K.abs(vggPenultimateDU - vggPenultimate)) return customLoss # optimization details # def lr_scheduler(epoch): # return self.learning_rate * (0.5 ** (epoch // self.learning_rate_drop)) # self.lr_scheduler = lr_scheduler # self.reduce_lr = keras.callbacks.LearningRateScheduler(lr_scheduler) # self.sgd = tf.keras.optimizers.Nadam()#Adadelta(learning_rate=self.learning_rate) # self.sgd = tf.keras.optimizers.SGD(lr=self.learning_rate, decay=0.000001, momentum=0.9, nesterov=True) if (optimizer==None): self.sgd = tf.keras.optimizers.Nadam()#Adadelta(learning_rate=self.learning_rate) self.reduceLR = None elif(optimizer=='SGD'): def lr_scheduler(epoch): return 0.1 * (0.5 ** (epoch // 20))#learning_rate * (0.5 ** (epoch // 20)) self.reduceLR = keras.callbacks.LearningRateScheduler(lr_scheduler) self.sgd = tf.keras.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #set up data augmentation self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # convert self.hiddenAdvLayers to a list for the model compilation, ascending order of keys is order of layers # outputsList is a list of outputs of the model constructed so that the first entry is the true output (ie prediction) layer # and each subsequent (i, i+1)th entries are the pair of hiddenAdvLayer, hiddenBenignLayer activations # this is going to be useful for calculating the MAE between benignly and adversarially induced hidden states # outputsList = [self.outputLayer] # for curHiddenLayer in range(len(self.ourVGG.hiddenAdvModelOutputs)): # outputsList.append(self.ourVGG.hiddenAdvModelOutputs[curHiddenLayer]) # outputsList.append(self.ourVGG.hiddenModelOutputs[curHiddenLayer]) # instantiate and compile the model self.customLossWrapper = customLossWrapper self.model = Model(inputs=[self.inputLayer, self.advInputLayer], outputs=[self.outputLayer], name='hlrgd_vgg16') self.model.compile(loss=customLossWrapper(vggPenultimateDU, vggPenultimate, self.penaltyCoeff), metrics=['acc'], optimizer=self.sgd) # double check weight trainability bug allVars = self.model.variables trainableVars = self.model.trainable_variables allVarNames = [self.model.variables[i].name for i in range(len(self.model.variables))] trainableVarNames = [self.model.trainable_variables[i].name for i in range(len(self.model.trainable_variables))] nonTrainableVars = np.setdiff1d(allVarNames, trainableVarNames) if (verbose): if (len(nonTrainableVars) > 0): print('the following variables are set to non-trainable; ensure that this is correct before publishing!!!!') print(nonTrainableVars) self.model.summary() #set data statistics to default values self.mean = 0 self.stddev = 1 # handle data augmentation with multiple inputs (example found on https://stackoverflow.com/questions/49404993/keras-how-to-use-fit-generator-with-multiple-inputs #so thanks to loannis and Julian def multiInputDataGenerator(self, X1, X2, Y, batch_size): genX1 = self.generator.flow(X1, Y, batch_size=batch_size) genX2 = self.generator.flow(X2, Y, batch_size=batch_size) while True: X1g = genX1.next() X2g = genX2.next() yield [X1g[0], X2g[0]], X1g[1] #this method trains the HLRGD #inputData is a list of training matrices, the 0th is adversarial (i.e., to be denoised), the 2nd is the benign input def train(self, inputTrainingData, trainingTargets, inputValidationData, validationTargets, training_epochs=1, normed=False, monitor='val_loss', patience=defaultPatience, model_path=None, keras_batch_size=None, validation_split=0.1, adversarialOrder=0, dataAugmentation=False): #if a path isn't provided by caller, just use the current time for restoring best weights from fit if (model_path is None): model_path = os.path.join('/tmp/models/', 'hlrgd_vgg16_'+str(int(round(time.time()*1000)))) #if the data are not normalized, normalize them trainingData, validationData = [[],[]], [[],[]] if (not normed): #don't store stats from the adversarially attacked data if (adversarialOrder == 0): trainingData[0] = self.normalize(inputTrainingData[0], storeStats=True) trainingData[1] = self.normalize(inputTrainingData[1], storeStats=False) else: trainingData[1] = self.normalize(inputTrainingData[1], storeStats=True) trainingData[0] = self.normalize(inputTrainingData[0], storeStats=False) #also don't store stats from validation data validationData[0] = self.normalize(inputValidationData[0], storeStats=False) validationData[1] = self.normalize(inputValidationData[1], storeStats=False) else: trainingData[0] = inputTrainingData[0] trainingData[1] = inputTrainingData[1] validationData[0] = inputValidationData[0] validationData[1] = inputValidationData[1] #collect our callbacks earlyStopper = EarlyStopping(monitor=monitor, mode='min', patience=patience, verbose=1, min_delta=defaultLossThreshold) checkpoint = ModelCheckpoint(model_path, verbose=1, monitor=monitor, save_weights_only=True, save_best_only=True, mode='auto') # history = self.model.fit(trainingData, trainingTargets, epochs=training_epochs, batch_size=keras_batch_size, # validation_split=validation_split, callbacks=[earlyStopper, self.reduce_lr]) #handle data augmentation if (not dataAugmentation): # set up data augmentation self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening # randomly shift images vertically (fraction of total height) horizontal_flip=False, # randomly flip images vertical_flip=False) self.generator.fit(trainingData[0]) history = self.model.fit(self.multiInputDataGenerator(trainingData[0], trainingData[1], trainingTargets, keras_batch_size), steps_per_epoch=trainingData[0].shape[0] // keras_batch_size, epochs=training_epochs, validation_data=(validationData, validationTargets), callbacks=[earlyStopper, checkpoint], verbose=1) #self.reduce_lr else: # set up data augmentation self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=False, # randomly flip images vertical_flip=False) self.generator.fit(trainingData[0]) history = self.model.fit(self.multiInputDataGenerator(trainingData[0], trainingData[1], trainingTargets, keras_batch_size), steps_per_epoch=trainingData[0].shape[0] // keras_batch_size, epochs=training_epochs, validation_data=(validationData, validationTargets), callbacks=[earlyStopper, checkpoint], verbose=1) #self.reduce_lr self.model.load_weights(model_path) loss, acc = history.history['loss'], history.history['val_acc'] return loss, acc, model_path def evaluate(self, inputData, targets, batchSize=None): evalData = [self.normalize(inputData[0], storeStats=False), self.normalize(inputData[1], storeStats=False)] fullEval = self.model.evaluate(evalData, targets, batch_size=batchSize) return fullEval #method to read model from the disk def readModelFromDisk(self, pathToFile, vggModelLocation): #rebuild the model self.buildModel(self.input_dimension, self.output_dimension, vggModelLocation, self.number_of_classes, loss_threshold=self.loss_threshold, patience=self.patience, dropout_rate=self.dropoutRate, max_relu_bound=self.max_relu_bound, adv_penalty=self.penaltyCoeff, unprotected=self.unprotected, verbose=False) # set the vgg weights self.model.load_weights(pathToFile) # #read in the picklebox pickleBox = pickle.load(open(pathToFile+'_pickle', 'rb')) # # self.bottleneckLayer = pickleBox['bottleneckLayer'] # # self.hiddenEncodingLayer = pickleBox['hiddenEncodingLayer'] # # self.inputLayer = pickleBox['inputLayer'] self.mean, self.std = pickleBox['scaleMean'], pickleBox['scaleSTD'] #first read in the inner model # self.ourVGG.readModelFromDisk('inner_'+pathToFile) # #read in the picklebox # pickleBox = pickle.load(open(pathToFile+'_pickle', 'rb')) # # self.bottleneckLayer = pickleBox['bottleneckLayer'] # # self.hiddenEncodingLayer = pickleBox['hiddenEncodingLayer'] # # self.inputLayer = pickleBox['inputLayer'] # self.chosenActivation = pickleBox['chosenActivation'] # formulate our loss function # define our custom loss function # this is L1 norm of the difference between the vggOutput # if (not self.unprotected): # def customLossWrapper(benProjs, advProjs, penaltyCoeff = self.penaltyCoeff): # def customLoss(y_true, y_pred): # return K.categorical_crossentropy(y_true, y_pred) + penaltyCoeff*K.sum(K.abs(benProjs - advProjs))/(0.00000001+tf.cast(tf.shape(benProjs)[0], tf.float32)) # return customLoss # else:#if we are using an unprotected model, don't force the machine to calculate this too # def customLossWrapper(benProjs, advProjs, penaltyCoeff = self.penaltyCoeff): # def customLoss(y_true, y_pred): # return K.categorical_crossentropy(y_true, y_pred) # return customLoss # self.customLossWrapper = customLossWrapper(self.benProjs, self.advProjs) # #load the model # self.model = load_model(pathToFile, custom_objects={'customLossWrapper': self.customLossWrapper}) # this routine is used to collect statistics on training data, as well as to preprocess the training data by normalizing # i.e. centering and dividing by standard deviation def normalize(self, inputData, storeStats=False): if (storeStats): self.mean = np.mean(inputData) self.stddev = np.std(inputData) outputData = (inputData - self.mean) / (self.stddev + 0.0000001) return outputData # routine to get a pointer to the optimizer of this model def getOptimizer(self): if (self.ourVGG is not None): return self.sgd, self.ourVGG.getOptimizer() else: return self.sgd
class AGZeroModel: def __init__(self, N, batch_size=32, archive_fit_samples=64, use_tpu=None, log_path='logs/tensorboard'): self.N = N self.batch_size = batch_size self.model = None self.archive_fit_samples = archive_fit_samples self.position_archive = [] self.tpu_grpc_url = use_tpu tpu_name_environ_key = 'TPU_NAME' # Check has server got TPU if use_tpu is not False and tpu_name_environ_key in os.environ: tpu_name = os.environ[tpu_name_environ_key].strip() if tpu_name != "": self.is_tpu = True self.tpu_grpc_url = TPUClusterResolver( tpu=[os.environ[tpu_name_environ_key]]).get_master() # TODO write an if condition to validate and resolve the TPU url provided self.__loss_functions = [ 'categorical_crossentropy', 'binary_crossentropy' ] self.model_name = time.strftime('GM{0}-%y%m%dT%H%M%S').format('%02d' % N) # print(self.model_name) log_path = os.path.join(log_path, self.model_name) if not os.path.exists(log_path): os.makedirs(log_path) self.callback = TensorBoard(log_path) def create(self): bn_axis = 3 N = self.N position = Input((N, N, 6)) resnet = ResNet(n_stages=N) resnet.create(N, N, 6) x = resnet.model(position) dist = Conv2D(2, (1, 1))(x) dist = BatchNormalization(axis=bn_axis)(dist) dist = Activation('relu')(dist) dist = Flatten()(dist) dist = Dense(N * N + 1, activation='softmax', name='distribution')(dist) res = Conv2D(1, (1, 1))(x) res = BatchNormalization(axis=bn_axis)(res) res = Activation('relu')(res) res = Flatten()(res) res = Dense(256, activation='relu')(res) res = Dense(1, activation='sigmoid', name='result')(res) self.model = Model(position, [dist, res]) self.model.compile(Adam(lr=2e-2), self.__loss_functions) self.callback.set_model(self.model) # check if TPU available if self.tpu_grpc_url is not None: self.model = tf.contrib.tpu.keras_to_tpu_model( self.model, strategy=tf.contrib.tpu.TPUDistributionStrategy( tf.contrib.cluster_resolver.TPUClusterResolver( self.tpu_grpc_url))) self.model.summary() def fit_game(self, X_positions, result): X_posres = [] for pos, dist in X_positions: X_posres.append((pos, dist, result)) result = -result if len(self.position_archive) >= self.archive_fit_samples: archive_samples = random.sample(self.position_archive, self.archive_fit_samples) else: # initial case archive_samples = self.position_archive self.position_archive.extend(X_posres) # I'm going to some lengths to avoid the potentially overloaded + operator X_fit_samples = list(itertools.chain(X_posres, archive_samples)) self.__fit_model(X_fit_samples, self.batch_size) def retrain_position_archive(self, batch_size=None): self.__fit_model(self.position_archive, batch_size if batch_size else self.batch_size * 8) def reduce_position_archive(self, ratio=0.5): try: self.position_archive = random.sample( self.position_archive, int(len(self.position_archive) * ratio)) except: pass def __fit_model(self, X_fit_samples, batch_size): batch_no = 1 X, y_dist, y_res = [], [], [] X_shuffled = random.sample(X_fit_samples, len(X_fit_samples)) X_shuffled = create_batches(X_shuffled, batch_size) for batch in X_shuffled: for pos, dist, res in batch: X.append(pos) y_dist.append(dist) y_res.append(float(res) / 2 + 0.5) logs = self.model.train_on_batch( np.array(X), [np.array(y_dist), np.array(y_res)]) self.write_log(self.__loss_functions, logs, batch_no) batch_no += 1 X, y_dist, y_res = [], [], [] def write_log(self, names, logs, batch_no): for name, value in zip(names, logs): summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value summary_value.tag = name self.callback.writer.add_summary(summary, batch_no) self.callback.writer.flush() def predict(self, X_positions): dist, res = self.model.predict(X_positions) res = np.array([r[0] * 2 - 1 for r in res]) return [dist, res] def save(self, snapshot_id, save_archive=False): self.model.save_weights('%s.weights.h5' % (snapshot_id, )) if save_archive: joblib.dump(self.position_archive, '%s.archive.joblib' % (snapshot_id, ), compress=5) def load(self, snapshot_id): self.model.load_weights('%s.weights.h5' % (snapshot_id, )) pos_fname = '%s.archive.joblib' % (snapshot_id, ) try: self.position_archive = joblib.load(pos_fname) except: print('Warning: Could not load position archive %s' % (pos_fname, )) def unload_pos_archive(self): self.position_archive = [] def load_pos_archive(self, archive_file): try: print('Attempting to load position archive %s' % (archive_file, )) self.position_archive = joblib.load(archive_file) print('Successfully loaded position archive %s' % (archive_file, )) return True except: import traceback traceback.print_exc() print('Warning: Could not load position archive %s' % (archive_file, )) return False def load_averaged(self, weights, log=None): new_weights = [] loaded_weights = [] for weight in weights: self.model.load_weights(weight) loaded_weights.append(self.model.get_weights()) print("Read weight: {0}".format(weight)) if log is not None: log("Read weight: {0}".format(weight), self.model_name) if len(loaded_weights) > 0: for weights_list_tuple in zip(*loaded_weights): new_weights.append([ np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple) ]) self.model.set_weights(new_weights) else: print( "No weights to load. Initializing the model with random weights!" )
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) # Third callback; save the best model-weights (best validation_accuracy) weights_dir = "temp/" model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=weights_dir, save_weights_only=True, monitor='val_accuracy', mode='max', save_best_only=True) model.fit(features_train, ltr, epochs=50, batch_size=128, validation_data=(features_test, ltt), callbacks=[ tensorboard_callback, model_checkpoint_callback, cm_callback ]) # Load the temporarly saved best model weights and save the entire model with this weights model.load_weights(weights_dir) model_dir = 'models/' + MODELNAME model.save(model_dir) print("Model trained and saved!") cc = cc + 1
def Xception(include_top=True, weights='hasc', input_shape=None, pooling=None, classes=6, classifier_activation='softmax'): if input_shape is None: input_shape = (256*3, 1) if weights in ['hasc', 'HASC'] and include_top and classes != 6: raise ValueError('If using `weights` as `"hasc"` with `include_top`' ' as true, `classes` should be 6') inputs = layers.Input(shape=input_shape) x = layers.Conv1D(32, 3, strides=2, use_bias=False, name='block1_conv1')(inputs) x = layers.BatchNormalization(name='block1_conv1_bn')(x) x = layers.Activation('relu', name='block1_conv1_act')(x) x = layers.Conv1D(64, 3, use_bias=False, name='block1_conv2')(x) x = layers.BatchNormalization(name='block1_conv2_bn')(x) x = layers.Activation('relu', name='block1_conv2_act')(x) residual = layers.Conv1D( 128, 1, strides=2, padding='same', use_bias=False )(x) residual = layers.BatchNormalization()(residual) x = layers.SeparableConv1D(128, 3, padding='same', use_bias=False, name='block2_sepconv1')(x) x = layers.BatchNormalization(name='block2_sepconv1_bn')(x) x = layers.Activation('relu', name='block2_sepconv2_act')(x) x = layers.SeparableConv1D(128, 3, padding='same', use_bias=False, name='block2_sepconv2')(x) x = layers.BatchNormalization(name='block2_sepconv2_bn')(x) x = layers.MaxPooling1D(3, strides=2, padding='same', name='block2_pool')(x) x = layers.add([x, residual]) residual = layers.Conv1D( 256, 1, strides=2, padding='same', use_bias=False )(x) residual = layers.BatchNormalization()(residual) x = layers.Activation('relu', name='block3_sepconv1_act')(x) x = layers.SeparableConv1D(256, 3, padding='same', use_bias=False, name='block3_sepconv1')(x) x = layers.BatchNormalization(name='block3_sepconv1_bn')(x) x = layers.Activation('relu', name='block3_sepconv2_act')(x) x = layers.SeparableConv1D(256, 3, padding='same', use_bias=False, name='block3_sepconv2')(x) x = layers.BatchNormalization(name='block3_sepconv2_bn')(x) x = layers.MaxPooling1D(3, strides=2, padding='same', name='block3_pool')(x) x = layers.add([x, residual]) residual = layers.Conv1D(728, 1, strides=2, padding='same', use_bias=False)(x) residual = layers.BatchNormalization()(residual) x = layers.Activation('relu', name='block4_sepconv1_act')(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block4_sepconv1')(x) x = layers.BatchNormalization(name='block4_sepconv1_bn')(x) x = layers.Activation('relu', name='block4_sepconv2_act')(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block4_sepconv2')(x) x = layers.BatchNormalization(name='block4_sepconv2_bn')(x) x = layers.MaxPooling1D(3, strides=2, padding='same', name='block4_pool')(x) x = layers.add([x, residual]) for i in range(8): residual = x prefix = 'block' + str(i + 5) x = layers.Activation('relu', name=prefix + "_sepconv1_act")(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv1")(x) x = layers.BatchNormalization(name=prefix + "_sepconv1_bn")(x) x = layers.Activation('relu', name=prefix + "_sepconv2_act")(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv2")(x) x = layers.BatchNormalization(name=prefix + "_sepconv2_bn")(x) x = layers.Activation('relu', name=prefix + "_sepconv3_act")(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv3")(x) x = layers.add([x, residual]) residual = layers.Conv1D(1024, 1, strides=2, padding='same', use_bias=False)(x) residual = layers.BatchNormalization()(residual) x = layers.Activation('relu', name='block13_sepconv1_act')(x) x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block13_sepconv1')(x) x = layers.BatchNormalization(name='block13_sepconv1_bn')(x) x = layers.Activation('relu', name='block13_speconv2_act')(x) x = layers.SeparableConv1D(1024, 3, padding='same', use_bias=False, name='block13_sepconv2')(x) x = layers.BatchNormalization(name='block13_sepconv2_bn')(x) x = layers.MaxPooling1D(3, strides=2, padding='same')(x) x = layers.add([x, residual]) x = layers.SeparableConv1D(1536, 3, padding='same', use_bias=False, name='block14_sepconv1')(x) x = layers.BatchNormalization(name='block14_sepconv1_bn')(x) x = layers.Activation('relu', name='block14_sepconv1_act')(x) x = layers.SeparableConv1D(2048, 3, padding='same', use_bias=False, name='block14_sepconv2')(x) x = layers.BatchNormalization(name='block14_sepconv2_bn')(x) x = layers.Activation('relu', name='block14_sepconv2_act')(x) x = layers.GlobalAveragePooling1D(name='avg_pool')(x) y = layers.Dense(classes, activation=classifier_activation, name='predictions')(x) model = Model(inputs, y) if weights is not None: if weights in ['hasc', "HASC"]: weights = 'weights/xception/xception_hasc_weights_{}_{}.hdf5'.format(int(input_shape[0]), int(input_shape[1])) # hasc or weights fileで初期化 if os.path.exists(weights): print("Load weights from {}".format(weights)) model.load_weights(weights) else: print("Not exist weights: {}".format(weights)) # topを含まないとき if not include_top: if pooling is None: # topを削除する model = Model(inputs=model.input, outputs=model.layers[-3].output) elif pooling == 'avg': y = layers.GlobalAveragePooling1D()(model.layers[-3].output) model = Model(inputs=model.input, outputs=y) elif pooling == 'max': y = layers.GlobalMaxPooling1D()(model.layers[-3].output) model = Model(inputs=model.input, outputs=y) else: print("Not exist pooling option: {}".format(pooling)) model = Model(inputs=model.input, outputs=model.layers[-3].output) return model
class SRGAN(): def __init__(self, quantize_flag): self.quantize_flag = quantize_flag # Input shape self.channels = 3 self.lr_height = 56 # Low resolution height self.lr_width = 56 # Low resolution width self.lr_shape = (self.lr_height, self.lr_width, self.channels) self.hr_height = self.lr_height * 4 # High resolution height self.hr_width = self.lr_width * 4 # High resolution width self.hr_shape = (self.hr_height, self.hr_width, self.channels) # Number of residual blocks in the generator self.n_residual_blocks = 16 optimizer = Adam(0.000002, 0.5) # We use a pre-trained VGG19 model to extract image features from the high resolution # and the generated high resolution images and minimize the mse between them self.vgg = self.build_vgg() self.vgg.trainable = False self.vgg.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Configure data loader self.dataset_name = 'img_align_celeba' self.test_dataset_name = 'test_images' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.hr_height, self.hr_width)) self.test_data_loader = DataLoader(dataset_name=self.test_dataset_name, img_res=(self.hr_height, self.hr_width)) # Calculate output shape of D (PatchGAN) patch = int(self.hr_height / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # High res. and low res. images img_hr = Input(shape=self.hr_shape) img_lr = Input(shape=self.lr_shape) # Generate high res. version from low res. fake_hr = self.generator(img_lr) # Extract image features of the generated img fake_features = self.vgg(fake_hr) # For the combined model we will only train the generator self.discriminator.trainable = False # Discriminator determines validity of generated high res. images validity = self.discriminator(fake_hr) self.combined = Model([img_lr, img_hr], [validity, fake_features]) self.combined.compile(loss=['binary_crossentropy', 'mse'], loss_weights=[1e-3, 1], optimizer=optimizer) def build_vgg(self): """ Builds a pre-trained VGG19 model that outputs image features extracted at the third block of the model """ vgg = VGG19(weights="imagenet") # Set outputs to outputs of last conv. layer in block 3 # See architecture at: https://github.com/tensorflow.keras-team/tensorflow.keras/blob/master/tensorflow.keras/applications/vgg19.py vgg.outputs = [vgg.layers[9].output] img = Input(shape=self.hr_shape) # Extract image features print(img.shape) img_features = vgg(img) return Model(img, img_features) def build_generator(self): def residual_block(layer_input, filters): """Residual block described in paper""" d = Conv2D(filters, kernel_size=3, strides=1, padding='same', activation='relu')(layer_input) d = BatchNormalization(momentum=0.8)(d) d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) d = Add()([d, layer_input]) return d def deconv2d(layer_input): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(256, kernel_size=3, strides=1, padding='same', activation="relu")(u) return u # Low resolution image input img_lr = Input(shape=self.lr_shape) # Pre-residual block c1 = Conv2D( 64, kernel_size=9, strides=1, padding='same', activation='relu', )(img_lr) # Propogate through residual blocks r = residual_block(c1, self.gf) for _ in range(self.n_residual_blocks - 1): r = residual_block(r, self.gf) # Post-residual block c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r) c2 = BatchNormalization(momentum=0.8)(c2) c2 = Add()([c2, c1]) # Upsampling u1 = deconv2d(c2) u2 = deconv2d(u1) # Generate high resolution output gen_hr = Conv2D(self.channels, kernel_size=9, strides=1, padding='same', activation='tanh')(u2) if self.quantize_flag == "quant": return Quantizer.apply_quantization(Model(img_lr, gen_hr), weight_precision=16, activation_precision=16) elif self.quantize_flag == "sparsity": return Sparsity.measure_sparsity(Model(img_lr, gen_hr)) else: return Model(img_lr, gen_hr) def build_discriminator(self): def d_block(layer_input, filters, strides=1, bn=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=3, strides=strides, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d # Input img d0 = Input(shape=self.hr_shape) d1 = d_block(d0, self.df, bn=False) d2 = d_block(d1, self.df, strides=2) d3 = d_block(d2, self.df * 2) d4 = d_block(d3, self.df * 2, strides=2) d5 = d_block(d4, self.df * 4) d6 = d_block(d5, self.df * 4, strides=2) d7 = d_block(d6, self.df * 8) d8 = d_block(d7, self.df * 8, strides=2) d9 = Dense(self.df * 16)(d8) d10 = LeakyReLU(alpha=0.2)(d9) validity = Dense(1, activation='sigmoid')(d10) return Model(d0, validity) def train(self, epochs, batch_size=1, sample_interval=50, start_from_ckp_number=None): start_time = datetime.datetime.now() # Set up ternsorboard logdir = "logs/scalars/" + datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") file_writer = tf.summary.create_file_writer(logdir + "/metrics") file_writer.set_as_default() # Restore checkpoint if present start_epoch = 0 if start_from_ckp_number is not None: start_epoch = start_from_ckp_number self.generator.load_weights( './saved_models/generator_ckp_epoch_{}'.format( start_from_ckp_number)) self.discriminator.load_weights( './saved_models/discriminator_ckp_epoch_{}'.format( start_from_ckp_number)) self.combined.load_weights( './saved_models/combined_ckp_epoch_{}'.format( start_from_ckp_number)) callbacks = [ TensorBoard(log_dir=r"./logs/tensorboard", histogram_freq=1, write_graph=True, write_images=False, update_freq='epoch', profile_batch=0, embeddings_freq=0, embeddings_metadata=None) ] for epoch in range(start_epoch, epochs): # ---------------------- # Train Discriminator # ---------------------- # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # From low res. image generate high res. version fake_hr = self.generator.predict(imgs_lr) valid = np.ones((batch_size, ) + self.disc_patch) fake = np.zeros((batch_size, ) + self.disc_patch) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid) d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # ------------------ # Train Generator # ------------------ # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # The generators want the discriminators to label the generated images as real valid = np.ones((batch_size, ) + self.disc_patch) # Extract ground truth image features using pre-trained VGG19 model image_features = self.vgg.predict(imgs_hr) # Train the generators try: self.combined.fit([imgs_lr, imgs_hr], [valid, image_features], epochs=epoch + 1, batch_size=len(imgs_lr), initial_epoch=epoch, callbacks=callbacks) #g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features]) except Exception as e: print("Something went wrong with this batch: {}".format(e)) elapsed_time = datetime.datetime.now() - start_time # Plot the progress if epoch % 10 == 0: print("%d time: %s" % (epoch, elapsed_time)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch, batch_size) def sample_images(self, epoch, batch_size): os.makedirs('images/%s' % self.test_dataset_name, exist_ok=True) # Get test data # imgs_hr, imgs_lr = self.data_loader.load_data(batch_size=2, is_testing=True) imgs_hr_a, imgs_lr_a = self.test_data_loader.load_data( batch_size=batch_size, is_testing=True) # Make prediction fake_hr_a = self.generator.predict(imgs_lr_a) image_idx = 0 for imgs_lr, fake_hr, imgs_hr in zip(imgs_lr_a, fake_hr_a, imgs_hr_a): # # Rescale images 0 - 1 imgs_lr = 0.5 * imgs_lr + 0.5 fake_hr = 0.5 * fake_hr + 0.5 imgs_hr = 0.5 * imgs_hr + 0.5 imgs_hr = imgs_hr.reshape(self.hr_shape) scipy.misc.toimage(imgs_hr).save( 'images/{}/batch_{}_idx_{}_original_highres.png'.format( self.test_dataset_name, epoch, image_idx, )) # Save lr version (if not present yet) imgs_lr = imgs_lr.reshape(self.lr_shape) scipy.misc.toimage(imgs_lr).save( 'images/{}/batch_{}_idx_{}_original_lowres.png'.format( self.test_dataset_name, epoch, image_idx, )) # Save generaed image fake_hr = fake_hr.reshape(self.hr_shape) scipy.misc.toimage(fake_hr).save( 'images/{}/batch_{}_idx_{}_generated_highres.png'.format( self.test_dataset_name, epoch, image_idx, )) image_idx = image_idx + 1 # Compute mean squared error mse = np.mean((np.array(imgs_hr_a, dtype=np.float32) - np.array(fake_hr_a, dtype=np.float32))**2) # Compute psnr if mse == 0: # avoid (improbable) division by 0 psnr = 1000000 else: max_pixel = 1.0 psnr = 20 * np.log10(max_pixel / np.sqrt(mse)) # save MSE and PSNR to tensorboard tf.summary.scalar('MSE', data=mse, step=epoch) tf.summary.scalar('PSNR', data=psnr, step=epoch) for index, layer in enumerate(self.generator.layers): try: sparsity = layer.get_weights()[4] tf.summary.scalar('Sparsity_{}'.format(index), data=sparsity, step=epoch) except IndexError: pass # try: # print(layer.layer) # except: # print("no", layer) # save the generator model weights self.generator.save_weights( './saved_models/generator_ckp_epoch_{}'.format(epoch)) self.discriminator.save_weights( './saved_models/discriminator_ckp_epoch_{}'.format(epoch)) self.combined.save_weights( './saved_models/combined_ckp_epoch_{}'.format(epoch))
model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy') checkpoint_path = "model-1/cp.ckpt" checkpoint_dir = os.path.dirname(checkpoint_path) es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3, restore_best_weights=True) cp_callback = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1) initial_epoch = 0 if os.path.exists(checkpoint_path+".index")==False: print("No model saved.") else: print("Model loaded.") latest = tf.train.latest_checkpoint(checkpoint_dir) initial_epoch = 0 model.load_weights(latest) history=model.fit([x_tr,y_tr[:,:-1]], y_tr.reshape(y_tr.shape[0],y_tr.shape[1], 1)[:,1:], epochs=25, initial_epoch=initial_epoch, callbacks=[es, cp_callback], batch_size=128, validation_data=([x_val,y_val[:,:-1]], y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:])) reverse_target_word_index=y_tokenizer.index_word reverse_source_word_index=x_tokenizer.index_word target_word_index=y_tokenizer.word_index
input_tensor=img_input) elif C.network == 'resnet152': from src.architectures import resnet152 as nn base_layers = ResNet152(weights=None, include_top=False, input_tensor=img_input) with tf.device(device): print('Loading weights from {}'.format(options.weights)) classifier = nn.classifier(base_layers.output, trainable=False) optimizer = Adam(learning_rate=0.0001) model = Model(inputs=base_layers.input, outputs=classifier) model.load_weights(options.weights) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) def load_binarizer(path): global binarizer with open(path, 'rb') as f: binarizer = pickle.load(f) def prepare_images(images, target): """ Args: image: list containing images paths
def Deeplabv3pXception(input_shape=(512, 512, 3), weights='pascal_voc', input_tensor=None, classes=21, OS=16, **kwargs): """ Instantiates the Deeplabv3+ architecture Optionally loads weights pre-trained on PASCAL VOC. This model is available for TensorFlow only, and can only be used with inputs following the TensorFlow data format `(width, height, channels)`. # Arguments input_shape: shape of input image. format HxWxC PASCAL VOC model was trained on (512,512,3) images weights: one of 'pascal_voc' (pre-trained on pascal voc) or None (random initialization) input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. classes: number of desired classes. If classes != 21, last layer is initialized randomly OS: determines input_shape/feature_extractor_output ratio. One of {8,16}. Used only for xception backbone. # Returns A Keras model instance. # Raises RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. ValueError: in case of invalid argument for `weights` or `backbone` """ if not (weights in {'pascal_voc', None}): raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `pascal_voc` ' '(pre-trained on PASCAL VOC)') if input_tensor is None: img_input = Input(shape=input_shape, name='image_input') else: img_input = input_tensor # normalize input image img_norm = Lambda(normalize, name='input_normalize')(img_input) # backbone body for feature extract x, skip_feature, backbone_len = Xception_body(img_norm, OS) # ASPP block x = ASPP_block(x, OS) # Deeplabv3+ decoder for feature projection x = Decoder_block(x, skip_feature) # Final prediction conv block x = DeeplabConv2D(classes, (1, 1), padding='same', name='logits_semantic')(x) x = Lambda(img_resize, arguments={ 'size': (input_shape[0], input_shape[1]), 'mode': 'bilinear' }, name='pred_resize')(x) x = Reshape((input_shape[0] * input_shape[1], classes))(x) x = Softmax(name='Predictions/Softmax')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. #if input_tensor is not None: #inputs = get_source_inputs(input_tensor) #else: #inputs = img_input model = Model(img_input, x, name='deeplabv3p_xception') # load weights if weights == 'pascal_voc': weights_path = get_file( 'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH_X, cache_subdir='models') model.load_weights(weights_path, by_name=True) return model, backbone_len
class dense2DkernelCNN(denseCNN): def __init__(self,name='',weights_f=''): self.name=name self.pams ={ 'CNN_layer_nodes' : [8], #n_filters 'CNN_kernel_size' : [3], 'CNN_padding' : ['same'], 'CNN_pool' : [False], 'share_filters' : True, 'Dense_layer_nodes': [], #does not include encoded layer 'encoded_dim' : 12, 'shape' : (4,4,3), 'channels_first' : False, 'arrange' : [], 'arrMask' : [], 'calQMask' : [], 'n_copy' : 0, # no. of copy for hi occ datasets 'loss' : '', 'optimizer' : 'adam', } self.weights_f =weights_f def init(self,printSummary=True): encoded_dim = self.pams['encoded_dim'] CNN_layer_nodes = self.pams['CNN_layer_nodes'] CNN_kernel_size = self.pams['CNN_kernel_size'] CNN_padding = self.pams['CNN_padding'] CNN_pool = self.pams['CNN_pool'] Dense_layer_nodes = self.pams['Dense_layer_nodes'] #does not include encoded layer channels_first = self.pams['channels_first'] share_filters = self.pams['share_filters'] # fix to one cnn layer for now nnodes =CNN_layer_nodes[0] #8 CNN_kernel =CNN_kernel_size[0] #3 CNN_padding=CNN_padding[0] inputs = Input(shape=self.pams['shape'], name='input_1') x = inputs x1 = Lambda(lambda x: x[:,:,:,0:1], name='lambda_1')(x) x2 = Lambda(lambda x: x[:,:,:,1:2], name='lambda_2')(x) x3 = Lambda(lambda x: x[:,:,:,2:3], name='lambda_3')(x) if share_filters: conv = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding, name='conv2d_1') x1 = conv(x1) x2 = conv(x2) x3 = conv(x3) else: x1 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x1) x2 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x2) x3 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x3) if CNN_pool[0]: x1 = MaxPooling2D( (2,2), padding='same')(x1) x2 = MaxPooling2D( (2,2), padding='same')(x2) x3 = MaxPooling2D( (2,2), padding='same')(x3) conv_vol_slice = K.int_shape(x1) x1 = Flatten(name='flatten_1')(x1) x2 = Flatten(name='flatten_2')(x2) x3 = Flatten(name='flatten_3')(x3) x = [x1,x2,x3] x = Concatenate(axis=-1,name='concat_1')(x) conv_vol = K.int_shape(x) encodedLayer = Dense(encoded_dim, activation='relu',name='encoded_vector')(x) # Instantiate Encoder Model self.encoder = Model(inputs, encodedLayer, name='encoder') if printSummary: self.encoder.summary() encoded_inputs = Input(shape=(encoded_dim,), name='decoder_input') x = encoded_inputs x = Dense(conv_vol[1], activation='relu',name='dense_2')(x) x = Reshape((conv_vol_slice[1],conv_vol_slice[2],nnodes,3,),name='reshape_1')(x) x1 = Lambda(lambda x: x[:,:,:,:,0], name='lambda_4')(x) x2 = Lambda(lambda x: x[:,:,:,:,1], name='lambda_5')(x) x3 = Lambda(lambda x: x[:,:,:,:,2], name='lambda_6')(x) if CNN_pool[0]: x1 = UpSampling2D( (2,2) )(x1) x2 = UpSampling2D( (2,2) )(x2) x3 = UpSampling2D( (2,2) )(x3) ## Use n filter here conv_t = Conv2DTranspose(nnodes, CNN_kernel, activation='relu', padding=CNN_padding, name='conv2d_transpose_1') x1 = conv_t(x1) x2 = conv_t(x2) x3 = conv_t(x3) ## Always use 1 filter conv_t2 = Conv2DTranspose(1, CNN_kernel, activation=None, padding='same', name='conv2d_transpose_2') x1 = conv_t2(x1) x2 = conv_t2(x2) x3 = conv_t2(x3) x = [x1,x2,x3] x = Concatenate(axis=-1,name='concat_2')(x) outputs = Activation('sigmoid', name='decoder_output')(x) self.decoder = Model(encoded_inputs, outputs, name='decoder') if printSummary: self.decoder.summary() self.autoencoder = Model(inputs, self.decoder(self.encoder(inputs)), name='autoencoder') if printSummary: self.autoencoder.summary() self.compileModels() CNN_layers='' if len(CNN_layer_nodes)>0: CNN_layers += '_Conv' for i,n in enumerate(CNN_layer_nodes): CNN_layers += f'_{n}x{CNN_kernel_size[i]}' if CNN_pool[i]: CNN_layers += 'pooled' Dense_layers = '' if len(Dense_layer_nodes)>0: Dense_layers += '_Dense' for n in Dense_layer_nodes: Dense_layers += f'_{n}' self.name = f'Autoencoded{CNN_layers}{Dense_layers}_Encoded_{encoded_dim}' if not self.weights_f=='': self.autoencoder.load_weights(self.weights_f)
# Decoder # ------- # Decoder는 1개 단어씩을 입력으로 받는다. decoderX = Input(batch_shape=(None, 1)) decEMB = wordEmbedding(decoderX) decLSTM1 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) decLSTM2 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) dy1, _, _ = decLSTM1(decEMB, initial_state=[eh1, ec1]) dy2, _, _ = decLSTM2(dy1, initial_state=[eh2, ec2]) decOutput = TimeDistributed(Dense(VOCAB_SIZE, activation='softmax')) outputY = decOutput(dy2) # Model # ----- model = Model([encoderX, decoderX], outputY) model.load_weights(MODEL_PATH) # Chatting용 model model_enc = Model(encoderX, [eh1, ec1, eh2, ec2]) ih1 = Input(batch_shape=(None, LSTM_HIDDEN)) ic1 = Input(batch_shape=(None, LSTM_HIDDEN)) ih2 = Input(batch_shape=(None, LSTM_HIDDEN)) ic2 = Input(batch_shape=(None, LSTM_HIDDEN)) dec_output1, dh1, dc1 = decLSTM1(decEMB, initial_state=[ih1, ic1]) dec_output2, dh2, dc2 = decLSTM2(dec_output1, initial_state=[ih2, ic2]) dec_output = decOutput(dec_output2) model_dec = Model([decoderX, ih1, ic1, ih2, ic2], [dec_output, dh1, dc1, dh2, dc2])
def build_darknet19(input_image=None, weights=None, top=True): # the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K) if input_image is None: input_image = Input(shape=(None, None, 3)) def space_to_depth_x2(x): return tf.nn.space_to_depth(x, block_size=2) # Layer 1 x = Conv2D(32, (3, 3), strides=(1, 1), padding='same', name='conv_1', use_bias=False)(input_image) x = BatchNormalization(name='norm_1')(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(2, 2))(x) # Layer 2 x = Conv2D(64, (3, 3), strides=(1, 1), padding='same', name='conv_2', use_bias=False)(x) x = BatchNormalization(name='norm_2')(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(2, 2))(x) # Layer 3 x = Conv2D(128, (3, 3), strides=(1, 1), padding='same', name='conv_3', use_bias=False)(x) x = BatchNormalization(name='norm_3')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 4 x = Conv2D(64, (1, 1), strides=(1, 1), padding='same', name='conv_4', use_bias=False)(x) x = BatchNormalization(name='norm_4')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 5 x = Conv2D(128, (3, 3), strides=(1, 1), padding='same', name='conv_5', use_bias=False)(x) x = BatchNormalization(name='norm_5')(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(2, 2))(x) # Layer 6 x = Conv2D(256, (3, 3), strides=(1, 1), padding='same', name='conv_6', use_bias=False)(x) x = BatchNormalization(name='norm_6')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 7 x = Conv2D(128, (1, 1), strides=(1, 1), padding='same', name='conv_7', use_bias=False)(x) x = BatchNormalization(name='norm_7')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 8 x = Conv2D(256, (3, 3), strides=(1, 1), padding='same', name='conv_8', use_bias=False)(x) x = BatchNormalization(name='norm_8')(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(2, 2))(x) # Layer 9 x = Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_9', use_bias=False)(x) x = BatchNormalization(name='norm_9')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 10 x = Conv2D(256, (1, 1), strides=(1, 1), padding='same', name='conv_10', use_bias=False)(x) x = BatchNormalization(name='norm_10')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 11 x = Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_11', use_bias=False)(x) x = BatchNormalization(name='norm_11')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 12 x = Conv2D(256, (1, 1), strides=(1, 1), padding='same', name='conv_12', use_bias=False)(x) x = BatchNormalization(name='norm_12')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 13 x = Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_13', use_bias=False)(x) x = BatchNormalization(name='norm_13')(x) x = LeakyReLU(alpha=0.1)(x) skip_connection = x x = MaxPooling2D(pool_size=(2, 2))(x) # Layer 14 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_14', use_bias=False)(x) x = BatchNormalization(name='norm_14')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 15 x = Conv2D(512, (1, 1), strides=(1, 1), padding='same', name='conv_15', use_bias=False)(x) x = BatchNormalization(name='norm_15')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 16 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_16', use_bias=False)(x) x = BatchNormalization(name='norm_16')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 17 x = Conv2D(512, (1, 1), strides=(1, 1), padding='same', name='conv_17', use_bias=False)(x) x = BatchNormalization(name='norm_17')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 18 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_18', use_bias=False)(x) x = BatchNormalization(name='norm_18')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 19 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_19', use_bias=False)(x) x = BatchNormalization(name='norm_19')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 20 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_20', use_bias=False)(x) x = BatchNormalization(name='norm_20')(x) x = LeakyReLU(alpha=0.1)(x) # Layer 21 skip_connection = Conv2D(64, (1, 1), strides=(1, 1), padding='same', name='conv_21', use_bias=False)(skip_connection) skip_connection = BatchNormalization(name='norm_21')(skip_connection) skip_connection = LeakyReLU(alpha=0.1)(skip_connection) skip_connection = Lambda(space_to_depth_x2)(skip_connection) x = concatenate([skip_connection, x]) # Layer 22 x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_22', use_bias=False)(x) x = BatchNormalization(name='norm_22')(x) x = LeakyReLU(alpha=0.1)(x) feature_extractor = Model(input_image, x, name='Full_YOLO_backend') if weights == 'imagenet': try: feature_extractor.load_weights( 'models/weights/full_yolo_backend.h5') except: print( "weights not found or directory does not exist, not loading imagenet weights..." ) if top == False: return feature_extractor else: # ================= # classification Framework # ================= output = Conv2D(kwargs.get('num_classes'), (1, 1))(feature_extractor.output) output = GlobalAveragePooling2D()(output) output = Softmax()(output) model = Model(inputs=feature_extractor.input, outputs=output) return model
from PIL import Image from tensorflow.keras.models import Sequential from tensorflow.keras.layers import (LSTM, Embedding, TimeDistributed, Dense, RepeatVector, Activation, Flatten, Reshape, concatenate, Dropout, BatchNormalization) from tensorflow.keras.optimizers import Adam, RMSprop from tensorflow.keras import Input, layers from tensorflow.keras.models import Model from tensorflow.keras.layers import add from tensorflow.keras.preprocessing.sequence import pad_sequences from tensorflow.keras.utils import to_categorical from utils import * inputs1 = Input(shape=(OUTPUT_DIM, )) fe1 = Dropout(0.5)(inputs1) fe2 = Dense(256, activation='relu')(fe1) inputs2 = Input(shape=(max_length, )) se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2) se2 = Dropout(0.5)(se1) se3 = LSTM(256)(se2) decoder1 = add([fe2, se3]) decoder2 = Dense(256, activation='relu')(decoder1) outputs = Dense(vocab_size, activation='softmax')(decoder2) caption_model = Model(inputs=[inputs1, inputs2], outputs=outputs) caption_model.load_weights(model_path)
def unet(): inputs = Input((image_row, image_col, image_depth)) conv11 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs) conc11 = concatenate([inputs, conv11], axis=3) conv12 = Conv2D(32, (3, 3), activation='relu', padding='same')(conc11) conc12 = concatenate([inputs, conv12], axis=3) pool1 = MaxPooling2D(pool_size=(2, 2))(conc12) conv21 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1) conc21 = concatenate([pool1, conv21], axis=3) conv22 = Conv2D(64, (3, 3), activation='relu', padding='same')(conc21) conc22 = concatenate([pool1, conv22], axis=3) pool2 = MaxPooling2D(pool_size=(2, 2))(conc22) conv31 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2) conc31 = concatenate([pool2, conv31], axis=3) conv32 = Conv2D(128, (3, 3), activation='relu', padding='same')(conc31) conc32 = concatenate([pool2, conv32], axis=3) pool3 = MaxPooling2D(pool_size=(2, 2))(conc32) conv41 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3) conc41 = concatenate([pool3, conv41], axis=3) conv42 = Conv2D(256, (3, 3), activation='relu', padding='same')(conc41) conc42 = concatenate([pool3, conv42], axis=3) drop4 = Dropout(0.5)(conv42) pool4 = MaxPooling2D(pool_size=(2, 2))(drop4) conv51 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4) conc51 = concatenate([pool4, conv51], axis=3) conv52 = Conv2D(512, (3, 3), activation='relu', padding='same')(conc51) conc52 = concatenate([pool4, conv52], axis=3) drop5 = Dropout(0.5)(conc52) up6 = concatenate([ Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(drop5), conc42 ], axis=3) conv61 = Conv2D(256, (3, 3), activation='relu', padding='same')(up6) conc61 = concatenate([up6, conv61], axis=3) conv62 = Conv2D(256, (3, 3), activation='relu', padding='same')(conc61) conc62 = concatenate([up6, conv62], axis=3) up7 = concatenate([ Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(conc62), conv32 ], axis=3) conv71 = Conv2D(128, (3, 3), activation='relu', padding='same')(up7) conc71 = concatenate([up7, conv71], axis=3) conv72 = Conv2D(128, (3, 3), activation='relu', padding='same')(conc71) conc72 = concatenate([up7, conv72], axis=3) up8 = concatenate([ Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(conc72), conv22 ], axis=3) conv81 = Conv2D(64, (3, 3), activation='relu', padding='same')(up8) conc81 = concatenate([up8, conv81], axis=3) conv82 = Conv2D(64, (3, 3), activation='relu', padding='same')(conc81) conc82 = concatenate([up8, conv82], axis=3) up9 = concatenate([ Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(conc82), conv12 ], axis=3) conv91 = Conv2D(32, (3, 3), activation='relu', padding='same')(up9) conc91 = concatenate([up9, conv91], axis=3) conv92 = Conv2D(32, (3, 3), activation='relu', padding='same')(conc91) conc92 = concatenate([up9, conv92], axis=3) conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conc92) model = Model(inputs=[inputs], outputs=[conv10]) model.summary() model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=[dice_coef, 'accuracy']) pretrained_weights = None if (pretrained_weights): model.load_weights(pretrained_weights) return model
class NAPr: def __init__(self, tss_train_file: str = None, tss_test_file: str = None, options: dict = None): """ Options """ self.opts = {"seed": 1, "n_epochs": 100, "n_batch_size": 64, "dropout_rate": 0.2, "eval_size": 0.1, "activation_function": "relu"} self.__setSeed() if options is not None: for key in options.keys(): self.opts[key] = options[key] """ Load data and setup """ if tss_test_file is not None: self.X_train, self.R_train, self.Y_train = self.__loadData(tss_train_file) self.X_test, self.R_test, self.Y_test = self.__loadData(tss_test_file) self.__oneHotEncoderSetup() self.Y_train = np.asarray( self.onehot_encoder.transform(self.label_encoder.transform(self.Y_train).reshape(-1, 1))) self.Y_test = np.asarray( self.onehot_encoder.transform(self.label_encoder.transform(self.Y_test).reshape(-1, 1))) self.stdScaler = MinMaxScaler() self.stdScaler.fit(self.X_train) self.X_train = self.stdScaler.transform(self.X_train) self.X_test = self.stdScaler.transform(self.X_test) self.stdScaler_res = MinMaxScaler() self.stdScaler_res.fit(self.R_train) self.R_train = self.stdScaler_res.transform(self.R_train) self.R_test = self.stdScaler_res.transform(self.R_test) self.X_train = np.concatenate([self.X_train, self.R_train], axis=1) self.X_test = np.concatenate([self.X_test, self.R_test], axis=1) self.X_train, self.X_val, self.Y_train, self.Y_val = train_test_split(self.X_train, self.Y_train, test_size=self.opts["eval_size"], random_state=self.opts["seed"], shuffle=True) insize = self.X_train.shape[1] outsize = len(self.Y_train[0]) class_inputs = Input(shape=(insize,)) l0 = Dropout(self.opts["dropout_rate"])(class_inputs) bn0 = BatchNormalization()(l0) ae_x = Dense(300, activation=self.opts["activation_function"])(bn0) bn1 = BatchNormalization()(ae_x) l1 = Dropout(self.opts["dropout_rate"])(bn1) l1 = Dense(200, activation=self.opts["activation_function"], name="classifier_l1")(l1) bn2 = BatchNormalization()(l1) l2 = Dropout(self.opts["dropout_rate"])(bn2) l2 = Dense(100, activation=self.opts["activation_function"], name="classifier_l2")(l2) bn3 = BatchNormalization()(l2) l2 = Dropout(self.opts["dropout_rate"])(bn3) l3 = Dense(50, activation=self.opts["activation_function"], name="classifier_l3")(l2) bn4 = BatchNormalization()(l3) l3 = Dropout(self.opts["dropout_rate"])(bn4) out = Dense(outsize, activation='softmax', name="classifier")(l3) self.model = Model([class_inputs], [out]) losses = { "classifier": "categorical_crossentropy", } self.model.compile(optimizer='adam', loss=losses, metrics=['accuracy']) def train(self, checkpoint_path: str, name: str, save_results: bool = False): event_dict_file = str(checkpoint_path) + "/" + str(name) + "_napr_onehotdict.json" with open(str(event_dict_file), 'w') as outfile: json.dump(self.one_hot_dict, outfile) with open(checkpoint_path + "/" + name + "_napr_model.json", 'w') as f: f.write(self.model.to_json()) ckpt_file = str(checkpoint_path) + "/" + str(name) + "_napr_weights.hdf5" checkpoint = ModelCheckpoint(ckpt_file, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') hist = self.model.fit(self.X_train, self.Y_train, batch_size=self.opts["n_batch_size"], epochs=self.opts["n_epochs"], shuffle=True, validation_data=([self.X_val], [self.Y_val]), callbacks=[self.EvaluationCallback(self.X_test, self.Y_test), checkpoint]) joblib.dump(self.stdScaler, str(checkpoint_path) + "/" + str(name) + "_napr_stdScaler.pkl") joblib.dump(self.stdScaler_res, str(checkpoint_path) + "/" + str(name) + "_napr_stdScaler_res.pkl") if save_results: results_file = str(checkpoint_path) + "/" + str(name) + "_napr_results.json" with open(str(results_file), 'w') as outfile: json.dump(str(hist.history), outfile) def __oneHotEncoderSetup(self): """ Events to One Hot""" events = np.unique(self.Y_train) self.label_encoder = LabelEncoder() integer_encoded = self.label_encoder.fit_transform(events) integer_encoded = integer_encoded.reshape(len(integer_encoded), 1) self.onehot_encoder = OneHotEncoder(sparse=False) self.onehot_encoder.fit(integer_encoded) self.one_hot_dict = {} for event in events: self.one_hot_dict[event] = list(self.onehot_encoder.transform([self.label_encoder.transform([event])])[0]) @staticmethod def __loadData(file: str): x, r, y = [], [], [] with open(file) as json_file: tss = json.load(json_file) for sample in tss: if sample["label"] is not None: x.append(list(itertools.chain(sample["tss"][0], sample["tss"][1], sample["tss"][2]))) r.append(list(sample["tss"][3])) y.append(sample["label"]) return np.array(x), np.array(r), np.array(y) def __setSeed(self): seed(self.opts["seed"]) tf.random.set_seed(self.opts["seed"]) def loadModel(self, path: str, name: str): with open(path + "/" + name + "_napr_model.json", 'r') as f: self.model = model_from_json(f.read()) self.model.load_weights(path + "/" + name + "_napr_weights.hdf5") with open(path + "/" + name + "_napr_onehotdict.json", 'r') as f: self.one_hot_dict = json.load(f) self.stdScaler = joblib.load(path + "/" + name + "_napr_stdScaler.pkl") self.stdScaler_res = joblib.load(path + "/" + name + "_napr_stdScaler_res.pkl") def __intToEvent(self, value: int): one_hot = list(np.eye(len(self.one_hot_dict.keys()))[value]) for k, v in self.one_hot_dict.items(): if str(v) == str(one_hot): return k def predict(self, tss: list): """ Predict from a list TimedStateSamples :param tss: list<TimedStateSamples> :return: tuple (DREAM-NAP output, translated next event) """ if not isinstance(tss, list) or not isinstance(tss[0], TimedStateSample): raise ValueError("Input is not a list with TimedStateSample") preds = [] next_events = [] for sample in tss: features = [list( itertools.chain(sample.export()["tss"][0], sample.export()["tss"][1], sample.export()["tss"][2]))] features = self.stdScaler.transform(features) r = [list(sample.export()["tss"][3])] r = self.stdScaler_res.transform(r) features = np.concatenate([features, r], axis=1) pred = np.argmax(self.model.predict(features), axis=1) preds.append(pred[0]) for p in pred: next_events.append(self.__intToEvent(p)) return preds, next_events """ Callback """ class EvaluationCallback(Callback): def __init__(self, X_test: np.ndarray, Y_test: np.ndarray): super().__init__() self.X_test = X_test self.Y_test = Y_test self.Y_test_int = np.argmax(self.Y_test, axis=1) self.test_accs = [] self.losses = [] def on_train_begin(self, logs: dict = {}): self.test_accs = [] self.losses = [] def on_epoch_end(self, epoch: int, logs: dict = {}): y_pred = self.model.predict(self.X_test) y_pred = y_pred.argmax(axis=1) test_acc = accuracy_score(self.Y_test_int, y_pred, normalize=True) test_loss, _ = self.model.evaluate(self.X_test, self.Y_test) precision, recall, fscore, _ = precision_recall_fscore_support(self.Y_test_int, y_pred, average='weighted', pos_label=None) auc = multiclass_roc_auc_score(self.Y_test_int, y_pred, average="weighted") logs['test_acc'] = test_acc logs['test_prec_weighted'] = precision logs['test_rec_weighted'] = recall logs['test_loss'] = test_loss logs['test_fscore_weighted'] = fscore logs['test_auc_weighted'] = auc precision, recall, fscore, support = precision_recall_fscore_support(self.Y_test_int, y_pred, average='macro', pos_label=None) auc = multiclass_roc_auc_score(self.Y_test_int, y_pred, average="macro") logs['test_prec_mean'] = precision logs['test_rec_mean'] = recall logs['test_fscore_mean'] = fscore logs['test_auc_mean'] = auc
input_path_4 = "/home/mukesh/Documents/IISc/DATA/4_1" # import random # random.shuffle(input_path_0) test_files_0 = glob.glob(os.path.join(input_path_0, "*.jpeg")) test_files_1 = glob.glob(os.path.join(input_path_1, "*.jpeg")) test_files_2 = glob.glob(os.path.join(input_path_2, "*.jpeg")) test_files_3 = glob.glob(os.path.join(input_path_3, "*.jpeg")) test_files_4 = glob.glob(os.path.join(input_path_4, "*.jpeg")) print(len(test_files_0)) print(len(test_files_1)) print(len(test_files_2)) print(len(test_files_3)) print(len(test_files_4)) model2.load_weights('Weights_multi.034-0.312.hdf5') test_files = [] test_files += test_files_1 test_files += test_files_2 test_files += test_files_3 test_files += test_files_4 print(len(test_files)) import random random.shuffle(test_files_0) random.shuffle(test_files) PT = [] GT = [] predict = []
class CommVAE1hot(object): """ in_dim : block length latent_dim : encoding dimension (half of number of channel uses) h_dim : number of hidden layers obj_fn : objective function with to optimize over n0 : noise power (over all components) sigma2 : prior variance (per component) """ def __init__(self, in_dim=None, latent_dim=None, h_dim=None, obj_fn='RBF', n0=1.0, sigma2=1.0): self.in_dim = in_dim self.latent_dim = latent_dim self.n0 = n0 self.sigma2 = sigma2 self.h_dim = h_dim self.obj_fn = obj_fn if self.in_dim and self.latent_dim: self.make_model() def make_model(self): # Input layer self.inputs = Input(shape=(self.in_dim, ), name="enc_in") # Hidden Layers x = self.inputs if self.h_dim is not None: for (i, d) in enumerate(self.h_dim): x = Dense(d, activation='relu', name="enc_l{}".format(i))(x) # Mean and Variance self.z_mean = Dense(self.latent_dim, name="z_mean")(x) # Channel self.z = Lambda(self.channel, output_shape=(self.latent_dim, ), name="z")(self.z_mean) # Encoder model self.encoder = Model(self.inputs, [self.z_mean, self.z], name="encoder") # Decoder self.latent_inputs = Input(shape=(self.latent_dim, ), name="z_sample") # Hidden layers x = self.latent_inputs if self.h_dim is not None: for (i, d) in enumerate(self.h_dim[::-1]): x = Dense(d, activation='relu', name="dec_l{}".format(i))(x) self.dec_outputs = Dense(self.in_dim, activation='softmax', name="decoder_out")(x) # Decoder model self.decoder = Model(self.latent_inputs, self.dec_outputs, name="decoder") # VAE self.outputs = self.decoder(self.encoder(self.inputs)[1]) self.model = Model(self.inputs, self.outputs, name="VAE") # Losses self.recon_loss = categorical_crossentropy(self.inputs, self.outputs) if self.obj_fn == 'AWGN': # print( "Model with AWGN ") sig_pow = 0.5 * 1.0 / self.sigma2 * K.sum(K.square(self.z_mean), axis=-1) noise_term = 0.5 * self.latent_dim * ( (self.n0 / self.latent_dim) / self.sigma2 - 1.0 - K.log( (self.n0 / self.latent_dim) / self.sigma2)) self.kl_loss = sig_pow + noise_term elif self.obj_fn == 'RBF': # print( "Model with RBF") sig_pow = 0.5 * 1.0 / self.sigma2 * K.sum(K.square(self.z_mean), axis=-1) noise_term = 0.5 * self.latent_dim * ( (self.n0 / self.latent_dim) / self.sigma2 - 1.0 - K.log( (self.n0 / self.latent_dim) / self.sigma2)) rbf_term = K.log(1.0 + 0.5 * self.latent_dim / self.n0 * sig_pow) self.kl_loss = sig_pow + noise_term - rbf_term else: raise NotImplementedError("Unknown obj_fn: {}".format(self.obj_fn)) self.vae_loss = K.mean(self.recon_loss + self.kl_loss) self.model.add_loss(self.vae_loss) self.model.compile(optimizer='adam') # self.model.compile( optimizer=tf.train.AdamOptimizer(learning_rate=0.01)) def channel(self, zMean): batch = K.shape(zMean)[0] # dims = K.shape( zMean )[1] epsilon = K.random_normal(shape=(batch, self.latent_dim)) return zMean + np.sqrt(self.n0 / self.latent_dim) * epsilon def fit(self, x_train, epochs=10, batch_size=128, validation_data=None, verbose=0, callbacks=None): train_log = self.model.fit(x_train, epochs=epochs, batch_size=batch_size, validation_data=validation_data, verbose=verbose, callbacks=callbacks) return train_log.history def encode(self, data): return self.encoder.predict(data) def decode(self, data): return self.decoder.predict(data) def analysis(self): xTest = np.eye(self.in_dim) enc_mu, enc_z = self.encode(xTest) dec_mu = self.decode(enc_mu) dec_z = self.decode(enc_z) chDim = self.latent_dim // 2 f = plt.figure(figsize=(5 * chDim, 9)) for i in range(chDim): ax1 = plt.subplot(2, chDim, i + 1) ax1.scatter(enc_mu[:, i], enc_mu[:, i + chDim], c=np.arange(self.in_dim)) for j in range(self.in_dim): ax1.annotate(j, (enc_mu[j, i], enc_mu[j, i + chDim])) ax1.set_title("TX Symbols n = {}".format(i + 1)) def save_model(self, fileprefix): with open(fileprefix + ".dil", "wb") as obj: dill.dump( { 'in_dim': self.in_dim, 'latent_dim': self.latent_dim, 'n0': self.n0, 'sigma2': self.sigma2, 'h_dim': self.h_dim, 'obj_fn': self.obj_fn }, obj) self.model.save_weights(fileprefix + ".h5") def load_model(self, fileprefix): with open(fileprefix + ".dil", "rb") as obj: config = dill.load(obj) self.in_dim = config['in_dim'] self.latent_dim = config['latent_dim'] self.n0 = config['n0'] self.h_dim = config['h_dim'] self.obj_fn = config['obj_fn'] if 'sigma2' in config: # For backward compatability self.sigma2 = config['sigma2'] else: self.sigma2 = 1.0 self.make_model() self.model.load_weights(fileprefix + ".h5")
def NASNet(input_shape=None, penultimate_filters=4032, nb_blocks=6, stem_filters=96, initial_reduction=True, skip_reduction_layer_input=True, use_auxiliary_branch=False, filters_multiplier=2, dropout=0.5, weight_decay=5e-5, include_top=True, weights=None, input_tensor=None, pooling=None, classes=1000, default_size=None, activation='softmax'): """Instantiates a NASNet architecture. Note that only TensorFlow is supported for now, therefore it only works with the data format `image_data_format='channels_last'` in your Keras config at `~/.keras/keras.json`. # Arguments input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(331, 331, 3)` for NASNetLarge or `(224, 224, 3)` for NASNetMobile It should have exactly 3 inputs channels, and width and height should be no smaller than 32. E.g. `(224, 224, 3)` would be one valid value. penultimate_filters: number of filters in the penultimate layer. NASNet models use the notation `NASNet (N @ P)`, where: - N is the number of blocks - P is the number of penultimate filters nb_blocks: number of repeated blocks of the NASNet model. NASNet models use the notation `NASNet (N @ P)`, where: - N is the number of blocks - P is the number of penultimate filters stem_filters: number of filters in the initial stem block initial_reduction: Whether to perform the reduction step at the beginning end of the network. Set to `True` for CIFAR models. skip_reduction_layer_input: Determines whether to skip the reduction layers when calculating the previous layer to connect to. use_auxiliary_branch: Whether to use the auxiliary branch during training or evaluation. filters_multiplier: controls the width of the network. - If `filters_multiplier` < 1.0, proportionally decreases the number of filters in each layer. - If `filters_multiplier` > 1.0, proportionally increases the number of filters in each layer. - If `filters_multiplier` = 1, default number of filters from the paper are used at each layer. dropout: dropout rate weight_decay: l2 regularization weight include_top: whether to include the fully-connected layer at the top of the network. weights: `None` (random initialization) or `imagenet` (ImageNet weights) input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. default_size: specifies the default image size of the model activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. """ if K.backend() != 'tensorflow': raise RuntimeError('Only Tensorflow backend is currently supported, ' 'as other backends do not support ' 'separable convolution.') if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as ImageNet with `include_top` ' 'as true, `classes` should be 1000') if default_size is None: default_size = 331 # Determine proper input shape and default size. input_shape = _obtain_input_shape(input_shape, default_size=default_size, min_size=32, data_format=K.image_data_format(), require_flatten=include_top or weights) if K.image_data_format() != 'channels_last': warnings.warn('The NASNet family of models is only available ' 'for the input data format "channels_last" ' '(width, height, channels). ' 'However your settings specify the default ' 'data format "channels_first" (channels, width, height).' ' You should set `image_data_format="channels_last"` ' 'in your Keras config located at ~/.keras/keras.json. ' 'The model being returned right now will expect inputs ' 'to follow the "channels_last" data format.') K.set_image_data_format('channels_last') old_data_format = 'channels_first' else: old_data_format = None if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be " \ "divisible by 24." channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 filters = penultimate_filters // 24 if initial_reduction: x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1', kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(img_input) else: x = Conv2D(stem_filters, (3, 3), strides=(1, 1), padding='same', use_bias=False, name='stem_conv1', kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(img_input) x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, name='stem_bn1')(x) p = None if initial_reduction: # imagenet / mobile mode x, p = _reduction_A(x, p, filters // (filters_multiplier ** 2), weight_decay, id='stem_1') x, p = _reduction_A(x, p, filters // filters_multiplier, weight_decay, id='stem_2') for i in range(nb_blocks): x, p = _normal_A(x, p, filters, weight_decay, id='%d' % i) x, p0 = _reduction_A(x, p, filters * filters_multiplier, weight_decay, id='reduce_%d' % nb_blocks) p = p0 if not skip_reduction_layer_input else p for i in range(nb_blocks): x, p = _normal_A(x, p, filters * filters_multiplier, weight_decay, id='%d' % (nb_blocks + i + 1)) auxiliary_x = None if not initial_reduction: # imagenet / mobile mode if use_auxiliary_branch: auxiliary_x = _add_auxiliary_head(x, classes, weight_decay, pooling, include_top, activation) x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, weight_decay, id='reduce_%d' % (2 * nb_blocks)) if initial_reduction: # CIFAR mode if use_auxiliary_branch: auxiliary_x = _add_auxiliary_head(x, classes, weight_decay, pooling, include_top, activation) p = p0 if not skip_reduction_layer_input else p for i in range(nb_blocks): x, p = _normal_A(x, p, filters * filters_multiplier ** 2, weight_decay, id='%d' % (2 * nb_blocks + i + 1)) x = Activation('relu')(x) if include_top: x = GlobalAveragePooling2D()(x) x = Dropout(dropout)(x) x = Dense(classes, activation=activation, kernel_regularizer=l2(weight_decay), name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. if use_auxiliary_branch: model = Model(inputs, [x, auxiliary_x], name='NASNet_with_auxiliary') else: model = Model(inputs, x, name='NASNet') # load weights if weights == 'imagenet': if default_size == 224: # mobile version if include_top: if use_auxiliary_branch: weight_path = NASNET_MOBILE_WEIGHT_PATH_WITH_AUXULARY model_name = 'nasnet_mobile_with_aux.h5' else: weight_path = NASNET_MOBILE_WEIGHT_PATH model_name = 'nasnet_mobile.h5' else: if use_auxiliary_branch: weight_path = NASNET_MOBILE_WEIGHT_PATH_WITH_AUXULARY_NO_TOP model_name = 'nasnet_mobile_with_aux_no_top.h5' else: weight_path = NASNET_MOBILE_WEIGHT_PATH_NO_TOP model_name = 'nasnet_mobile_no_top.h5' weights_file = get_file(model_name, weight_path, cache_subdir='models') model.load_weights(weights_file, by_name=True) elif default_size == 331: # large version if include_top: if use_auxiliary_branch: weight_path = NASNET_LARGE_WEIGHT_PATH_WITH_auxiliary model_name = 'nasnet_large_with_aux.h5' else: weight_path = NASNET_LARGE_WEIGHT_PATH model_name = 'nasnet_large.h5' else: if use_auxiliary_branch: weight_path = NASNET_LARGE_WEIGHT_PATH_WITH_auxiliary_NO_TOP model_name = 'nasnet_large_with_aux_no_top.h5' else: weight_path = NASNET_LARGE_WEIGHT_PATH_NO_TOP model_name = 'nasnet_large_no_top.h5' weights_file = get_file(model_name, weight_path, cache_subdir='models') model.load_weights(weights_file, by_name=True) else: raise ValueError('ImageNet weights can only be loaded on NASNetLarge ' 'or NASNetMobile') if old_data_format: K.set_image_data_format(old_data_format) return model
def ShuffleNet(include_top=True, input_tensor=None, scale_factor=1.0, pooling=None, input_shape=None, groups=1, weights='imagenet', num_shuffle_units=[3, 7, 3], bottleneck_ratio=0.25, classes=1000, **kwargs): """ ShuffleNet implementation for Keras 2 ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices Xiangyu Zhang, Xinyu Zhou, Mengxiao Lin, Jian Sun https://arxiv.org/pdf/1707.01083.pdf Note that only TensorFlow is supported for now, therefore it only works with the data format `image_data_format='channels_last'` in your Keras config at `~/.keras/keras.json`. Parameters ---------- include_top: bool(True) whether to include the fully-connected layer at the top of the network. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. scale_factor: scales the number of output channels input_shape: pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. groups: int number of groups per channel num_shuffle_units: list([3,7,3]) number of stages (list length) and the number of shufflenet units in a stage beginning with stage 2 because stage 1 is fixed e.g. idx 0 contains 3 + 1 (first shuffle unit in each stage differs) shufflenet units for stage 2 idx 1 contains 7 + 1 Shufflenet Units for stage 3 and idx 2 contains 3 + 1 Shufflenet Units bottleneck_ratio: bottleneck ratio implies the ratio of bottleneck channels to output channels. For example, bottleneck ratio = 1 : 4 means the output feature map is 4 times the width of the bottleneck feature map. classes: int(1000) number of classes to predict Returns ------- A Keras model instance References ---------- - [ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices] (http://www.arxiv.org/pdf/1707.01083.pdf) """ if K.backend() != 'tensorflow': raise RuntimeError('Only TensorFlow backend is currently supported, ' 'as other backends do not support ') name = "ShuffleNet_%.2gX_g%d_br_%.2g_%s" % ( scale_factor, groups, bottleneck_ratio, "".join( [str(x) for x in num_shuffle_units])) input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=28, require_flatten=include_top, data_format=K.image_data_format()) out_dim_stage_two = {1: 144, 2: 200, 3: 240, 4: 272, 8: 384} if groups not in out_dim_stage_two: raise ValueError("Invalid number of groups.") if pooling not in ['max', 'avg', None]: raise ValueError("Invalid value for pooling.") if not (float(scale_factor) * 4).is_integer(): raise ValueError("Invalid value for scale_factor. Should be x over 4.") exp = np.insert(np.arange(0, len(num_shuffle_units), dtype=np.float32), 0, 0) out_channels_in_stage = 2**exp out_channels_in_stage *= out_dim_stage_two[ groups] # calculate output channels for each stage out_channels_in_stage[0] = 24 # first stage has always 24 output channels out_channels_in_stage *= scale_factor out_channels_in_stage = out_channels_in_stage.astype(int) if input_tensor is None: img_input = Input(shape=input_shape) else: #if not K.is_keras_tensor(input_tensor): #img_input = Input(tensor=input_tensor, shape=input_shape) #else: #img_input = input_tensor img_input = input_tensor # create shufflenet architecture x = YoloConv2D(filters=out_channels_in_stage[0], kernel_size=(3, 3), padding='same', use_bias=False, strides=(2, 2), activation="relu", name="conv1")(img_input) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same', name="maxpool1")(x) # create stages containing shufflenet units beginning at stage 2 for stage in range(0, len(num_shuffle_units)): repeat = num_shuffle_units[stage] x = _block(x, out_channels_in_stage, repeat=repeat, bottleneck_ratio=bottleneck_ratio, groups=groups, stage=stage + 2) if include_top: #x = Dense(units=classes, name="fc")(x) #x = Activation('softmax', name='softmax')(x) x = GlobalAveragePooling2D(name='global_avg_pool')(x) x = Dense(units=classes, activation='softmax', use_bias=True, name='Logits')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D(name='global_avg_pool')(x) elif pooling == 'max': x = GlobalMaxPooling2D(name='global_max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs=inputs, outputs=x, name=name) # Load weights. if weights == 'imagenet': if K.image_data_format() == 'channels_first': raise ValueError('Weights for "channels_first" format ' 'are not available.') if include_top: model_name = ('shufflenet_weights_tf_dim_ordering_tf_kernels_' + str(alpha) + '_' + str(rows) + '.h5') weigh_path = BASE_WEIGHT_PATH + model_name weights_path = get_file(model_name, weigh_path, cache_subdir='models') else: model_name = ('shufflenet_weights_tf_dim_ordering_tf_kernels_' + str(alpha) + '_' + str(rows) + '_no_top' + '.h5') weigh_path = BASE_WEIGHT_PATH + model_name weights_path = get_file(model_name, weigh_path, cache_subdir='models') model.load_weights(weights_path) elif weights is not None: model.load_weights(weights) return model
C.num_rois, nb_classes=len(classes_count), trainable=True) ## build model for each network Model(input, output) model_rpn = Model(img_input, rpn[:2]) ## because rpn[2] is base_layers(input) model_classifier = Model([img_input, roi_input], classifier) model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: ##base_net_weight not in config.py but defined above # print('loading weights from {}'.format('finalmodel_frcnn_cat.h5')) # model_rpn.load_weights('finalmodel_frcnn_cat.h5', by_name=True) # model_classifier.load_weights('finalmodel_frcnn_cat.h5', by_name = True) print('loading weights from {}'.format( 'vgg16_weights_tf_dim_ordering_tf_kernels.h5')) model_rpn.load_weights('vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name=True) model_classifier.load_weights( 'vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name=True) except: print( 'Could not load pretrained model weights. Weights can be found in the keras application folder \ https://github.com/fchollet/keras/tree/master/keras/applications') ## optimizer for each network optimizer = Adam(lr=1e-6) optimizer_classifier = Adam(lr=1e-6) model_rpn.compile( optimizer=optimizer, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_reg(num_anchors)]) model_classifier.compile(
class VGGish(KerasModelContainer): """ KerasModelContainer for VGGish model Jort F. Gemmeke et al. Audio Set: An ontology and human-labeled dataset for audio events International Conference on Acoustics, Speech, and Signal Processing. New Orleans, LA, 2017. Notes ----- https://research.google.com/audioset/ Based on vggish-keras https://pypi.org/project/vggish-keras/ Parameters ---------- n_frames_cnn : int or None, default=96 Length of the input (number of frames of each sequence). n_freq_cnn : int, default=64 Number of frequency bins. The model's input has shape (n_frames, n_freqs). n_classes : int, default=10 Number of classes (dimmension output). n_channels : int, default=0 Number of input channels 0 : mono signals. Input shape = (n_frames_cnn, n_freq_cnn) 1 : mono signals. Input shape = (n_frames_cnn, n_freq_cnn, 1) 2 : stereo signals. Input shape = (n_frames_cnn, n_freq_cnn, 2) n > 2 : multi-representations. Input shape = (n_frames_cnn, n_freq_cnn, n_channels) embedding_size : int, default=128 Number of units in the embeddings layer. pooling : {'avg', max}, default='avg' Use AveragePooling or Maxpooling. include_top : bool, default=False Include fully-connected layers. compress : bool, default=False Apply PCA. Attributes ---------- model : keras.models.Model Keras model. Examples -------- >>> from dcase_models.model.models import VGGish >>> model_container = VGGish() >>> model_container.model.summary() _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= input (InputLayer) (None, 96, 64) 0 _________________________________________________________________ lambda (Lambda) (None, 96, 64, 1) 0 _________________________________________________________________ conv1 (Conv2D) (None, 96, 64, 64) 640 _________________________________________________________________ pool1 (MaxPooling2D) (None, 48, 32, 64) 0 _________________________________________________________________ conv2 (Conv2D) (None, 48, 32, 128) 73856 _________________________________________________________________ pool2 (MaxPooling2D) (None, 24, 16, 128) 0 _________________________________________________________________ conv3/conv3_1 (Conv2D) (None, 24, 16, 256) 295168 _________________________________________________________________ conv3/conv3_2 (Conv2D) (None, 24, 16, 256) 590080 _________________________________________________________________ pool3 (MaxPooling2D) (None, 12, 8, 256) 0 _________________________________________________________________ conv4/conv4_1 (Conv2D) (None, 12, 8, 512) 1180160 _________________________________________________________________ conv4/conv4_2 (Conv2D) (None, 12, 8, 512) 2359808 _________________________________________________________________ pool4 (MaxPooling2D) (None, 6, 4, 512) 0 _________________________________________________________________ global_average_pooling2d_1 ( (None, 512) 0 ================================================================= Total params: 4,499,712 Trainable params: 4,499,712 Non-trainable params: 0 _________________________________________________________________ """ def __init__(self, model=None, model_path=None, metrics=['classification'], n_frames_cnn=96, n_freq_cnn=64, n_classes=10, n_channels=0, embedding_size=128, pooling='avg', include_top=False, compress=False): self.n_frames_cnn = n_frames_cnn self.n_freq_cnn = n_freq_cnn self.n_classes = n_classes self.n_channels = n_channels self.embedding_size = embedding_size self.pooling = pooling self.include_top = include_top self.compress = compress super().__init__( model=model, model_path=model_path, model_name='VGGish', metrics=metrics ) class Postprocess(Layer): """ Keras layer that applies PCA and quantizes the ouput. Based on vggish-keras https://pypi.org/project/vggish-keras/ """ def __init__(self, output_shape=None, **kw): self.emb_shape = output_shape super().__init__(**kw) def build(self, input_shape): input_shape = tuple(int(x) for x in tuple(input_shape)[1:]) emb_shape = (self.emb_shape,) if self.emb_shape else input_shape self.pca_matrix = self.add_weight( name='pca_matrix', shape=emb_shape + input_shape, initializer='uniform') self.pca_means = self.add_weight( name='pca_means', shape=input_shape + (1,), initializer='uniform') def call(self, x): # Apply PCA. # - Embeddings come in as [batch_size, embedding_size]. # - Transpose to [embedding_size, batch_size]. # - Subtract pca_means column vector from each column. # - Premultiply by PCA matrix of shape [output_dims, input_dims] # where both are are equal to embedding_size in our case. # - Transpose result back to [batch_size, embedding_size]. x = K.dot(self.pca_matrix, (K.transpose(x) - self.pca_means)) x = K.transpose(x) # Quantize by: # - clipping to [min, max] range # - convert to 8-bit in range [0.0, 255.0] # - cast 8-bit float to uint8 QUANTIZE_MIN_VAL = -2.0 QUANTIZE_MAX_VAL = +2.0 x = clip_by_value(x, QUANTIZE_MIN_VAL, QUANTIZE_MAX_VAL) x = ((x - QUANTIZE_MIN_VAL) * (255.0 / (QUANTIZE_MAX_VAL - QUANTIZE_MIN_VAL))) return K.cast(x, 'uint8') def build(self): """ Builds the VGGish Keras model. """ if self.n_channels == 0: inputs = Input(shape=(self.n_frames_cnn, self.n_freq_cnn), dtype='float32', name='input') x = Lambda( lambda x: K.expand_dims(x, -1), name='lambda' )(inputs) else: inputs = Input( shape=(self.n_frames_cnn, self.n_freq_cnn, self.n_channels), dtype='float32', name='input' ) x = Lambda(lambda x: x, name='lambda')(inputs) # setup layer params conv = partial(Conv2D, kernel_size=(3, 3), strides=( 1, 1), activation='relu', padding='same') maxpool = partial(MaxPooling2D, pool_size=(2, 2), strides=(2, 2), padding='same') # Block 1 x = conv(64, name='conv1')(x) x = maxpool(name='pool1')(x) # Block 2 x = conv(128, name='conv2')(x) x = maxpool(name='pool2')(x) # Block 3 x = conv(256, name='conv3/conv3_1')(x) x = conv(256, name='conv3/conv3_2')(x) x = maxpool(name='pool3')(x) # Block 4 x = conv(512, name='conv4/conv4_1')(x) x = conv(512, name='conv4/conv4_2')(x) x = maxpool(name='pool4')(x) if self.include_top: dense = partial(Dense, activation='relu') # FC block x = Flatten(name='flatten_')(x) x = dense(4096, name='fc1/fc1_1')(x) x = dense(4096, name='fc1/fc1_2')(x) x = dense(self.embedding_size, name='fc2')(x) if self.compress: x = self.Postprocess()(x) else: globalpool = ( GlobalAveragePooling2D() if self.pooling == 'avg' else GlobalMaxPooling2D() if self.pooling == 'max' else None) if globalpool: x = globalpool(x) # Create model self.model = Model(inputs, x, name='vggish_model') super().build() def load_pretrained_model_weights(self, weights_folder='./pretrained_weights'): """ Loads pretrained weights to self.model weights. Parameters ---------- weights_folder : str Path to load the weights file """ basepath = os.path.dirname(__file__) weights_file = self.model_name + '.hdf5' weights_path = os.path.join(basepath, weights_folder, weights_file) if not os.path.isfile(weights_path): self.download_pretrained_weights() self.model.load_weights(weights_path, by_name=True) def download_pretrained_weights(self, weights_folder='./pretrained_weights'): """ Download pretrained weights from: https://github.com/DTaoo/VGGish https://drive.google.com/file/d/1mhqXZ8CANgHyepum7N4yrjiyIg6qaMe6/view Code based on: https://github.com/beasteers/VGGish/blob/master/vggish_keras/download_helpers/download_weights.py Parameters ---------- weights_folder : str Path to save the weights file """ import requests import tqdm DRIVE_URL = 'https://drive.google.com/uc?id={id}&export=download' DRIVE_CONFIRM_URL = ('https://drive.google.com/uc?id={id}&export' '=download&confirm={confirm}') basepath = os.path.dirname(__file__) weights_file = self.model_name + '.hdf5' weights_path = os.path.join(basepath, weights_folder, weights_file) # gdrive_id = '1mhqXZ8CANgHyepum7N4yrjiyIg6qaMe6' # This file includes PCA weights gdrive_id = '1QbMNrhu4RBUO6hIcpLqgeuVye51XyMKM' if not os.path.isfile(weights_path): print('Downloading weights...') sess = requests.Session() r = sess.get(DRIVE_URL.format(id=gdrive_id), stream=True) # check for google virus message confirm = next( (v for k, v in r.cookies.get_dict().items() if 'download_warning_' in k), None) if confirm: # print('Using confirmation code {}...'.format(confirm)) r = sess.get( DRIVE_CONFIRM_URL.format(id=gdrive_id, confirm=confirm), stream=True) # download w/ progress bar chunk_size = 1024 unit = 1024 ** 2 with open(weights_path, 'wb') as f: pbar = tqdm.tqdm( unit='mb', leave=False, total=int( r.headers.get('Content-Length', 0)) / unit or None) for chunk in r.iter_content(chunk_size=chunk_size): if chunk: # filter out keep-alive new chunks pbar.update(len(chunk) / unit) f.write(chunk) print('Done!')
def InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the Inception v3 architecture. Optionally loads weights pre-trained on ImageNet. Note that when using TensorFlow, for best performance you should set `image_data_format="channels_last"` in your Keras config at ~/.keras/keras.json. The model and the weights are compatible with both TensorFlow and Theano. The data format convention used by the model is the one specified in your Keras config file. Note that the default input image size for this model is 299x299. Arguments: include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet). input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(299, 299, 3)` (with `channels_last` data format) or `(3, 299, 299)` (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 139. E.g. `(150, 150, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. Returns: A Keras model instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape # input_shape = _obtain_input_shape( # input_shape, # default_size=299, # min_size=139, # data_format=K.image_data_format(), # include_top=include_top) input_shape = _obtain_input_shape(input_shape, default_size=299, min_size=139, data_format=K.image_data_format(), require_flatten=True) if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = Input(tensor=input_tensor, shape=input_shape) if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') x = conv2d_bn(x, 32, 3, 3, padding='valid') x = conv2d_bn(x, 64, 3, 3) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv2d_bn(x, 80, 1, 1, padding='valid') x = conv2d_bn(x, 192, 3, 3, padding='valid') x = MaxPooling2D((3, 3), strides=(2, 2))(x) # mixed 0, 1, 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 32, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed0') # mixed 1: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed1') # mixed 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed2') # mixed 3: 17 x 17 x 768 branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') # mixed 4: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 128, 1, 1) branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 128, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed4') # mixed 5, 6: 17 x 17 x 768 for i in range(2): branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 160, 1, 1) branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 160, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed' + str(5 + i)) # mixed 7: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 192, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed7') # mixed 8: 8 x 8 x 1280 branch3x3 = conv2d_bn(x, 192, 1, 1) branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid') branch7x7x3 = conv2d_bn(x, 192, 1, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate([branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') # mixed 9: 8 x 8 x 2048 for i in range(2): branch1x1 = conv2d_bn(x, 320, 1, 1) branch3x3 = conv2d_bn(x, 384, 1, 1) branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) branch3x3dbl = conv2d_bn(x, 448, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed' + str(9 + i)) if include_top: # Classification block x = GlobalAveragePooling2D(name='avg_pool')(x) x = Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='inception_v3') # load weights if weights == 'imagenet': if K.image_data_format() == 'channels_first': if K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') if include_top: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models', md5_hash='9a0d58056eeedaa3f26cb7ebd46da564') else: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models', md5_hash='bcbd6486424b2319ff4ef7d526e38f63') model.load_weights(weights_path) if K.backend() == 'theano': convert_all_kernels_in_model(model) return model