def VGGish(load_weights=True, weights='audioset',
input_tensor=None, input_shape=None,
out_dim=None, include_top=True, pooling='avg'):
if weights not in {'audioset', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `audioset` '
'(pre-training on audioset).')
if out_dim is None:
out_dim = 128
# input shape
if input_shape is None:
input_shape = (496, 64, 1)
if input_tensor is None:
aud_input = Input(shape=input_shape, name='input_1')
else:
if not K.is_keras_tensor(input_tensor):
aud_input = Input(tensor=input_tensor, shape=input_shape, name='input_1')
else:
aud_input = input_tensor
# Block 1
x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv1')(aud_input)
x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool1')(x)
# Block 2
x = Conv2D(128, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool2')(x)
# Block 3
x = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_1')(x)
x = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool3')(x)
# Block 4
x = Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_1')(x)
x = Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool4')(x)
if include_top:
# FC block
x = Flatten(name='flatten_')(x)
x = Dense(4096, activation='relu', name='vggish_fc1/fc1_1')(x)
x = Dense(4096, activation='relu', name='vggish_fc1/fc1_2')(x)
x = Dense(out_dim, activation='relu', name='vggish_fc2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = aud_input
# Create model.
model = Model(inputs, x, name='VGGish')
# load weights
if load_weights:
if weights == 'audioset':
if include_top:
model.load_weights(WEIGHTS_PATH_TOP)
else:
print("POGGERS")
model.load_weights(WEIGHTS_PATH)
else:
print("failed to load weights")
return model
imgs += Path(args.dir).files('*.jpeg')
elif args.img[0] is not None:
print("Loading images from path(s) : ", args.img)
imgs = args.img
else:
raise RuntimeError('Either -dir or -img arguments must be passed as argument')
with tf.device('/CPU:0'):
base_model = MobileNet((None, None, 3), alpha=1, include_top=False, pooling='avg', weights=None)
x = Dropout(0.75)(base_model.output)
x = Dense(10, activation='softmax')(x)
model = Model(base_model.input, x)
model.load_weights('weights/mobilenet_weights.h5')
score_list = []
NIMA_scores = []
for img_path in tqdm(imgs):
img = load_img(img_path, target_size=target_size)
x = img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
scores = model.predict(x, batch_size=1, verbose=0)[0]
mean = mean_score(scores)
std = std_score(scores)
save_best_only=True,
verbose=1)
model.compile(loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(learning_rate=0.0005),
metrics=['acc'])
learning_history = model.fit_generator(train_generator,
epochs=300,
steps_per_epoch=len(x_train) / 32,
validation_data=valid_generator,
callbacks=[early_stopping, lr, mc])
model.save('../data/lotte/h5/b4_weight_model.h5')
# predict
model.load_weights('../data/lotte/mc/lotte_b4.h5')
result = model.predict(x_pred, verbose=True)
# tta_steps = 30
# # tta : 증강된 이미지를 여러번 보여준 다음 각각의 단계에 대해서 prediction을 평균하고 이 결과를 최종값으로 사용하는 것
# predictions = []
# for i in tqdm(range(tta_steps)):
# # generator 초기화
# test_generator.reset()
# preds = model.predict_generator(generator = test_generator, verbose = 1)
# predictions.append(preds)
# 평균을 통한 final prediction
# pred = np.mean(predictions, axis=0)
class AEOshea1hot(object):
def __init__(self, in_dim=None, latent_dim=None, h_dim=None, train_snr_dB=10.0):
self.in_dim = in_dim
self.latent_dim = latent_dim
self.train_snr_dB = train_snr_dB
self.h_dim = h_dim
self.train_noisepow = 10.0**(-self.train_snr_dB/10.0)
if self.in_dim and self.latent_dim:
self.make_model()
def make_model(self):
# Input layer
self.inputs = Input(shape=(self.in_dim,), name="enc_in")
# Hidden Layers
x = self.inputs
if self.h_dim is not None:
for (i,d) in enumerate(self.h_dim):
x = Dense( d, activation='relu', name="enc_l{}".format(i))(x)
# Mean and Variance
x = Dense(self.latent_dim)(x)
self.z_mean = BatchNormalization(center=False, scale=False)(x)
# Channel
self.z = Lambda(self.channel, output_shape=(self.latent_dim,), name="z")(self.z_mean)
# Encoder model
self.encoder = Model(self.inputs, [self.z_mean,self.z], name="encoder")
# Decoder
self.latent_inputs = Input(shape=(self.latent_dim,), name="z_sample")
# Hidden layers
x = self.latent_inputs
if self.h_dim is not None:
for (i,d) in enumerate(self.h_dim[::-1]):
x = Dense( d, activation='relu', name="dec_l{}".format(i))(x)
self.dec_outputs = Dense( self.in_dim, activation='softmax', name="decoder_out")(x)
# Decoder model
self.decoder = Model(self.latent_inputs, self.dec_outputs, name="decoder")
# VAE
self.outputs = self.decoder(self.encoder(self.inputs)[1])
self.model = Model(self.inputs, self.outputs, name="VAE")
# Losses
self.recon_loss = categorical_crossentropy(self.inputs, self.outputs)
self.model.add_loss(self.recon_loss)
self.model.compile(optimizer='adam')
# self.model.compile( optimizer=tf.train.AdamOptimizer(learning_rate=0.01))
def channel( self, zMean ):
batch = K.shape( zMean )[0]
epsilon = K.random_normal( shape = (batch,self.latent_dim) )
# Because BatchNormalization produces z vector with signal power 'latentDim',
# we should not scale the noise power here.
return zMean + np.sqrt(self.train_noisepow)*epsilon
def fit(self, x_train, epochs=10, batch_size=128, validation_data=None,
verbose=0, callbacks=None):
train_log = self.model.fit(x_train, epochs=epochs, batch_size=batch_size,
validation_data=validation_data, verbose=verbose,
callbacks=callbacks)
return train_log.history
def encode( self, data ):
return self.encoder.predict(data)
def decode( self, data ):
return self.decoder.predict(data)
def analysis( self ):
xTest = np.eye(self.in_dim)
enc_mu, enc_z = self.encode(xTest)
dec_mu = self.decode(enc_mu)
dec_z = self.decode(enc_z)
chDim = self.latent_dim//2
f = plt.figure(figsize=(5*chDim,9))
for i in range(chDim):
ax1 = plt.subplot(2,chDim,i+1)
ax1.scatter(enc_mu[:,i],enc_mu[:,i+chDim],c=np.arange(self.in_dim))
for j in range(self.in_dim):
ax1.annotate( j, (enc_mu[j,i],enc_mu[j,i+chDim]) )
ax1.set_title( "TX Symbols n = {}".format(i+1) )
def save_model(self, fileprefix):
with open(fileprefix+".dil","wb") as obj:
dill.dump( {'in_dim': self.in_dim,
'latent_dim': self.latent_dim,
'train_noisepow': self.train_noisepow,
'train_snr_dB': self.train_snr_dB,
'h_dim': self.h_dim}, obj )
self.model.save_weights(fileprefix+".h5")
def load_model(self, fileprefix):
with open(fileprefix+".dil","rb") as obj:
config = dill.load(obj)
self.in_dim = config['in_dim']
self.latent_dim = config['latent_dim']
self.train_snr_dB = config['train_snr_dB']
self.train_noisepow = config['train_noisepow']
self.h_dim = config['h_dim']
self.make_model()
self.model.load_weights(fileprefix+".h5")
x = Conv2D(512, (3, 3), activation='relu', name='conv5_2', padding='same')(x)
x = Conv2D(512, (3, 3), activation='relu', name='conv5_3', padding='same')(x)
x = pooling_func(x)
model = Model(ip, x)
if model == "vgg19":
weights = get_file('vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_19_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
else:
weights = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights)
if K.backend() == 'tensorflow' and K.image_data_format() == "channels_first":
convert_all_kernels_in_model(model)
print('Model loaded.')
outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
shape_dict = dict([(layer.name, layer.output_shape) for layer in model.layers])
def gram_matrix(input_tensor):
assert K.ndim(input_tensor) == 3
features = K.batch_flatten(K.permute_dimensions(input_tensor, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
gan = Model(inputs=[random_vec_input, labels_input], outputs=decision)
gan.compile(loss="binary_crossentropy", optimizer=tf.keras.optimizers.Adam(lr=1e-4))
if verbose:
print("Model is built.")
print(gan.summary())
# Training
# -------------------------
if verbose:
print("Training GAN model...")
if path_weights is not None and os.path.exists(path_weights):
gan.load_weights(path_weights)
# Adversarial ground truths
real = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
iterations = len(x_train) // batch_size
for e in range(epochs):
if verbose:
print("Epoch {}/{} :".format(e, epochs), end="")
for i in range(iterations):
# Training discriminator
# -------------------------
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath="/content/drive/My Drive/weights{epoch:02d}.h5",
save_weights_only=True,
save_freq="epoch")
N = X_train.shape[0]
X_L, Y_L = X_train[:N // 2], Y_train[:N // 2]
U_hat = X_train[N // 2:]
history = model.fit(generate_data(X_L, Y_L, U_hat, X_val, Y_val, batch_size),
epochs=30,
steps_per_epoch=X_L.shape[0] // batch_size,
callbacks=model_checkpoint_callback)
data = np.load('/content/drive/My Drive/Dataset/splitted_mfccs_order0_new.npz')
X_test = data['X_test']
Y_test = data['Y_test']
model_U.load_weights("/content/drive/My Drive/weights11.h5")
model_U.evaluate(X_test, Y_test)
inp = Input(shape=(129, 20))
x = LSTM(128, return_sequences=True)(inp)
x = LSTM(32)(x)
y = Dense(10,
activation='softmax',
kernel_regularizer=tf.keras.regularizers.l2(1e-4))(x)
model_base = Model(inputs=inp, outputs=y)
print(model_base.summary())
model_base.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer=tf.keras.optimizers.Adam(),
metrics=['accuracy'])
model_base.fit(X_train, Y_train, epochs=5, validation_data=(X_val, Y_val))
metrics=['accuracy'])
with tf.Session() as session:
K.set_session(session)
session.run(tf.global_variables_initializer())
session.run(tf.tables_initializer())
history = model.fit(x_train, y_train, epochs=10, batch_size=32)
model.save_weights('./elmo-model-9.h5')
model.summary()
with tf.Session() as session:
K.set_session(session)
session.run(tf.global_variables_initializer())
session.run(tf.tables_initializer())
model.load_weights('./elmo-model-12.h5')
predicts = model.predict(x_test, batch_size=10)
y_test = decode(le, y_test)
y_preds = decode(le, predicts)
#for item in y_test:
# print(item.strip())
#print(y_test)
print('-----')
for item in y_preds:
print(item.strip())
from sklearn import metrics
print(metrics.confusion_matrix(y_test, y_preds))
classifier = nn.classifier(shared_layers,
roi_input,
C.num_rois,
nb_classes=len(classes_count),
trainable=True)
model_rpn = Model(img_input, rpn[:2])
model_classifier = Model([img_input, roi_input], classifier)
# this is a model that holds both the RPN and the classifier, used to load/save weights for the models
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
try:
print('loading weights from {}'.format(C.base_net_weights))
model_rpn.load_weights(C.base_net_weights, by_name=True)
model_classifier.load_weights(C.base_net_weights, by_name=True)
except:
print(
'Could not load pretrained model weights. Weights can be found in the keras application folder \
https://github.com/fchollet/keras/tree/master/keras/applications')
optimizer = Adam(lr=1e-5)
optimizer_classifier = Adam(lr=1e-5)
model_rpn.compile(
optimizer=optimizer,
loss=[losses.rpn_loss_cls(num_anchors),
losses.rpn_loss_regr(num_anchors)])
model_classifier.compile(
optimizer=optimizer_classifier,
loss=[
class HLRGDVGG(object):
def __init__(self, input_dimension, output_dimension, vggModelLocation, number_of_classes=2, optimizer=None,
loss_threshold=defaultLossThreshold, patience=defaultPatience, dropout_rate=defaultDropoutRate,
max_relu_bound=None, adv_penalty=0.01, unprotected=False, verbose=False):
self.buildModel(input_dimension=input_dimension, output_dimension=output_dimension , vggModelLocation=vggModelLocation, optimizer=optimizer,
number_of_classes=number_of_classes, loss_threshold=loss_threshold, patience=patience, dropout_rate=dropout_rate,
max_relu_bound=max_relu_bound, adv_penalty=adv_penalty, unprotected=unprotected, verbose=verbose)
def buildModel(self, input_dimension, output_dimension, vggModelLocation, number_of_classes=2, optimizer=None,
loss_threshold=defaultLossThreshold, patience=defaultPatience, dropout_rate=defaultDropoutRate,
max_relu_bound=None, adv_penalty=0.01, unprotected=False, verbose=False):
#instantiate the vgg instance
self.ourVGG = HLVGG(input_dimension=input_dimension, output_dimension=output_dimension, dual_outputs=True, number_of_classes=number_of_classes,
loss_threshold=loss_threshold, patience=patience, dropout_rate=0, freezeWeights=True, optimizer=optimizer,
max_relu_bound=max_relu_bound, adv_penalty=adv_penalty, unprotected=True, verbose=verbose)
#set the vgg weights
self.ourVGG.model.load_weights(vggModelLocation)
#set protected flag
self.unprotected = unprotected
#instantiate our CAE with residual connections
self.input_dimension, self.output_dimension = input_dimension, np.copy(output_dimension)
self.loss_threshold, self.number_of_classes = np.copy(loss_threshold), np.copy(number_of_classes)
self.dropoutRate, self.max_relu_bound = dropout_rate, np.copy(max_relu_bound)
self.patience = np.copy(patience)
# self.learning_rate, self.learning_rate_drop = np.copy(learning_rate), np.copy(learning_rate_drop)
self.image_size = 32
self.num_channels = 3
self.num_labels = number_of_classes
self.penaltyCoeff = adv_penalty
if (verbose):
print("input dimension: %s"%str(self.input_dimension))
# define input layer
self.inputLayer = layers.Input(shape=self.input_dimension)
self.advInputLayer = layers.Input(shape=self.input_dimension)
previousLayer = self.advInputLayer
#following the architecture shown in figure 3 of the hlrgd paper, [ https://openaccess.thecvf.com/content_cvpr_2018/papers/Liao_Defense_Against_Adversarial_CVPR_2018_paper.pdf ]
#
self.hiddenEncoderLayers = dict()
#first encoder block
previousLayer = layers.Conv2D(numberOfFilters[0], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[0], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
self.hiddenEncoderLayers[0] = previousLayer
#second encoder block
previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[1], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
self.hiddenEncoderLayers[1] = previousLayer
#third encoder block
previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[2], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
self.hiddenEncoderLayers[2] = previousLayer
#fourth encoder block
previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[3], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
self.hiddenEncoderLayers[3] = previousLayer
#fifth encoder block
previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2D(numberOfFilters[4], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
self.hiddenEncoderLayers[4] = previousLayer
#first decoding block
#fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer
previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer)
previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[3]])
#perform convolutions
previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[5], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
#second decoding block
#fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer
# previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer)
previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[2]])
#perform convolutions
previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[6], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
#third decoding block
#fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer
# previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer)
previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[1]])
#perform convolutions
previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(2, 2), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[7], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
#fourth decoding block
#fuse inputs (i.e. concatenate the residual connection with an upscaled version of the previousLayer
# previousLayer = layers.UpSampling2D((2, 2), interpolation='nearest')(previousLayer)
previousLayer = layers.concatenate([previousLayer, self.hiddenEncoderLayers[0]])
#perform convolutions
previousLayer = layers.Conv2DTranspose(numberOfFilters[8], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
previousLayer = layers.Conv2DTranspose(numberOfFilters[8], kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')(previousLayer)
previousLayer = layers.BatchNormalization()(previousLayer)
if (self.dropoutRate > 0):
previousLayer = Dropout(self.dropoutRate)(previousLayer)
#1x1 convolutional layer
previousLayer = layers.Conv2DTranspose(1, kernel_size=(1,1), strides=(1, 1), padding='same', activation='relu')(previousLayer)
#compute the output
#\hat{x} = x^{\asterisk} - d\hat{x}
duOutput = self.inputLayer - previousLayer
#compute vgg penultimate layer post-activation outputs when the vgg subnet is stimulated with duOutput and the benign sample
# inputList = [self.inputLayer, self.inputLayer]
# duList = [duOutput, duOutput]
vggOutputBenign, vggPenultimate = self.ourVGG.model([self.inputLayer, self.inputLayer])
vggOutputDU, vggPenultimateDU = self.ourVGG.model([duOutput, duOutput])
#assign output layer as the evaluation of vgg that depends on the (potentially) adv. input
self.outputLayer = vggOutputDU
#formulate our loss function
#define our custom loss function
#this is L1 norm of the difference between the vggOutput
def customLossWrapper(vggPenultimateDU, vggPenultimate, penaltyCoeff = self.penaltyCoeff):
def customLoss(y_true, y_pred):
return K.sum(K.abs(vggPenultimateDU - vggPenultimate))
return customLoss
# optimization details
# def lr_scheduler(epoch):
# return self.learning_rate * (0.5 ** (epoch // self.learning_rate_drop))
# self.lr_scheduler = lr_scheduler
# self.reduce_lr = keras.callbacks.LearningRateScheduler(lr_scheduler)
# self.sgd = tf.keras.optimizers.Nadam()#Adadelta(learning_rate=self.learning_rate)
# self.sgd = tf.keras.optimizers.SGD(lr=self.learning_rate, decay=0.000001, momentum=0.9, nesterov=True)
if (optimizer==None):
self.sgd = tf.keras.optimizers.Nadam()#Adadelta(learning_rate=self.learning_rate)
self.reduceLR = None
elif(optimizer=='SGD'):
def lr_scheduler(epoch):
return 0.1 * (0.5 ** (epoch // 20))#learning_rate * (0.5 ** (epoch // 20))
self.reduceLR = keras.callbacks.LearningRateScheduler(lr_scheduler)
self.sgd = tf.keras.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
#set up data augmentation
self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False)
# convert self.hiddenAdvLayers to a list for the model compilation, ascending order of keys is order of layers
# outputsList is a list of outputs of the model constructed so that the first entry is the true output (ie prediction) layer
# and each subsequent (i, i+1)th entries are the pair of hiddenAdvLayer, hiddenBenignLayer activations
# this is going to be useful for calculating the MAE between benignly and adversarially induced hidden states
# outputsList = [self.outputLayer]
# for curHiddenLayer in range(len(self.ourVGG.hiddenAdvModelOutputs)):
# outputsList.append(self.ourVGG.hiddenAdvModelOutputs[curHiddenLayer])
# outputsList.append(self.ourVGG.hiddenModelOutputs[curHiddenLayer])
# instantiate and compile the model
self.customLossWrapper = customLossWrapper
self.model = Model(inputs=[self.inputLayer, self.advInputLayer], outputs=[self.outputLayer], name='hlrgd_vgg16')
self.model.compile(loss=customLossWrapper(vggPenultimateDU, vggPenultimate, self.penaltyCoeff), metrics=['acc'], optimizer=self.sgd)
# double check weight trainability bug
allVars = self.model.variables
trainableVars = self.model.trainable_variables
allVarNames = [self.model.variables[i].name for i in range(len(self.model.variables))]
trainableVarNames = [self.model.trainable_variables[i].name for i in range(len(self.model.trainable_variables))]
nonTrainableVars = np.setdiff1d(allVarNames, trainableVarNames)
if (verbose):
if (len(nonTrainableVars) > 0):
print('the following variables are set to non-trainable; ensure that this is correct before publishing!!!!')
print(nonTrainableVars)
self.model.summary()
#set data statistics to default values
self.mean = 0
self.stddev = 1
# handle data augmentation with multiple inputs (example found on https://stackoverflow.com/questions/49404993/keras-how-to-use-fit-generator-with-multiple-inputs
#so thanks to loannis and Julian
def multiInputDataGenerator(self, X1, X2, Y, batch_size):
genX1 = self.generator.flow(X1, Y, batch_size=batch_size)
genX2 = self.generator.flow(X2, Y, batch_size=batch_size)
while True:
X1g = genX1.next()
X2g = genX2.next()
yield [X1g[0], X2g[0]], X1g[1]
#this method trains the HLRGD
#inputData is a list of training matrices, the 0th is adversarial (i.e., to be denoised), the 2nd is the benign input
def train(self, inputTrainingData, trainingTargets, inputValidationData, validationTargets, training_epochs=1, normed=False, monitor='val_loss',
patience=defaultPatience, model_path=None, keras_batch_size=None, validation_split=0.1, adversarialOrder=0,
dataAugmentation=False):
#if a path isn't provided by caller, just use the current time for restoring best weights from fit
if (model_path is None):
model_path = os.path.join('/tmp/models/', 'hlrgd_vgg16_'+str(int(round(time.time()*1000))))
#if the data are not normalized, normalize them
trainingData, validationData = [[],[]], [[],[]]
if (not normed):
#don't store stats from the adversarially attacked data
if (adversarialOrder == 0):
trainingData[0] = self.normalize(inputTrainingData[0], storeStats=True)
trainingData[1] = self.normalize(inputTrainingData[1], storeStats=False)
else:
trainingData[1] = self.normalize(inputTrainingData[1], storeStats=True)
trainingData[0] = self.normalize(inputTrainingData[0], storeStats=False)
#also don't store stats from validation data
validationData[0] = self.normalize(inputValidationData[0], storeStats=False)
validationData[1] = self.normalize(inputValidationData[1], storeStats=False)
else:
trainingData[0] = inputTrainingData[0]
trainingData[1] = inputTrainingData[1]
validationData[0] = inputValidationData[0]
validationData[1] = inputValidationData[1]
#collect our callbacks
earlyStopper = EarlyStopping(monitor=monitor, mode='min', patience=patience,
verbose=1, min_delta=defaultLossThreshold)
checkpoint = ModelCheckpoint(model_path, verbose=1, monitor=monitor, save_weights_only=True,
save_best_only=True, mode='auto')
# history = self.model.fit(trainingData, trainingTargets, epochs=training_epochs, batch_size=keras_batch_size,
# validation_split=validation_split, callbacks=[earlyStopper, self.reduce_lr])
#handle data augmentation
if (not dataAugmentation):
# set up data augmentation
self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False,
# divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
# randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False)
self.generator.fit(trainingData[0])
history = self.model.fit(self.multiInputDataGenerator(trainingData[0], trainingData[1], trainingTargets, keras_batch_size),
steps_per_epoch=trainingData[0].shape[0] // keras_batch_size,
epochs=training_epochs, validation_data=(validationData, validationTargets),
callbacks=[earlyStopper, checkpoint], verbose=1) #self.reduce_lr
else:
# set up data augmentation
self.generator = ImageDataGenerator(featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False,
# divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=15,
# randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1,
# randomly shift images horizontally (fraction of total width)
height_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False)
self.generator.fit(trainingData[0])
history = self.model.fit(self.multiInputDataGenerator(trainingData[0], trainingData[1], trainingTargets, keras_batch_size),
steps_per_epoch=trainingData[0].shape[0] // keras_batch_size,
epochs=training_epochs, validation_data=(validationData, validationTargets),
callbacks=[earlyStopper, checkpoint], verbose=1) #self.reduce_lr
self.model.load_weights(model_path)
loss, acc = history.history['loss'], history.history['val_acc']
return loss, acc, model_path
def evaluate(self, inputData, targets, batchSize=None):
evalData = [self.normalize(inputData[0], storeStats=False), self.normalize(inputData[1], storeStats=False)]
fullEval = self.model.evaluate(evalData, targets, batch_size=batchSize)
return fullEval
#method to read model from the disk
def readModelFromDisk(self, pathToFile, vggModelLocation):
#rebuild the model
self.buildModel(self.input_dimension, self.output_dimension, vggModelLocation, self.number_of_classes,
loss_threshold=self.loss_threshold, patience=self.patience, dropout_rate=self.dropoutRate,
max_relu_bound=self.max_relu_bound, adv_penalty=self.penaltyCoeff, unprotected=self.unprotected,
verbose=False)
# set the vgg weights
self.model.load_weights(pathToFile)
# #read in the picklebox
pickleBox = pickle.load(open(pathToFile+'_pickle', 'rb'))
# # self.bottleneckLayer = pickleBox['bottleneckLayer']
# # self.hiddenEncodingLayer = pickleBox['hiddenEncodingLayer']
# # self.inputLayer = pickleBox['inputLayer']
self.mean, self.std = pickleBox['scaleMean'], pickleBox['scaleSTD']
#first read in the inner model
# self.ourVGG.readModelFromDisk('inner_'+pathToFile)
# #read in the picklebox
# pickleBox = pickle.load(open(pathToFile+'_pickle', 'rb'))
# # self.bottleneckLayer = pickleBox['bottleneckLayer']
# # self.hiddenEncodingLayer = pickleBox['hiddenEncodingLayer']
# # self.inputLayer = pickleBox['inputLayer']
# self.chosenActivation = pickleBox['chosenActivation']
# formulate our loss function
# define our custom loss function
# this is L1 norm of the difference between the vggOutput
# if (not self.unprotected):
# def customLossWrapper(benProjs, advProjs, penaltyCoeff = self.penaltyCoeff):
# def customLoss(y_true, y_pred):
# return K.categorical_crossentropy(y_true, y_pred) + penaltyCoeff*K.sum(K.abs(benProjs - advProjs))/(0.00000001+tf.cast(tf.shape(benProjs)[0], tf.float32))
# return customLoss
# else:#if we are using an unprotected model, don't force the machine to calculate this too
# def customLossWrapper(benProjs, advProjs, penaltyCoeff = self.penaltyCoeff):
# def customLoss(y_true, y_pred):
# return K.categorical_crossentropy(y_true, y_pred)
# return customLoss
# self.customLossWrapper = customLossWrapper(self.benProjs, self.advProjs)
# #load the model
# self.model = load_model(pathToFile, custom_objects={'customLossWrapper': self.customLossWrapper})
# this routine is used to collect statistics on training data, as well as to preprocess the training data by normalizing
# i.e. centering and dividing by standard deviation
def normalize(self, inputData, storeStats=False):
if (storeStats):
self.mean = np.mean(inputData)
self.stddev = np.std(inputData)
outputData = (inputData - self.mean) / (self.stddev + 0.0000001)
return outputData
# routine to get a pointer to the optimizer of this model
def getOptimizer(self):
if (self.ourVGG is not None):
return self.sgd, self.ourVGG.getOptimizer()
else:
return self.sgd
class AGZeroModel:
def __init__(self,
N,
batch_size=32,
archive_fit_samples=64,
use_tpu=None,
log_path='logs/tensorboard'):
self.N = N
self.batch_size = batch_size
self.model = None
self.archive_fit_samples = archive_fit_samples
self.position_archive = []
self.tpu_grpc_url = use_tpu
tpu_name_environ_key = 'TPU_NAME'
# Check has server got TPU
if use_tpu is not False and tpu_name_environ_key in os.environ:
tpu_name = os.environ[tpu_name_environ_key].strip()
if tpu_name != "":
self.is_tpu = True
self.tpu_grpc_url = TPUClusterResolver(
tpu=[os.environ[tpu_name_environ_key]]).get_master()
# TODO write an if condition to validate and resolve the TPU url provided
self.__loss_functions = [
'categorical_crossentropy', 'binary_crossentropy'
]
self.model_name = time.strftime('GM{0}-%y%m%dT%H%M%S').format('%02d' %
N)
# print(self.model_name)
log_path = os.path.join(log_path, self.model_name)
if not os.path.exists(log_path):
os.makedirs(log_path)
self.callback = TensorBoard(log_path)
def create(self):
bn_axis = 3
N = self.N
position = Input((N, N, 6))
resnet = ResNet(n_stages=N)
resnet.create(N, N, 6)
x = resnet.model(position)
dist = Conv2D(2, (1, 1))(x)
dist = BatchNormalization(axis=bn_axis)(dist)
dist = Activation('relu')(dist)
dist = Flatten()(dist)
dist = Dense(N * N + 1, activation='softmax',
name='distribution')(dist)
res = Conv2D(1, (1, 1))(x)
res = BatchNormalization(axis=bn_axis)(res)
res = Activation('relu')(res)
res = Flatten()(res)
res = Dense(256, activation='relu')(res)
res = Dense(1, activation='sigmoid', name='result')(res)
self.model = Model(position, [dist, res])
self.model.compile(Adam(lr=2e-2), self.__loss_functions)
self.callback.set_model(self.model)
# check if TPU available
if self.tpu_grpc_url is not None:
self.model = tf.contrib.tpu.keras_to_tpu_model(
self.model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(
self.tpu_grpc_url)))
self.model.summary()
def fit_game(self, X_positions, result):
X_posres = []
for pos, dist in X_positions:
X_posres.append((pos, dist, result))
result = -result
if len(self.position_archive) >= self.archive_fit_samples:
archive_samples = random.sample(self.position_archive,
self.archive_fit_samples)
else:
# initial case
archive_samples = self.position_archive
self.position_archive.extend(X_posres)
# I'm going to some lengths to avoid the potentially overloaded + operator
X_fit_samples = list(itertools.chain(X_posres, archive_samples))
self.__fit_model(X_fit_samples, self.batch_size)
def retrain_position_archive(self, batch_size=None):
self.__fit_model(self.position_archive,
batch_size if batch_size else self.batch_size * 8)
def reduce_position_archive(self, ratio=0.5):
try:
self.position_archive = random.sample(
self.position_archive, int(len(self.position_archive) * ratio))
except:
pass
def __fit_model(self, X_fit_samples, batch_size):
batch_no = 1
X, y_dist, y_res = [], [], []
X_shuffled = random.sample(X_fit_samples, len(X_fit_samples))
X_shuffled = create_batches(X_shuffled, batch_size)
for batch in X_shuffled:
for pos, dist, res in batch:
X.append(pos)
y_dist.append(dist)
y_res.append(float(res) / 2 + 0.5)
logs = self.model.train_on_batch(
np.array(X),
[np.array(y_dist), np.array(y_res)])
self.write_log(self.__loss_functions, logs, batch_no)
batch_no += 1
X, y_dist, y_res = [], [], []
def write_log(self, names, logs, batch_no):
for name, value in zip(names, logs):
summary = tf.Summary()
summary_value = summary.value.add()
summary_value.simple_value = value
summary_value.tag = name
self.callback.writer.add_summary(summary, batch_no)
self.callback.writer.flush()
def predict(self, X_positions):
dist, res = self.model.predict(X_positions)
res = np.array([r[0] * 2 - 1 for r in res])
return [dist, res]
def save(self, snapshot_id, save_archive=False):
self.model.save_weights('%s.weights.h5' % (snapshot_id, ))
if save_archive:
joblib.dump(self.position_archive,
'%s.archive.joblib' % (snapshot_id, ),
compress=5)
def load(self, snapshot_id):
self.model.load_weights('%s.weights.h5' % (snapshot_id, ))
pos_fname = '%s.archive.joblib' % (snapshot_id, )
try:
self.position_archive = joblib.load(pos_fname)
except:
print('Warning: Could not load position archive %s' %
(pos_fname, ))
def unload_pos_archive(self):
self.position_archive = []
def load_pos_archive(self, archive_file):
try:
print('Attempting to load position archive %s' % (archive_file, ))
self.position_archive = joblib.load(archive_file)
print('Successfully loaded position archive %s' % (archive_file, ))
return True
except:
import traceback
traceback.print_exc()
print('Warning: Could not load position archive %s' %
(archive_file, ))
return False
def load_averaged(self, weights, log=None):
new_weights = []
loaded_weights = []
for weight in weights:
self.model.load_weights(weight)
loaded_weights.append(self.model.get_weights())
print("Read weight: {0}".format(weight))
if log is not None:
log("Read weight: {0}".format(weight), self.model_name)
if len(loaded_weights) > 0:
for weights_list_tuple in zip(*loaded_weights):
new_weights.append([
np.array(weights_).mean(axis=0)
for weights_ in zip(*weights_list_tuple)
])
self.model.set_weights(new_weights)
else:
print(
"No weights to load. Initializing the model with random weights!"
)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
# Third callback; save the best model-weights (best validation_accuracy)
weights_dir = "temp/"
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=weights_dir,
save_weights_only=True,
monitor='val_accuracy',
mode='max',
save_best_only=True)
model.fit(features_train,
ltr,
epochs=50,
batch_size=128,
validation_data=(features_test, ltt),
callbacks=[
tensorboard_callback, model_checkpoint_callback, cm_callback
])
# Load the temporarly saved best model weights and save the entire model with this weights
model.load_weights(weights_dir)
model_dir = 'models/' + MODELNAME
model.save(model_dir)
print("Model trained and saved!")
cc = cc + 1
def Xception(include_top=True, weights='hasc', input_shape=None, pooling=None, classes=6, classifier_activation='softmax'):
if input_shape is None:
input_shape = (256*3, 1)
if weights in ['hasc', 'HASC'] and include_top and classes != 6:
raise ValueError('If using `weights` as `"hasc"` with `include_top`'
' as true, `classes` should be 6')
inputs = layers.Input(shape=input_shape)
x = layers.Conv1D(32, 3, strides=2, use_bias=False, name='block1_conv1')(inputs)
x = layers.BatchNormalization(name='block1_conv1_bn')(x)
x = layers.Activation('relu', name='block1_conv1_act')(x)
x = layers.Conv1D(64, 3, use_bias=False, name='block1_conv2')(x)
x = layers.BatchNormalization(name='block1_conv2_bn')(x)
x = layers.Activation('relu', name='block1_conv2_act')(x)
residual = layers.Conv1D(
128, 1, strides=2, padding='same', use_bias=False
)(x)
residual = layers.BatchNormalization()(residual)
x = layers.SeparableConv1D(128, 3, padding='same', use_bias=False, name='block2_sepconv1')(x)
x = layers.BatchNormalization(name='block2_sepconv1_bn')(x)
x = layers.Activation('relu', name='block2_sepconv2_act')(x)
x = layers.SeparableConv1D(128, 3, padding='same', use_bias=False, name='block2_sepconv2')(x)
x = layers.BatchNormalization(name='block2_sepconv2_bn')(x)
x = layers.MaxPooling1D(3, strides=2, padding='same', name='block2_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv1D(
256, 1, strides=2, padding='same', use_bias=False
)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block3_sepconv1_act')(x)
x = layers.SeparableConv1D(256, 3, padding='same', use_bias=False, name='block3_sepconv1')(x)
x = layers.BatchNormalization(name='block3_sepconv1_bn')(x)
x = layers.Activation('relu', name='block3_sepconv2_act')(x)
x = layers.SeparableConv1D(256, 3, padding='same', use_bias=False, name='block3_sepconv2')(x)
x = layers.BatchNormalization(name='block3_sepconv2_bn')(x)
x = layers.MaxPooling1D(3, strides=2, padding='same', name='block3_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv1D(728, 1, strides=2, padding='same', use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block4_sepconv1_act')(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block4_sepconv1')(x)
x = layers.BatchNormalization(name='block4_sepconv1_bn')(x)
x = layers.Activation('relu', name='block4_sepconv2_act')(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block4_sepconv2')(x)
x = layers.BatchNormalization(name='block4_sepconv2_bn')(x)
x = layers.MaxPooling1D(3, strides=2, padding='same', name='block4_pool')(x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = layers.Activation('relu', name=prefix + "_sepconv1_act")(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv1")(x)
x = layers.BatchNormalization(name=prefix + "_sepconv1_bn")(x)
x = layers.Activation('relu', name=prefix + "_sepconv2_act")(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv2")(x)
x = layers.BatchNormalization(name=prefix + "_sepconv2_bn")(x)
x = layers.Activation('relu', name=prefix + "_sepconv3_act")(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name=prefix + "_sepconv3")(x)
x = layers.add([x, residual])
residual = layers.Conv1D(1024, 1, strides=2, padding='same', use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block13_sepconv1_act')(x)
x = layers.SeparableConv1D(728, 3, padding='same', use_bias=False, name='block13_sepconv1')(x)
x = layers.BatchNormalization(name='block13_sepconv1_bn')(x)
x = layers.Activation('relu', name='block13_speconv2_act')(x)
x = layers.SeparableConv1D(1024, 3, padding='same', use_bias=False, name='block13_sepconv2')(x)
x = layers.BatchNormalization(name='block13_sepconv2_bn')(x)
x = layers.MaxPooling1D(3, strides=2, padding='same')(x)
x = layers.add([x, residual])
x = layers.SeparableConv1D(1536, 3, padding='same', use_bias=False, name='block14_sepconv1')(x)
x = layers.BatchNormalization(name='block14_sepconv1_bn')(x)
x = layers.Activation('relu', name='block14_sepconv1_act')(x)
x = layers.SeparableConv1D(2048, 3, padding='same', use_bias=False, name='block14_sepconv2')(x)
x = layers.BatchNormalization(name='block14_sepconv2_bn')(x)
x = layers.Activation('relu', name='block14_sepconv2_act')(x)
x = layers.GlobalAveragePooling1D(name='avg_pool')(x)
y = layers.Dense(classes, activation=classifier_activation,
name='predictions')(x)
model = Model(inputs, y)
if weights is not None:
if weights in ['hasc', "HASC"]:
weights = 'weights/xception/xception_hasc_weights_{}_{}.hdf5'.format(int(input_shape[0]),
int(input_shape[1]))
# hasc or weights fileで初期化
if os.path.exists(weights):
print("Load weights from {}".format(weights))
model.load_weights(weights)
else:
print("Not exist weights: {}".format(weights))
# topを含まないとき
if not include_top:
if pooling is None:
# topを削除する
model = Model(inputs=model.input, outputs=model.layers[-3].output)
elif pooling == 'avg':
y = layers.GlobalAveragePooling1D()(model.layers[-3].output)
model = Model(inputs=model.input, outputs=y)
elif pooling == 'max':
y = layers.GlobalMaxPooling1D()(model.layers[-3].output)
model = Model(inputs=model.input, outputs=y)
else:
print("Not exist pooling option: {}".format(pooling))
model = Model(inputs=model.input, outputs=model.layers[-3].output)
return model
class SRGAN():
def __init__(self, quantize_flag):
self.quantize_flag = quantize_flag
# Input shape
self.channels = 3
self.lr_height = 56 # Low resolution height
self.lr_width = 56 # Low resolution width
self.lr_shape = (self.lr_height, self.lr_width, self.channels)
self.hr_height = self.lr_height * 4 # High resolution height
self.hr_width = self.lr_width * 4 # High resolution width
self.hr_shape = (self.hr_height, self.hr_width, self.channels)
# Number of residual blocks in the generator
self.n_residual_blocks = 16
optimizer = Adam(0.000002, 0.5)
# We use a pre-trained VGG19 model to extract image features from the high resolution
# and the generated high resolution images and minimize the mse between them
self.vgg = self.build_vgg()
self.vgg.trainable = False
self.vgg.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
# Configure data loader
self.dataset_name = 'img_align_celeba'
self.test_dataset_name = 'test_images'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.hr_height, self.hr_width))
self.test_data_loader = DataLoader(dataset_name=self.test_dataset_name,
img_res=(self.hr_height,
self.hr_width))
# Calculate output shape of D (PatchGAN)
patch = int(self.hr_height / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# High res. and low res. images
img_hr = Input(shape=self.hr_shape)
img_lr = Input(shape=self.lr_shape)
# Generate high res. version from low res.
fake_hr = self.generator(img_lr)
# Extract image features of the generated img
fake_features = self.vgg(fake_hr)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminator determines validity of generated high res. images
validity = self.discriminator(fake_hr)
self.combined = Model([img_lr, img_hr], [validity, fake_features])
self.combined.compile(loss=['binary_crossentropy', 'mse'],
loss_weights=[1e-3, 1],
optimizer=optimizer)
def build_vgg(self):
"""
Builds a pre-trained VGG19 model that outputs image features extracted at the
third block of the model
"""
vgg = VGG19(weights="imagenet")
# Set outputs to outputs of last conv. layer in block 3
# See architecture at: https://github.com/tensorflow.keras-team/tensorflow.keras/blob/master/tensorflow.keras/applications/vgg19.py
vgg.outputs = [vgg.layers[9].output]
img = Input(shape=self.hr_shape)
# Extract image features
print(img.shape)
img_features = vgg(img)
return Model(img, img_features)
def build_generator(self):
def residual_block(layer_input, filters):
"""Residual block described in paper"""
d = Conv2D(filters,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(layer_input)
d = BatchNormalization(momentum=0.8)(d)
d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
def deconv2d(layer_input):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(256,
kernel_size=3,
strides=1,
padding='same',
activation="relu")(u)
return u
# Low resolution image input
img_lr = Input(shape=self.lr_shape)
# Pre-residual block
c1 = Conv2D(
64,
kernel_size=9,
strides=1,
padding='same',
activation='relu',
)(img_lr)
# Propogate through residual blocks
r = residual_block(c1, self.gf)
for _ in range(self.n_residual_blocks - 1):
r = residual_block(r, self.gf)
# Post-residual block
c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
c2 = BatchNormalization(momentum=0.8)(c2)
c2 = Add()([c2, c1])
# Upsampling
u1 = deconv2d(c2)
u2 = deconv2d(u1)
# Generate high resolution output
gen_hr = Conv2D(self.channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(u2)
if self.quantize_flag == "quant":
return Quantizer.apply_quantization(Model(img_lr, gen_hr),
weight_precision=16,
activation_precision=16)
elif self.quantize_flag == "sparsity":
return Sparsity.measure_sparsity(Model(img_lr, gen_hr))
else:
return Model(img_lr, gen_hr)
def build_discriminator(self):
def d_block(layer_input, filters, strides=1, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=3, strides=strides,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
# Input img
d0 = Input(shape=self.hr_shape)
d1 = d_block(d0, self.df, bn=False)
d2 = d_block(d1, self.df, strides=2)
d3 = d_block(d2, self.df * 2)
d4 = d_block(d3, self.df * 2, strides=2)
d5 = d_block(d4, self.df * 4)
d6 = d_block(d5, self.df * 4, strides=2)
d7 = d_block(d6, self.df * 8)
d8 = d_block(d7, self.df * 8, strides=2)
d9 = Dense(self.df * 16)(d8)
d10 = LeakyReLU(alpha=0.2)(d9)
validity = Dense(1, activation='sigmoid')(d10)
return Model(d0, validity)
def train(self,
epochs,
batch_size=1,
sample_interval=50,
start_from_ckp_number=None):
start_time = datetime.datetime.now()
# Set up ternsorboard
logdir = "logs/scalars/" + datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S")
file_writer = tf.summary.create_file_writer(logdir + "/metrics")
file_writer.set_as_default()
# Restore checkpoint if present
start_epoch = 0
if start_from_ckp_number is not None:
start_epoch = start_from_ckp_number
self.generator.load_weights(
'./saved_models/generator_ckp_epoch_{}'.format(
start_from_ckp_number))
self.discriminator.load_weights(
'./saved_models/discriminator_ckp_epoch_{}'.format(
start_from_ckp_number))
self.combined.load_weights(
'./saved_models/combined_ckp_epoch_{}'.format(
start_from_ckp_number))
callbacks = [
TensorBoard(log_dir=r"./logs/tensorboard",
histogram_freq=1,
write_graph=True,
write_images=False,
update_freq='epoch',
profile_batch=0,
embeddings_freq=0,
embeddings_metadata=None)
]
for epoch in range(start_epoch, epochs):
# ----------------------
# Train Discriminator
# ----------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# From low res. image generate high res. version
fake_hr = self.generator.predict(imgs_lr)
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid)
d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ------------------
# Train Generator
# ------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# The generators want the discriminators to label the generated images as real
valid = np.ones((batch_size, ) + self.disc_patch)
# Extract ground truth image features using pre-trained VGG19 model
image_features = self.vgg.predict(imgs_hr)
# Train the generators
try:
self.combined.fit([imgs_lr, imgs_hr], [valid, image_features],
epochs=epoch + 1,
batch_size=len(imgs_lr),
initial_epoch=epoch,
callbacks=callbacks)
#g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features])
except Exception as e:
print("Something went wrong with this batch: {}".format(e))
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
if epoch % 10 == 0:
print("%d time: %s" % (epoch, elapsed_time))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch, batch_size)
def sample_images(self, epoch, batch_size):
os.makedirs('images/%s' % self.test_dataset_name, exist_ok=True)
# Get test data
# imgs_hr, imgs_lr = self.data_loader.load_data(batch_size=2, is_testing=True)
imgs_hr_a, imgs_lr_a = self.test_data_loader.load_data(
batch_size=batch_size, is_testing=True)
# Make prediction
fake_hr_a = self.generator.predict(imgs_lr_a)
image_idx = 0
for imgs_lr, fake_hr, imgs_hr in zip(imgs_lr_a, fake_hr_a, imgs_hr_a):
# # Rescale images 0 - 1
imgs_lr = 0.5 * imgs_lr + 0.5
fake_hr = 0.5 * fake_hr + 0.5
imgs_hr = 0.5 * imgs_hr + 0.5
imgs_hr = imgs_hr.reshape(self.hr_shape)
scipy.misc.toimage(imgs_hr).save(
'images/{}/batch_{}_idx_{}_original_highres.png'.format(
self.test_dataset_name,
epoch,
image_idx,
))
# Save lr version (if not present yet)
imgs_lr = imgs_lr.reshape(self.lr_shape)
scipy.misc.toimage(imgs_lr).save(
'images/{}/batch_{}_idx_{}_original_lowres.png'.format(
self.test_dataset_name,
epoch,
image_idx,
))
# Save generaed image
fake_hr = fake_hr.reshape(self.hr_shape)
scipy.misc.toimage(fake_hr).save(
'images/{}/batch_{}_idx_{}_generated_highres.png'.format(
self.test_dataset_name,
epoch,
image_idx,
))
image_idx = image_idx + 1
# Compute mean squared error
mse = np.mean((np.array(imgs_hr_a, dtype=np.float32) -
np.array(fake_hr_a, dtype=np.float32))**2)
# Compute psnr
if mse == 0:
# avoid (improbable) division by 0
psnr = 1000000
else:
max_pixel = 1.0
psnr = 20 * np.log10(max_pixel / np.sqrt(mse))
# save MSE and PSNR to tensorboard
tf.summary.scalar('MSE', data=mse, step=epoch)
tf.summary.scalar('PSNR', data=psnr, step=epoch)
for index, layer in enumerate(self.generator.layers):
try:
sparsity = layer.get_weights()[4]
tf.summary.scalar('Sparsity_{}'.format(index),
data=sparsity,
step=epoch)
except IndexError:
pass
# try:
# print(layer.layer)
# except:
# print("no", layer)
# save the generator model weights
self.generator.save_weights(
'./saved_models/generator_ckp_epoch_{}'.format(epoch))
self.discriminator.save_weights(
'./saved_models/discriminator_ckp_epoch_{}'.format(epoch))
self.combined.save_weights(
'./saved_models/combined_ckp_epoch_{}'.format(epoch))
model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')
checkpoint_path = "model-1/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3, restore_best_weights=True)
cp_callback = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
initial_epoch = 0
if os.path.exists(checkpoint_path+".index")==False:
print("No model saved.")
else:
print("Model loaded.")
latest = tf.train.latest_checkpoint(checkpoint_dir)
initial_epoch = 0
model.load_weights(latest)
history=model.fit([x_tr,y_tr[:,:-1]],
y_tr.reshape(y_tr.shape[0],y_tr.shape[1], 1)[:,1:],
epochs=25,
initial_epoch=initial_epoch,
callbacks=[es, cp_callback],
batch_size=128,
validation_data=([x_val,y_val[:,:-1]],
y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:]))
reverse_target_word_index=y_tokenizer.index_word
reverse_source_word_index=x_tokenizer.index_word
target_word_index=y_tokenizer.word_index
input_tensor=img_input)
elif C.network == 'resnet152':
from src.architectures import resnet152 as nn
base_layers = ResNet152(weights=None,
include_top=False,
input_tensor=img_input)
with tf.device(device):
print('Loading weights from {}'.format(options.weights))
classifier = nn.classifier(base_layers.output, trainable=False)
optimizer = Adam(learning_rate=0.0001)
model = Model(inputs=base_layers.input, outputs=classifier)
model.load_weights(options.weights)
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
def load_binarizer(path):
global binarizer
with open(path, 'rb') as f:
binarizer = pickle.load(f)
def prepare_images(images, target):
"""
Args:
image: list containing images paths
def Deeplabv3pXception(input_shape=(512, 512, 3),
weights='pascal_voc',
input_tensor=None,
classes=21,
OS=16,
**kwargs):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC. This model is available for TensorFlow only,
and can only be used with inputs following the TensorFlow
data format `(width, height, channels)`.
# Arguments
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images
weights: one of 'pascal_voc' (pre-trained on pascal voc)
or None (random initialization)
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
classes: number of desired classes. If classes != 21,
last layer is initialized randomly
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
# Returns
A Keras model instance.
# Raises
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
ValueError: in case of invalid argument for `weights` or `backbone`
"""
if not (weights in {'pascal_voc', None}):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `pascal_voc` '
'(pre-trained on PASCAL VOC)')
if input_tensor is None:
img_input = Input(shape=input_shape, name='image_input')
else:
img_input = input_tensor
# normalize input image
img_norm = Lambda(normalize, name='input_normalize')(img_input)
# backbone body for feature extract
x, skip_feature, backbone_len = Xception_body(img_norm, OS)
# ASPP block
x = ASPP_block(x, OS)
# Deeplabv3+ decoder for feature projection
x = Decoder_block(x, skip_feature)
# Final prediction conv block
x = DeeplabConv2D(classes, (1, 1), padding='same',
name='logits_semantic')(x)
x = Lambda(img_resize,
arguments={
'size': (input_shape[0], input_shape[1]),
'mode': 'bilinear'
},
name='pred_resize')(x)
x = Reshape((input_shape[0] * input_shape[1], classes))(x)
x = Softmax(name='Predictions/Softmax')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
#if input_tensor is not None:
#inputs = get_source_inputs(input_tensor)
#else:
#inputs = img_input
model = Model(img_input, x, name='deeplabv3p_xception')
# load weights
if weights == 'pascal_voc':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_X,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
return model, backbone_len
class dense2DkernelCNN(denseCNN):
def __init__(self,name='',weights_f=''):
self.name=name
self.pams ={
'CNN_layer_nodes' : [8], #n_filters
'CNN_kernel_size' : [3],
'CNN_padding' : ['same'],
'CNN_pool' : [False],
'share_filters' : True,
'Dense_layer_nodes': [], #does not include encoded layer
'encoded_dim' : 12,
'shape' : (4,4,3),
'channels_first' : False,
'arrange' : [],
'arrMask' : [],
'calQMask' : [],
'n_copy' : 0, # no. of copy for hi occ datasets
'loss' : '',
'optimizer' : 'adam',
}
self.weights_f =weights_f
def init(self,printSummary=True):
encoded_dim = self.pams['encoded_dim']
CNN_layer_nodes = self.pams['CNN_layer_nodes']
CNN_kernel_size = self.pams['CNN_kernel_size']
CNN_padding = self.pams['CNN_padding']
CNN_pool = self.pams['CNN_pool']
Dense_layer_nodes = self.pams['Dense_layer_nodes'] #does not include encoded layer
channels_first = self.pams['channels_first']
share_filters = self.pams['share_filters']
# fix to one cnn layer for now
nnodes =CNN_layer_nodes[0] #8
CNN_kernel =CNN_kernel_size[0] #3
CNN_padding=CNN_padding[0]
inputs = Input(shape=self.pams['shape'], name='input_1')
x = inputs
x1 = Lambda(lambda x: x[:,:,:,0:1], name='lambda_1')(x)
x2 = Lambda(lambda x: x[:,:,:,1:2], name='lambda_2')(x)
x3 = Lambda(lambda x: x[:,:,:,2:3], name='lambda_3')(x)
if share_filters:
conv = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding, name='conv2d_1')
x1 = conv(x1)
x2 = conv(x2)
x3 = conv(x3)
else:
x1 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x1)
x2 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x2)
x3 = Conv2D(nnodes, CNN_kernel, activation='relu',padding=CNN_padding)(x3)
if CNN_pool[0]:
x1 = MaxPooling2D( (2,2), padding='same')(x1)
x2 = MaxPooling2D( (2,2), padding='same')(x2)
x3 = MaxPooling2D( (2,2), padding='same')(x3)
conv_vol_slice = K.int_shape(x1)
x1 = Flatten(name='flatten_1')(x1)
x2 = Flatten(name='flatten_2')(x2)
x3 = Flatten(name='flatten_3')(x3)
x = [x1,x2,x3]
x = Concatenate(axis=-1,name='concat_1')(x)
conv_vol = K.int_shape(x)
encodedLayer = Dense(encoded_dim, activation='relu',name='encoded_vector')(x)
# Instantiate Encoder Model
self.encoder = Model(inputs, encodedLayer, name='encoder')
if printSummary:
self.encoder.summary()
encoded_inputs = Input(shape=(encoded_dim,), name='decoder_input')
x = encoded_inputs
x = Dense(conv_vol[1], activation='relu',name='dense_2')(x)
x = Reshape((conv_vol_slice[1],conv_vol_slice[2],nnodes,3,),name='reshape_1')(x)
x1 = Lambda(lambda x: x[:,:,:,:,0], name='lambda_4')(x)
x2 = Lambda(lambda x: x[:,:,:,:,1], name='lambda_5')(x)
x3 = Lambda(lambda x: x[:,:,:,:,2], name='lambda_6')(x)
if CNN_pool[0]:
x1 = UpSampling2D( (2,2) )(x1)
x2 = UpSampling2D( (2,2) )(x2)
x3 = UpSampling2D( (2,2) )(x3)
## Use n filter here
conv_t = Conv2DTranspose(nnodes, CNN_kernel, activation='relu', padding=CNN_padding, name='conv2d_transpose_1')
x1 = conv_t(x1)
x2 = conv_t(x2)
x3 = conv_t(x3)
## Always use 1 filter
conv_t2 = Conv2DTranspose(1, CNN_kernel, activation=None, padding='same', name='conv2d_transpose_2')
x1 = conv_t2(x1)
x2 = conv_t2(x2)
x3 = conv_t2(x3)
x = [x1,x2,x3]
x = Concatenate(axis=-1,name='concat_2')(x)
outputs = Activation('sigmoid', name='decoder_output')(x)
self.decoder = Model(encoded_inputs, outputs, name='decoder')
if printSummary:
self.decoder.summary()
self.autoencoder = Model(inputs, self.decoder(self.encoder(inputs)), name='autoencoder')
if printSummary:
self.autoencoder.summary()
self.compileModels()
CNN_layers=''
if len(CNN_layer_nodes)>0:
CNN_layers += '_Conv'
for i,n in enumerate(CNN_layer_nodes):
CNN_layers += f'_{n}x{CNN_kernel_size[i]}'
if CNN_pool[i]:
CNN_layers += 'pooled'
Dense_layers = ''
if len(Dense_layer_nodes)>0:
Dense_layers += '_Dense'
for n in Dense_layer_nodes:
Dense_layers += f'_{n}'
self.name = f'Autoencoded{CNN_layers}{Dense_layers}_Encoded_{encoded_dim}'
if not self.weights_f=='':
self.autoencoder.load_weights(self.weights_f)
# Decoder
# -------
# Decoder는 1개 단어씩을 입력으로 받는다.
decoderX = Input(batch_shape=(None, 1))
decEMB = wordEmbedding(decoderX)
decLSTM1 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True)
decLSTM2 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True)
dy1, _, _ = decLSTM1(decEMB, initial_state=[eh1, ec1])
dy2, _, _ = decLSTM2(dy1, initial_state=[eh2, ec2])
decOutput = TimeDistributed(Dense(VOCAB_SIZE, activation='softmax'))
outputY = decOutput(dy2)
# Model
# -----
model = Model([encoderX, decoderX], outputY)
model.load_weights(MODEL_PATH)
# Chatting용 model
model_enc = Model(encoderX, [eh1, ec1, eh2, ec2])
ih1 = Input(batch_shape=(None, LSTM_HIDDEN))
ic1 = Input(batch_shape=(None, LSTM_HIDDEN))
ih2 = Input(batch_shape=(None, LSTM_HIDDEN))
ic2 = Input(batch_shape=(None, LSTM_HIDDEN))
dec_output1, dh1, dc1 = decLSTM1(decEMB, initial_state=[ih1, ic1])
dec_output2, dh2, dc2 = decLSTM2(dec_output1, initial_state=[ih2, ic2])
dec_output = decOutput(dec_output2)
model_dec = Model([decoderX, ih1, ic1, ih2, ic2],
[dec_output, dh1, dc1, dh2, dc2])
def build_darknet19(input_image=None, weights=None, top=True):
# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)
if input_image is None:
input_image = Input(shape=(None, None, 3))
def space_to_depth_x2(x):
return tf.nn.space_to_depth(x, block_size=2)
# Layer 1
x = Conv2D(32, (3, 3),
strides=(1, 1),
padding='same',
name='conv_1',
use_bias=False)(input_image)
x = BatchNormalization(name='norm_1')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 2
x = Conv2D(64, (3, 3),
strides=(1, 1),
padding='same',
name='conv_2',
use_bias=False)(x)
x = BatchNormalization(name='norm_2')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 3
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_3',
use_bias=False)(x)
x = BatchNormalization(name='norm_3')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 4
x = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_4',
use_bias=False)(x)
x = BatchNormalization(name='norm_4')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 5
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_5',
use_bias=False)(x)
x = BatchNormalization(name='norm_5')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 6
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_6',
use_bias=False)(x)
x = BatchNormalization(name='norm_6')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 7
x = Conv2D(128, (1, 1),
strides=(1, 1),
padding='same',
name='conv_7',
use_bias=False)(x)
x = BatchNormalization(name='norm_7')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 8
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_8',
use_bias=False)(x)
x = BatchNormalization(name='norm_8')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 9
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_9',
use_bias=False)(x)
x = BatchNormalization(name='norm_9')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 10
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_10',
use_bias=False)(x)
x = BatchNormalization(name='norm_10')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 11
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_11',
use_bias=False)(x)
x = BatchNormalization(name='norm_11')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 12
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_12',
use_bias=False)(x)
x = BatchNormalization(name='norm_12')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 13
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_13',
use_bias=False)(x)
x = BatchNormalization(name='norm_13')(x)
x = LeakyReLU(alpha=0.1)(x)
skip_connection = x
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 14
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_14',
use_bias=False)(x)
x = BatchNormalization(name='norm_14')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 15
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_15',
use_bias=False)(x)
x = BatchNormalization(name='norm_15')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 16
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_16',
use_bias=False)(x)
x = BatchNormalization(name='norm_16')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 17
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_17',
use_bias=False)(x)
x = BatchNormalization(name='norm_17')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 18
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_18',
use_bias=False)(x)
x = BatchNormalization(name='norm_18')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 19
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_19',
use_bias=False)(x)
x = BatchNormalization(name='norm_19')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 20
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_20',
use_bias=False)(x)
x = BatchNormalization(name='norm_20')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 21
skip_connection = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_21',
use_bias=False)(skip_connection)
skip_connection = BatchNormalization(name='norm_21')(skip_connection)
skip_connection = LeakyReLU(alpha=0.1)(skip_connection)
skip_connection = Lambda(space_to_depth_x2)(skip_connection)
x = concatenate([skip_connection, x])
# Layer 22
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_22',
use_bias=False)(x)
x = BatchNormalization(name='norm_22')(x)
x = LeakyReLU(alpha=0.1)(x)
feature_extractor = Model(input_image, x, name='Full_YOLO_backend')
if weights == 'imagenet':
try:
feature_extractor.load_weights(
'models/weights/full_yolo_backend.h5')
except:
print(
"weights not found or directory does not exist, not loading imagenet weights..."
)
if top == False:
return feature_extractor
else:
# =================
# classification Framework
# =================
output = Conv2D(kwargs.get('num_classes'),
(1, 1))(feature_extractor.output)
output = GlobalAveragePooling2D()(output)
output = Softmax()(output)
model = Model(inputs=feature_extractor.input, outputs=output)
return model
from PIL import Image
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import (LSTM, Embedding, TimeDistributed, Dense,
RepeatVector, Activation, Flatten,
Reshape, concatenate, Dropout,
BatchNormalization)
from tensorflow.keras.optimizers import Adam, RMSprop
from tensorflow.keras import Input, layers
from tensorflow.keras.models import Model
from tensorflow.keras.layers import add
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.utils import to_categorical
from utils import *
inputs1 = Input(shape=(OUTPUT_DIM, ))
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation='relu')(fe1)
inputs2 = Input(shape=(max_length, ))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
se3 = LSTM(256)(se2)
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
outputs = Dense(vocab_size, activation='softmax')(decoder2)
caption_model = Model(inputs=[inputs1, inputs2], outputs=outputs)
caption_model.load_weights(model_path)
def unet():
inputs = Input((image_row, image_col, image_depth))
conv11 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
conc11 = concatenate([inputs, conv11], axis=3)
conv12 = Conv2D(32, (3, 3), activation='relu', padding='same')(conc11)
conc12 = concatenate([inputs, conv12], axis=3)
pool1 = MaxPooling2D(pool_size=(2, 2))(conc12)
conv21 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
conc21 = concatenate([pool1, conv21], axis=3)
conv22 = Conv2D(64, (3, 3), activation='relu', padding='same')(conc21)
conc22 = concatenate([pool1, conv22], axis=3)
pool2 = MaxPooling2D(pool_size=(2, 2))(conc22)
conv31 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
conc31 = concatenate([pool2, conv31], axis=3)
conv32 = Conv2D(128, (3, 3), activation='relu', padding='same')(conc31)
conc32 = concatenate([pool2, conv32], axis=3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conc32)
conv41 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)
conc41 = concatenate([pool3, conv41], axis=3)
conv42 = Conv2D(256, (3, 3), activation='relu', padding='same')(conc41)
conc42 = concatenate([pool3, conv42], axis=3)
drop4 = Dropout(0.5)(conv42)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv51 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)
conc51 = concatenate([pool4, conv51], axis=3)
conv52 = Conv2D(512, (3, 3), activation='relu', padding='same')(conc51)
conc52 = concatenate([pool4, conv52], axis=3)
drop5 = Dropout(0.5)(conc52)
up6 = concatenate([
Conv2DTranspose(256,
(2, 2), strides=(2, 2), padding='same')(drop5), conc42
],
axis=3)
conv61 = Conv2D(256, (3, 3), activation='relu', padding='same')(up6)
conc61 = concatenate([up6, conv61], axis=3)
conv62 = Conv2D(256, (3, 3), activation='relu', padding='same')(conc61)
conc62 = concatenate([up6, conv62], axis=3)
up7 = concatenate([
Conv2DTranspose(128,
(2, 2), strides=(2, 2), padding='same')(conc62), conv32
],
axis=3)
conv71 = Conv2D(128, (3, 3), activation='relu', padding='same')(up7)
conc71 = concatenate([up7, conv71], axis=3)
conv72 = Conv2D(128, (3, 3), activation='relu', padding='same')(conc71)
conc72 = concatenate([up7, conv72], axis=3)
up8 = concatenate([
Conv2DTranspose(64,
(2, 2), strides=(2, 2), padding='same')(conc72), conv22
],
axis=3)
conv81 = Conv2D(64, (3, 3), activation='relu', padding='same')(up8)
conc81 = concatenate([up8, conv81], axis=3)
conv82 = Conv2D(64, (3, 3), activation='relu', padding='same')(conc81)
conc82 = concatenate([up8, conv82], axis=3)
up9 = concatenate([
Conv2DTranspose(32,
(2, 2), strides=(2, 2), padding='same')(conc82), conv12
],
axis=3)
conv91 = Conv2D(32, (3, 3), activation='relu', padding='same')(up9)
conc91 = concatenate([up9, conv91], axis=3)
conv92 = Conv2D(32, (3, 3), activation='relu', padding='same')(conc91)
conc92 = concatenate([up9, conv92], axis=3)
conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conc92)
model = Model(inputs=[inputs], outputs=[conv10])
model.summary()
model.compile(optimizer=Adam(lr=1e-5),
loss='binary_crossentropy',
metrics=[dice_coef, 'accuracy'])
pretrained_weights = None
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
class NAPr:
def __init__(self,
tss_train_file: str = None,
tss_test_file: str = None,
options: dict = None):
""" Options """
self.opts = {"seed": 1,
"n_epochs": 100,
"n_batch_size": 64,
"dropout_rate": 0.2,
"eval_size": 0.1,
"activation_function": "relu"}
self.__setSeed()
if options is not None:
for key in options.keys():
self.opts[key] = options[key]
""" Load data and setup """
if tss_test_file is not None:
self.X_train, self.R_train, self.Y_train = self.__loadData(tss_train_file)
self.X_test, self.R_test, self.Y_test = self.__loadData(tss_test_file)
self.__oneHotEncoderSetup()
self.Y_train = np.asarray(
self.onehot_encoder.transform(self.label_encoder.transform(self.Y_train).reshape(-1, 1)))
self.Y_test = np.asarray(
self.onehot_encoder.transform(self.label_encoder.transform(self.Y_test).reshape(-1, 1)))
self.stdScaler = MinMaxScaler()
self.stdScaler.fit(self.X_train)
self.X_train = self.stdScaler.transform(self.X_train)
self.X_test = self.stdScaler.transform(self.X_test)
self.stdScaler_res = MinMaxScaler()
self.stdScaler_res.fit(self.R_train)
self.R_train = self.stdScaler_res.transform(self.R_train)
self.R_test = self.stdScaler_res.transform(self.R_test)
self.X_train = np.concatenate([self.X_train, self.R_train], axis=1)
self.X_test = np.concatenate([self.X_test, self.R_test], axis=1)
self.X_train, self.X_val, self.Y_train, self.Y_val = train_test_split(self.X_train,
self.Y_train,
test_size=self.opts["eval_size"],
random_state=self.opts["seed"],
shuffle=True)
insize = self.X_train.shape[1]
outsize = len(self.Y_train[0])
class_inputs = Input(shape=(insize,))
l0 = Dropout(self.opts["dropout_rate"])(class_inputs)
bn0 = BatchNormalization()(l0)
ae_x = Dense(300, activation=self.opts["activation_function"])(bn0)
bn1 = BatchNormalization()(ae_x)
l1 = Dropout(self.opts["dropout_rate"])(bn1)
l1 = Dense(200, activation=self.opts["activation_function"], name="classifier_l1")(l1)
bn2 = BatchNormalization()(l1)
l2 = Dropout(self.opts["dropout_rate"])(bn2)
l2 = Dense(100, activation=self.opts["activation_function"], name="classifier_l2")(l2)
bn3 = BatchNormalization()(l2)
l2 = Dropout(self.opts["dropout_rate"])(bn3)
l3 = Dense(50, activation=self.opts["activation_function"], name="classifier_l3")(l2)
bn4 = BatchNormalization()(l3)
l3 = Dropout(self.opts["dropout_rate"])(bn4)
out = Dense(outsize, activation='softmax', name="classifier")(l3)
self.model = Model([class_inputs], [out])
losses = {
"classifier": "categorical_crossentropy",
}
self.model.compile(optimizer='adam', loss=losses, metrics=['accuracy'])
def train(self, checkpoint_path: str,
name: str,
save_results: bool = False):
event_dict_file = str(checkpoint_path) + "/" + str(name) + "_napr_onehotdict.json"
with open(str(event_dict_file), 'w') as outfile:
json.dump(self.one_hot_dict, outfile)
with open(checkpoint_path + "/" + name + "_napr_model.json", 'w') as f:
f.write(self.model.to_json())
ckpt_file = str(checkpoint_path) + "/" + str(name) + "_napr_weights.hdf5"
checkpoint = ModelCheckpoint(ckpt_file, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
hist = self.model.fit(self.X_train,
self.Y_train,
batch_size=self.opts["n_batch_size"],
epochs=self.opts["n_epochs"], shuffle=True,
validation_data=([self.X_val], [self.Y_val]),
callbacks=[self.EvaluationCallback(self.X_test, self.Y_test),
checkpoint])
joblib.dump(self.stdScaler, str(checkpoint_path) + "/" + str(name) + "_napr_stdScaler.pkl")
joblib.dump(self.stdScaler_res, str(checkpoint_path) + "/" + str(name) + "_napr_stdScaler_res.pkl")
if save_results:
results_file = str(checkpoint_path) + "/" + str(name) + "_napr_results.json"
with open(str(results_file), 'w') as outfile:
json.dump(str(hist.history), outfile)
def __oneHotEncoderSetup(self):
""" Events to One Hot"""
events = np.unique(self.Y_train)
self.label_encoder = LabelEncoder()
integer_encoded = self.label_encoder.fit_transform(events)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
self.onehot_encoder = OneHotEncoder(sparse=False)
self.onehot_encoder.fit(integer_encoded)
self.one_hot_dict = {}
for event in events:
self.one_hot_dict[event] = list(self.onehot_encoder.transform([self.label_encoder.transform([event])])[0])
@staticmethod
def __loadData(file: str):
x, r, y = [], [], []
with open(file) as json_file:
tss = json.load(json_file)
for sample in tss:
if sample["label"] is not None:
x.append(list(itertools.chain(sample["tss"][0], sample["tss"][1],
sample["tss"][2])))
r.append(list(sample["tss"][3]))
y.append(sample["label"])
return np.array(x), np.array(r), np.array(y)
def __setSeed(self):
seed(self.opts["seed"])
tf.random.set_seed(self.opts["seed"])
def loadModel(self,
path: str,
name: str):
with open(path + "/" + name + "_napr_model.json", 'r') as f:
self.model = model_from_json(f.read())
self.model.load_weights(path + "/" + name + "_napr_weights.hdf5")
with open(path + "/" + name + "_napr_onehotdict.json", 'r') as f:
self.one_hot_dict = json.load(f)
self.stdScaler = joblib.load(path + "/" + name + "_napr_stdScaler.pkl")
self.stdScaler_res = joblib.load(path + "/" + name + "_napr_stdScaler_res.pkl")
def __intToEvent(self, value: int):
one_hot = list(np.eye(len(self.one_hot_dict.keys()))[value])
for k, v in self.one_hot_dict.items():
if str(v) == str(one_hot):
return k
def predict(self, tss: list):
"""
Predict from a list TimedStateSamples
:param tss: list<TimedStateSamples>
:return: tuple (DREAM-NAP output, translated next event)
"""
if not isinstance(tss, list) or not isinstance(tss[0], TimedStateSample):
raise ValueError("Input is not a list with TimedStateSample")
preds = []
next_events = []
for sample in tss:
features = [list(
itertools.chain(sample.export()["tss"][0], sample.export()["tss"][1],
sample.export()["tss"][2]))]
features = self.stdScaler.transform(features)
r = [list(sample.export()["tss"][3])]
r = self.stdScaler_res.transform(r)
features = np.concatenate([features, r], axis=1)
pred = np.argmax(self.model.predict(features), axis=1)
preds.append(pred[0])
for p in pred:
next_events.append(self.__intToEvent(p))
return preds, next_events
""" Callback """
class EvaluationCallback(Callback):
def __init__(self,
X_test: np.ndarray,
Y_test: np.ndarray):
super().__init__()
self.X_test = X_test
self.Y_test = Y_test
self.Y_test_int = np.argmax(self.Y_test, axis=1)
self.test_accs = []
self.losses = []
def on_train_begin(self,
logs: dict = {}):
self.test_accs = []
self.losses = []
def on_epoch_end(self,
epoch: int,
logs: dict = {}):
y_pred = self.model.predict(self.X_test)
y_pred = y_pred.argmax(axis=1)
test_acc = accuracy_score(self.Y_test_int, y_pred, normalize=True)
test_loss, _ = self.model.evaluate(self.X_test, self.Y_test)
precision, recall, fscore, _ = precision_recall_fscore_support(self.Y_test_int, y_pred, average='weighted',
pos_label=None)
auc = multiclass_roc_auc_score(self.Y_test_int, y_pred, average="weighted")
logs['test_acc'] = test_acc
logs['test_prec_weighted'] = precision
logs['test_rec_weighted'] = recall
logs['test_loss'] = test_loss
logs['test_fscore_weighted'] = fscore
logs['test_auc_weighted'] = auc
precision, recall, fscore, support = precision_recall_fscore_support(self.Y_test_int, y_pred,
average='macro', pos_label=None)
auc = multiclass_roc_auc_score(self.Y_test_int, y_pred, average="macro")
logs['test_prec_mean'] = precision
logs['test_rec_mean'] = recall
logs['test_fscore_mean'] = fscore
logs['test_auc_mean'] = auc
input_path_4 = "/home/mukesh/Documents/IISc/DATA/4_1"
# import random
# random.shuffle(input_path_0)
test_files_0 = glob.glob(os.path.join(input_path_0, "*.jpeg"))
test_files_1 = glob.glob(os.path.join(input_path_1, "*.jpeg"))
test_files_2 = glob.glob(os.path.join(input_path_2, "*.jpeg"))
test_files_3 = glob.glob(os.path.join(input_path_3, "*.jpeg"))
test_files_4 = glob.glob(os.path.join(input_path_4, "*.jpeg"))
print(len(test_files_0))
print(len(test_files_1))
print(len(test_files_2))
print(len(test_files_3))
print(len(test_files_4))
model2.load_weights('Weights_multi.034-0.312.hdf5')
test_files = []
test_files += test_files_1
test_files += test_files_2
test_files += test_files_3
test_files += test_files_4
print(len(test_files))
import random
random.shuffle(test_files_0)
random.shuffle(test_files)
PT = []
GT = []
predict = []
class CommVAE1hot(object):
"""
in_dim : block length
latent_dim : encoding dimension (half of number of channel uses)
h_dim : number of hidden layers
obj_fn : objective function with to optimize over
n0 : noise power (over all components)
sigma2 : prior variance (per component)
"""
def __init__(self,
in_dim=None,
latent_dim=None,
h_dim=None,
obj_fn='RBF',
n0=1.0,
sigma2=1.0):
self.in_dim = in_dim
self.latent_dim = latent_dim
self.n0 = n0
self.sigma2 = sigma2
self.h_dim = h_dim
self.obj_fn = obj_fn
if self.in_dim and self.latent_dim:
self.make_model()
def make_model(self):
# Input layer
self.inputs = Input(shape=(self.in_dim, ), name="enc_in")
# Hidden Layers
x = self.inputs
if self.h_dim is not None:
for (i, d) in enumerate(self.h_dim):
x = Dense(d, activation='relu', name="enc_l{}".format(i))(x)
# Mean and Variance
self.z_mean = Dense(self.latent_dim, name="z_mean")(x)
# Channel
self.z = Lambda(self.channel,
output_shape=(self.latent_dim, ),
name="z")(self.z_mean)
# Encoder model
self.encoder = Model(self.inputs, [self.z_mean, self.z],
name="encoder")
# Decoder
self.latent_inputs = Input(shape=(self.latent_dim, ), name="z_sample")
# Hidden layers
x = self.latent_inputs
if self.h_dim is not None:
for (i, d) in enumerate(self.h_dim[::-1]):
x = Dense(d, activation='relu', name="dec_l{}".format(i))(x)
self.dec_outputs = Dense(self.in_dim,
activation='softmax',
name="decoder_out")(x)
# Decoder model
self.decoder = Model(self.latent_inputs,
self.dec_outputs,
name="decoder")
# VAE
self.outputs = self.decoder(self.encoder(self.inputs)[1])
self.model = Model(self.inputs, self.outputs, name="VAE")
# Losses
self.recon_loss = categorical_crossentropy(self.inputs, self.outputs)
if self.obj_fn == 'AWGN':
# print( "Model with AWGN ")
sig_pow = 0.5 * 1.0 / self.sigma2 * K.sum(K.square(self.z_mean),
axis=-1)
noise_term = 0.5 * self.latent_dim * (
(self.n0 / self.latent_dim) / self.sigma2 - 1.0 - K.log(
(self.n0 / self.latent_dim) / self.sigma2))
self.kl_loss = sig_pow + noise_term
elif self.obj_fn == 'RBF':
# print( "Model with RBF")
sig_pow = 0.5 * 1.0 / self.sigma2 * K.sum(K.square(self.z_mean),
axis=-1)
noise_term = 0.5 * self.latent_dim * (
(self.n0 / self.latent_dim) / self.sigma2 - 1.0 - K.log(
(self.n0 / self.latent_dim) / self.sigma2))
rbf_term = K.log(1.0 + 0.5 * self.latent_dim / self.n0 * sig_pow)
self.kl_loss = sig_pow + noise_term - rbf_term
else:
raise NotImplementedError("Unknown obj_fn: {}".format(self.obj_fn))
self.vae_loss = K.mean(self.recon_loss + self.kl_loss)
self.model.add_loss(self.vae_loss)
self.model.compile(optimizer='adam')
# self.model.compile( optimizer=tf.train.AdamOptimizer(learning_rate=0.01))
def channel(self, zMean):
batch = K.shape(zMean)[0]
# dims = K.shape( zMean )[1]
epsilon = K.random_normal(shape=(batch, self.latent_dim))
return zMean + np.sqrt(self.n0 / self.latent_dim) * epsilon
def fit(self,
x_train,
epochs=10,
batch_size=128,
validation_data=None,
verbose=0,
callbacks=None):
train_log = self.model.fit(x_train,
epochs=epochs,
batch_size=batch_size,
validation_data=validation_data,
verbose=verbose,
callbacks=callbacks)
return train_log.history
def encode(self, data):
return self.encoder.predict(data)
def decode(self, data):
return self.decoder.predict(data)
def analysis(self):
xTest = np.eye(self.in_dim)
enc_mu, enc_z = self.encode(xTest)
dec_mu = self.decode(enc_mu)
dec_z = self.decode(enc_z)
chDim = self.latent_dim // 2
f = plt.figure(figsize=(5 * chDim, 9))
for i in range(chDim):
ax1 = plt.subplot(2, chDim, i + 1)
ax1.scatter(enc_mu[:, i],
enc_mu[:, i + chDim],
c=np.arange(self.in_dim))
for j in range(self.in_dim):
ax1.annotate(j, (enc_mu[j, i], enc_mu[j, i + chDim]))
ax1.set_title("TX Symbols n = {}".format(i + 1))
def save_model(self, fileprefix):
with open(fileprefix + ".dil", "wb") as obj:
dill.dump(
{
'in_dim': self.in_dim,
'latent_dim': self.latent_dim,
'n0': self.n0,
'sigma2': self.sigma2,
'h_dim': self.h_dim,
'obj_fn': self.obj_fn
}, obj)
self.model.save_weights(fileprefix + ".h5")
def load_model(self, fileprefix):
with open(fileprefix + ".dil", "rb") as obj:
config = dill.load(obj)
self.in_dim = config['in_dim']
self.latent_dim = config['latent_dim']
self.n0 = config['n0']
self.h_dim = config['h_dim']
self.obj_fn = config['obj_fn']
if 'sigma2' in config: # For backward compatability
self.sigma2 = config['sigma2']
else:
self.sigma2 = 1.0
self.make_model()
self.model.load_weights(fileprefix + ".h5")
def NASNet(input_shape=None,
penultimate_filters=4032,
nb_blocks=6,
stem_filters=96,
initial_reduction=True,
skip_reduction_layer_input=True,
use_auxiliary_branch=False,
filters_multiplier=2,
dropout=0.5,
weight_decay=5e-5,
include_top=True,
weights=None,
input_tensor=None,
pooling=None,
classes=1000,
default_size=None,
activation='softmax'):
"""Instantiates a NASNet architecture.
Note that only TensorFlow is supported for now,
therefore it only works with the data format
`image_data_format='channels_last'` in your Keras config
at `~/.keras/keras.json`.
# Arguments
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(331, 331, 3)` for NASNetLarge or
`(224, 224, 3)` for NASNetMobile
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(224, 224, 3)` would be one valid value.
penultimate_filters: number of filters in the penultimate layer.
NASNet models use the notation `NASNet (N @ P)`, where:
- N is the number of blocks
- P is the number of penultimate filters
nb_blocks: number of repeated blocks of the NASNet model.
NASNet models use the notation `NASNet (N @ P)`, where:
- N is the number of blocks
- P is the number of penultimate filters
stem_filters: number of filters in the initial stem block
initial_reduction: Whether to perform the reduction step at the beginning
end of the network. Set to `True` for CIFAR models.
skip_reduction_layer_input: Determines whether to skip the reduction layers
when calculating the previous layer to connect to.
use_auxiliary_branch: Whether to use the auxiliary branch during
training or evaluation.
filters_multiplier: controls the width of the network.
- If `filters_multiplier` < 1.0, proportionally decreases the number
of filters in each layer.
- If `filters_multiplier` > 1.0, proportionally increases the number
of filters in each layer.
- If `filters_multiplier` = 1, default number of filters from the paper
are used at each layer.
dropout: dropout rate
weight_decay: l2 regularization weight
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
`imagenet` (ImageNet weights)
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
default_size: specifies the default image size of the model
activation: Type of activation at the top layer.
Can be one of 'softmax' or 'sigmoid'.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if K.backend() != 'tensorflow':
raise RuntimeError('Only Tensorflow backend is currently supported, '
'as other backends do not support '
'separable convolution.')
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
if default_size is None:
default_size = 331
# Determine proper input shape and default size.
input_shape = _obtain_input_shape(input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top or weights)
if K.image_data_format() != 'channels_last':
warnings.warn('The NASNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be " \
"divisible by 24."
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
filters = penultimate_filters // 24
if initial_reduction:
x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid',
use_bias=False, name='stem_conv1', kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(img_input)
else:
x = Conv2D(stem_filters, (3, 3), strides=(1, 1), padding='same', use_bias=False,
name='stem_conv1', kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(img_input)
x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='stem_bn1')(x)
p = None
if initial_reduction: # imagenet / mobile mode
x, p = _reduction_A(x, p, filters // (filters_multiplier ** 2), weight_decay,
id='stem_1')
x, p = _reduction_A(x, p, filters // filters_multiplier, weight_decay,
id='stem_2')
for i in range(nb_blocks):
x, p = _normal_A(x, p, filters, weight_decay, id='%d' % i)
x, p0 = _reduction_A(x, p, filters * filters_multiplier, weight_decay,
id='reduce_%d' % nb_blocks)
p = p0 if not skip_reduction_layer_input else p
for i in range(nb_blocks):
x, p = _normal_A(x, p, filters * filters_multiplier, weight_decay,
id='%d' % (nb_blocks + i + 1))
auxiliary_x = None
if not initial_reduction: # imagenet / mobile mode
if use_auxiliary_branch:
auxiliary_x = _add_auxiliary_head(x, classes, weight_decay, pooling,
include_top, activation)
x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, weight_decay,
id='reduce_%d' % (2 * nb_blocks))
if initial_reduction: # CIFAR mode
if use_auxiliary_branch:
auxiliary_x = _add_auxiliary_head(x, classes, weight_decay, pooling,
include_top, activation)
p = p0 if not skip_reduction_layer_input else p
for i in range(nb_blocks):
x, p = _normal_A(x, p, filters * filters_multiplier ** 2, weight_decay,
id='%d' % (2 * nb_blocks + i + 1))
x = Activation('relu')(x)
if include_top:
x = GlobalAveragePooling2D()(x)
x = Dropout(dropout)(x)
x = Dense(classes, activation=activation,
kernel_regularizer=l2(weight_decay), name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
if use_auxiliary_branch:
model = Model(inputs, [x, auxiliary_x], name='NASNet_with_auxiliary')
else:
model = Model(inputs, x, name='NASNet')
# load weights
if weights == 'imagenet':
if default_size == 224: # mobile version
if include_top:
if use_auxiliary_branch:
weight_path = NASNET_MOBILE_WEIGHT_PATH_WITH_AUXULARY
model_name = 'nasnet_mobile_with_aux.h5'
else:
weight_path = NASNET_MOBILE_WEIGHT_PATH
model_name = 'nasnet_mobile.h5'
else:
if use_auxiliary_branch:
weight_path = NASNET_MOBILE_WEIGHT_PATH_WITH_AUXULARY_NO_TOP
model_name = 'nasnet_mobile_with_aux_no_top.h5'
else:
weight_path = NASNET_MOBILE_WEIGHT_PATH_NO_TOP
model_name = 'nasnet_mobile_no_top.h5'
weights_file = get_file(model_name, weight_path, cache_subdir='models')
model.load_weights(weights_file, by_name=True)
elif default_size == 331: # large version
if include_top:
if use_auxiliary_branch:
weight_path = NASNET_LARGE_WEIGHT_PATH_WITH_auxiliary
model_name = 'nasnet_large_with_aux.h5'
else:
weight_path = NASNET_LARGE_WEIGHT_PATH
model_name = 'nasnet_large.h5'
else:
if use_auxiliary_branch:
weight_path = NASNET_LARGE_WEIGHT_PATH_WITH_auxiliary_NO_TOP
model_name = 'nasnet_large_with_aux_no_top.h5'
else:
weight_path = NASNET_LARGE_WEIGHT_PATH_NO_TOP
model_name = 'nasnet_large_no_top.h5'
weights_file = get_file(model_name, weight_path, cache_subdir='models')
model.load_weights(weights_file, by_name=True)
else:
raise ValueError('ImageNet weights can only be loaded on NASNetLarge '
'or NASNetMobile')
if old_data_format:
K.set_image_data_format(old_data_format)
return model
def ShuffleNet(include_top=True,
input_tensor=None,
scale_factor=1.0,
pooling=None,
input_shape=None,
groups=1,
weights='imagenet',
num_shuffle_units=[3, 7, 3],
bottleneck_ratio=0.25,
classes=1000,
**kwargs):
"""
ShuffleNet implementation for Keras 2
ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices
Xiangyu Zhang, Xinyu Zhou, Mengxiao Lin, Jian Sun
https://arxiv.org/pdf/1707.01083.pdf
Note that only TensorFlow is supported for now, therefore it only works
with the data format `image_data_format='channels_last'` in your Keras
config at `~/.keras/keras.json`.
Parameters
----------
include_top: bool(True)
whether to include the fully-connected layer at the top of the network.
input_tensor:
optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
scale_factor:
scales the number of output channels
input_shape:
pooling:
Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
groups: int
number of groups per channel
num_shuffle_units: list([3,7,3])
number of stages (list length) and the number of shufflenet units in a
stage beginning with stage 2 because stage 1 is fixed
e.g. idx 0 contains 3 + 1 (first shuffle unit in each stage differs) shufflenet units for stage 2
idx 1 contains 7 + 1 Shufflenet Units for stage 3 and
idx 2 contains 3 + 1 Shufflenet Units
bottleneck_ratio:
bottleneck ratio implies the ratio of bottleneck channels to output channels.
For example, bottleneck ratio = 1 : 4 means the output feature map is 4 times
the width of the bottleneck feature map.
classes: int(1000)
number of classes to predict
Returns
-------
A Keras model instance
References
----------
- [ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices]
(http://www.arxiv.org/pdf/1707.01083.pdf)
"""
if K.backend() != 'tensorflow':
raise RuntimeError('Only TensorFlow backend is currently supported, '
'as other backends do not support ')
name = "ShuffleNet_%.2gX_g%d_br_%.2g_%s" % (
scale_factor, groups, bottleneck_ratio, "".join(
[str(x) for x in num_shuffle_units]))
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=28,
require_flatten=include_top,
data_format=K.image_data_format())
out_dim_stage_two = {1: 144, 2: 200, 3: 240, 4: 272, 8: 384}
if groups not in out_dim_stage_two:
raise ValueError("Invalid number of groups.")
if pooling not in ['max', 'avg', None]:
raise ValueError("Invalid value for pooling.")
if not (float(scale_factor) * 4).is_integer():
raise ValueError("Invalid value for scale_factor. Should be x over 4.")
exp = np.insert(np.arange(0, len(num_shuffle_units), dtype=np.float32), 0,
0)
out_channels_in_stage = 2**exp
out_channels_in_stage *= out_dim_stage_two[
groups] # calculate output channels for each stage
out_channels_in_stage[0] = 24 # first stage has always 24 output channels
out_channels_in_stage *= scale_factor
out_channels_in_stage = out_channels_in_stage.astype(int)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
#if not K.is_keras_tensor(input_tensor):
#img_input = Input(tensor=input_tensor, shape=input_shape)
#else:
#img_input = input_tensor
img_input = input_tensor
# create shufflenet architecture
x = YoloConv2D(filters=out_channels_in_stage[0],
kernel_size=(3, 3),
padding='same',
use_bias=False,
strides=(2, 2),
activation="relu",
name="conv1")(img_input)
x = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
padding='same',
name="maxpool1")(x)
# create stages containing shufflenet units beginning at stage 2
for stage in range(0, len(num_shuffle_units)):
repeat = num_shuffle_units[stage]
x = _block(x,
out_channels_in_stage,
repeat=repeat,
bottleneck_ratio=bottleneck_ratio,
groups=groups,
stage=stage + 2)
if include_top:
#x = Dense(units=classes, name="fc")(x)
#x = Activation('softmax', name='softmax')(x)
x = GlobalAveragePooling2D(name='global_avg_pool')(x)
x = Dense(units=classes,
activation='softmax',
use_bias=True,
name='Logits')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D(name='global_avg_pool')(x)
elif pooling == 'max':
x = GlobalMaxPooling2D(name='global_max_pool')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs=inputs, outputs=x, name=name)
# Load weights.
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
raise ValueError('Weights for "channels_first" format '
'are not available.')
if include_top:
model_name = ('shufflenet_weights_tf_dim_ordering_tf_kernels_' +
str(alpha) + '_' + str(rows) + '.h5')
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
else:
model_name = ('shufflenet_weights_tf_dim_ordering_tf_kernels_' +
str(alpha) + '_' + str(rows) + '_no_top' + '.h5')
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
C.num_rois,
nb_classes=len(classes_count),
trainable=True)
## build model for each network Model(input, output)
model_rpn = Model(img_input, rpn[:2]) ## because rpn[2] is base_layers(input)
model_classifier = Model([img_input, roi_input], classifier)
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
try:
##base_net_weight not in config.py but defined above
# print('loading weights from {}'.format('finalmodel_frcnn_cat.h5'))
# model_rpn.load_weights('finalmodel_frcnn_cat.h5', by_name=True)
# model_classifier.load_weights('finalmodel_frcnn_cat.h5', by_name = True)
print('loading weights from {}'.format(
'vgg16_weights_tf_dim_ordering_tf_kernels.h5'))
model_rpn.load_weights('vgg16_weights_tf_dim_ordering_tf_kernels.h5',
by_name=True)
model_classifier.load_weights(
'vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name=True)
except:
print(
'Could not load pretrained model weights. Weights can be found in the keras application folder \
https://github.com/fchollet/keras/tree/master/keras/applications')
## optimizer for each network
optimizer = Adam(lr=1e-6)
optimizer_classifier = Adam(lr=1e-6)
model_rpn.compile(
optimizer=optimizer,
loss=[losses.rpn_loss_cls(num_anchors),
losses.rpn_loss_reg(num_anchors)])
model_classifier.compile(
class VGGish(KerasModelContainer):
""" KerasModelContainer for VGGish model
Jort F. Gemmeke et al.
Audio Set: An ontology and human-labeled dataset for audio events
International Conference on Acoustics, Speech, and Signal Processing.
New Orleans, LA, 2017.
Notes
-----
https://research.google.com/audioset/
Based on vggish-keras https://pypi.org/project/vggish-keras/
Parameters
----------
n_frames_cnn : int or None, default=96
Length of the input (number of frames of each sequence).
n_freq_cnn : int, default=64
Number of frequency bins. The model's input has shape
(n_frames, n_freqs).
n_classes : int, default=10
Number of classes (dimmension output).
n_channels : int, default=0
Number of input channels
0 : mono signals.
Input shape = (n_frames_cnn, n_freq_cnn)
1 : mono signals.
Input shape = (n_frames_cnn, n_freq_cnn, 1)
2 : stereo signals.
Input shape = (n_frames_cnn, n_freq_cnn, 2)
n > 2 : multi-representations.
Input shape = (n_frames_cnn, n_freq_cnn, n_channels)
embedding_size : int, default=128
Number of units in the embeddings layer.
pooling : {'avg', max}, default='avg'
Use AveragePooling or Maxpooling.
include_top : bool, default=False
Include fully-connected layers.
compress : bool, default=False
Apply PCA.
Attributes
----------
model : keras.models.Model
Keras model.
Examples
--------
>>> from dcase_models.model.models import VGGish
>>> model_container = VGGish()
>>> model_container.model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input (InputLayer) (None, 96, 64) 0
_________________________________________________________________
lambda (Lambda) (None, 96, 64, 1) 0
_________________________________________________________________
conv1 (Conv2D) (None, 96, 64, 64) 640
_________________________________________________________________
pool1 (MaxPooling2D) (None, 48, 32, 64) 0
_________________________________________________________________
conv2 (Conv2D) (None, 48, 32, 128) 73856
_________________________________________________________________
pool2 (MaxPooling2D) (None, 24, 16, 128) 0
_________________________________________________________________
conv3/conv3_1 (Conv2D) (None, 24, 16, 256) 295168
_________________________________________________________________
conv3/conv3_2 (Conv2D) (None, 24, 16, 256) 590080
_________________________________________________________________
pool3 (MaxPooling2D) (None, 12, 8, 256) 0
_________________________________________________________________
conv4/conv4_1 (Conv2D) (None, 12, 8, 512) 1180160
_________________________________________________________________
conv4/conv4_2 (Conv2D) (None, 12, 8, 512) 2359808
_________________________________________________________________
pool4 (MaxPooling2D) (None, 6, 4, 512) 0
_________________________________________________________________
global_average_pooling2d_1 ( (None, 512) 0
=================================================================
Total params: 4,499,712
Trainable params: 4,499,712
Non-trainable params: 0
_________________________________________________________________
"""
def __init__(self, model=None, model_path=None, metrics=['classification'],
n_frames_cnn=96, n_freq_cnn=64, n_classes=10,
n_channels=0, embedding_size=128, pooling='avg',
include_top=False, compress=False):
self.n_frames_cnn = n_frames_cnn
self.n_freq_cnn = n_freq_cnn
self.n_classes = n_classes
self.n_channels = n_channels
self.embedding_size = embedding_size
self.pooling = pooling
self.include_top = include_top
self.compress = compress
super().__init__(
model=model, model_path=model_path,
model_name='VGGish', metrics=metrics
)
class Postprocess(Layer):
""" Keras layer that applies PCA and quantizes the ouput.
Based on vggish-keras https://pypi.org/project/vggish-keras/
"""
def __init__(self, output_shape=None, **kw):
self.emb_shape = output_shape
super().__init__(**kw)
def build(self, input_shape):
input_shape = tuple(int(x) for x in tuple(input_shape)[1:])
emb_shape = (self.emb_shape,) if self.emb_shape else input_shape
self.pca_matrix = self.add_weight(
name='pca_matrix', shape=emb_shape + input_shape,
initializer='uniform')
self.pca_means = self.add_weight(
name='pca_means', shape=input_shape + (1,),
initializer='uniform')
def call(self, x):
# Apply PCA.
# - Embeddings come in as [batch_size, embedding_size].
# - Transpose to [embedding_size, batch_size].
# - Subtract pca_means column vector from each column.
# - Premultiply by PCA matrix of shape [output_dims, input_dims]
# where both are are equal to embedding_size in our case.
# - Transpose result back to [batch_size, embedding_size].
x = K.dot(self.pca_matrix, (K.transpose(x) - self.pca_means))
x = K.transpose(x)
# Quantize by:
# - clipping to [min, max] range
# - convert to 8-bit in range [0.0, 255.0]
# - cast 8-bit float to uint8
QUANTIZE_MIN_VAL = -2.0
QUANTIZE_MAX_VAL = +2.0
x = clip_by_value(x, QUANTIZE_MIN_VAL, QUANTIZE_MAX_VAL)
x = ((x - QUANTIZE_MIN_VAL) *
(255.0 / (QUANTIZE_MAX_VAL - QUANTIZE_MIN_VAL)))
return K.cast(x, 'uint8')
def build(self):
""" Builds the VGGish Keras model.
"""
if self.n_channels == 0:
inputs = Input(shape=(self.n_frames_cnn, self.n_freq_cnn),
dtype='float32', name='input')
x = Lambda(
lambda x: K.expand_dims(x, -1), name='lambda'
)(inputs)
else:
inputs = Input(
shape=(self.n_frames_cnn, self.n_freq_cnn, self.n_channels),
dtype='float32', name='input'
)
x = Lambda(lambda x: x, name='lambda')(inputs)
# setup layer params
conv = partial(Conv2D, kernel_size=(3, 3), strides=(
1, 1), activation='relu', padding='same')
maxpool = partial(MaxPooling2D, pool_size=(2, 2),
strides=(2, 2), padding='same')
# Block 1
x = conv(64, name='conv1')(x)
x = maxpool(name='pool1')(x)
# Block 2
x = conv(128, name='conv2')(x)
x = maxpool(name='pool2')(x)
# Block 3
x = conv(256, name='conv3/conv3_1')(x)
x = conv(256, name='conv3/conv3_2')(x)
x = maxpool(name='pool3')(x)
# Block 4
x = conv(512, name='conv4/conv4_1')(x)
x = conv(512, name='conv4/conv4_2')(x)
x = maxpool(name='pool4')(x)
if self.include_top:
dense = partial(Dense, activation='relu')
# FC block
x = Flatten(name='flatten_')(x)
x = dense(4096, name='fc1/fc1_1')(x)
x = dense(4096, name='fc1/fc1_2')(x)
x = dense(self.embedding_size, name='fc2')(x)
if self.compress:
x = self.Postprocess()(x)
else:
globalpool = (
GlobalAveragePooling2D() if self.pooling == 'avg' else
GlobalMaxPooling2D() if self.pooling == 'max' else None)
if globalpool:
x = globalpool(x)
# Create model
self.model = Model(inputs, x, name='vggish_model')
super().build()
def load_pretrained_model_weights(self,
weights_folder='./pretrained_weights'):
"""
Loads pretrained weights to self.model weights.
Parameters
----------
weights_folder : str
Path to load the weights file
"""
basepath = os.path.dirname(__file__)
weights_file = self.model_name + '.hdf5'
weights_path = os.path.join(basepath, weights_folder, weights_file)
if not os.path.isfile(weights_path):
self.download_pretrained_weights()
self.model.load_weights(weights_path, by_name=True)
def download_pretrained_weights(self,
weights_folder='./pretrained_weights'):
"""
Download pretrained weights from:
https://github.com/DTaoo/VGGish
https://drive.google.com/file/d/1mhqXZ8CANgHyepum7N4yrjiyIg6qaMe6/view
Code based on:
https://github.com/beasteers/VGGish/blob/master/vggish_keras/download_helpers/download_weights.py
Parameters
----------
weights_folder : str
Path to save the weights file
"""
import requests
import tqdm
DRIVE_URL = 'https://drive.google.com/uc?id={id}&export=download'
DRIVE_CONFIRM_URL = ('https://drive.google.com/uc?id={id}&export'
'=download&confirm={confirm}')
basepath = os.path.dirname(__file__)
weights_file = self.model_name + '.hdf5'
weights_path = os.path.join(basepath, weights_folder, weights_file)
# gdrive_id = '1mhqXZ8CANgHyepum7N4yrjiyIg6qaMe6'
# This file includes PCA weights
gdrive_id = '1QbMNrhu4RBUO6hIcpLqgeuVye51XyMKM'
if not os.path.isfile(weights_path):
print('Downloading weights...')
sess = requests.Session()
r = sess.get(DRIVE_URL.format(id=gdrive_id), stream=True)
# check for google virus message
confirm = next(
(v for k, v in r.cookies.get_dict().items()
if 'download_warning_' in k), None)
if confirm:
# print('Using confirmation code {}...'.format(confirm))
r = sess.get(
DRIVE_CONFIRM_URL.format(id=gdrive_id, confirm=confirm),
stream=True)
# download w/ progress bar
chunk_size = 1024
unit = 1024 ** 2
with open(weights_path, 'wb') as f:
pbar = tqdm.tqdm(
unit='mb', leave=False,
total=int(
r.headers.get('Content-Length', 0)) / unit or None)
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk: # filter out keep-alive new chunks
pbar.update(len(chunk) / unit)
f.write(chunk)
print('Done!')
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
# input_shape = _obtain_input_shape(
# input_shape,
# default_size=299,
# min_size=139,
# data_format=K.image_data_format(),
# include_top=include_top)
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=True)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = Input(tensor=input_tensor, shape=input_shape)
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl,
96,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3,
320,
3,
3,
strides=(2, 2),
padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(branch7x7x3,
192,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2],
axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
if K.backend() == 'theano':
convert_all_kernels_in_model(model)
return model
