def __init__(self, lr, momentum):
SGD.__init__(self, lr=lr, decay=1e-6, momentum=momentum, nesterov=True)
def __init__(self,
n_tasks,
n_features,
task_type,
n_layers=1,
n_hidden=1000,
init="glorot_uniform",
batchnorm=False,
dropout=0.5,
activation="relu",
learning_rate=.001,
decay=1e-6,
momentum=0.9,
nesterov=False,
**kwargs):
super(MultiTaskDNN, self).__init__()
# Store hyperparameters
assert task_type in ["classification", "regression"]
self.task_type = task_type
self.n_features = n_features
self.n_tasks = n_tasks
self.n_layers = n_layers
self.n_hidden = n_hidden
self.init = init
self.batchnorm = batchnorm
self.dropout = dropout
self.activation = activation
self.learning_rate = learning_rate
self.decay = decay
self.momentum = momentum
self.nesterov = nesterov
self.add_input(name="input", input_shape=(self.n_features, ))
prev_layer = "input"
for ind, layer in enumerate(range(self.n_layers)):
dense_layer_name = "dense%d" % ind
activation_layer_name = "activation%d" % ind
batchnorm_layer_name = "batchnorm%d" % ind
dropout_layer_name = "dropout%d" % ind
self.add_node(Dense(self.n_hidden, init=self.init),
name=dense_layer_name,
input=prev_layer)
prev_layer = dense_layer_name
if self.batchnorm:
self.add_node(BatchNormalization(),
input=prev_layer,
name=batchnorm_layer_name)
prev_layer = batchnorm_layer_name
self.add_node(Activation(self.activation),
name=activation_layer_name,
input=prev_layer)
prev_layer = activation_layer_name
if self.dropout > 0:
self.add_node(Dropout(self.dropout),
name=dropout_layer_name,
input=prev_layer)
prev_layer = dropout_layer_name
for task in range(self.n_tasks):
if self.task_type == "classification":
self.add_node(Dense(2, init=self.init, activation="softmax"),
name="dense_head%d" % task,
input=prev_layer)
elif self.task_type == "regression":
self.add_node(Dense(1, init=self.init),
name="dense_head%d" % task,
input=prev_layer)
self.add_output(name="task%d" % task, input="dense_head%d" % task)
loss_dict = {}
for task in range(self.n_tasks):
taskname = "task%d" % task
if self.task_type == "classification":
loss_dict[taskname] = "binary_crossentropy"
elif self.task_type == "regression":
loss_dict[taskname] = "mean_squared_error"
sgd = SGD(lr=self.learning_rate,
decay=self.decay,
momentum=self.momentum,
nesterov=self.nesterov)
self.compile(optimizer=sgd, loss=loss_dict)
''' Import keras to build a DL model '''
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
print 'Building model using SGD(lr=0.1)'
''' 1. Model using large learning rate '''
model_large = Sequential()
model_large.add(Dense(128, input_dim=200))
model_large.add(Activation('sigmoid'))
model_large.add(Dense(256))
model_large.add(Activation('sigmoid'))
model_large.add(Dense(5))
model_large.add(Activation('softmax'))
''' set the learning rate of SGD optimizer to 0.1 '''
from keras.optimizers import SGD, Adam, RMSprop, Adagrad
sgd010 = SGD(lr=0.1, momentum=0.0, decay=0.0, nesterov=False)
model_large.compile(loss='categorical_crossentropy',
optimizer=sgd010,
metrics=['accuracy'])
history_large = model_large.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=0,
shuffle=True,
validation_split=0.1)
loss_large = history_large.history.get('loss')
acc_large = history_large.history.get('acc')
def main():
init_detector()
args = parser.parse_args()
train_path = args.train_dir
test_path = args.test_dir
if args.gpu:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.85
K.tensorflow_backend.set_session(tf.Session(config=config))
img, age, gender = load_data(train_path)
x_train_data = img
y_train_age = np_utils.to_categorical(age, AGE_CATEGORY)
y_train_gender = np_utils.to_categorical(gender, 2)
img, age, gender = load_data(test_path)
x_test_data = img
y_test_age = np_utils.to_categorical(age, AGE_CATEGORY)
y_test_gender = np_utils.to_categorical(gender, 2)
inputs = Input(shape=INPUT_SHAPE)
x = Conv2D(32, (3, 3), activation='relu')(inputs)
x = MaxPooling2D(2, 2)(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(2, 2)(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D(2, 2)(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = BatchNormalization(axis=-1)(x)
pool = AveragePooling2D(pool_size=(4, 4), strides=(1, 1),
padding="same")(x)
flatten = Flatten()(pool)
predictions_g = Dense(units=2,
kernel_initializer=WEIGHT_INIT,
use_bias=USE_BIAS,
kernel_regularizer=l2(WEIGHT_DECAY),
activation="softmax")(flatten)
predictions_a = Dense(units=AGE_CATEGORY,
kernel_initializer=WEIGHT_INIT,
use_bias=USE_BIAS,
kernel_regularizer=l2(WEIGHT_DECAY),
activation="softmax")(flatten)
model = Model(inputs=inputs, outputs=[predictions_g, predictions_a])
model.summary()
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(
optimizer=sgd,
loss=["categorical_crossentropy", "categorical_crossentropy"],
metrics=['accuracy'])
callbacks = [
EarlyStopping(monitor='val_loss',
patience=100,
verbose=0,
mode='auto',
baseline=None,
restore_best_weights=True),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
hist = model.fit(x_train_data, [y_train_gender, y_train_age],
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=callbacks,
validation_data=(x_test_data, [y_test_gender,
y_test_age]))
print('Train complete.')
with K.get_session() as sess:
# frozen graph
additional_nodes = ['input_1', 'dense_1/Softmax', 'dense_2/Softmax']
frozen_graph = freeze_session(sess, output_names=additional_nodes)
# save model to pb file
tf.train.write_graph(frozen_graph,
"./",
"age_gender_v1.pb",
as_text=False)
tf.train.write_graph(frozen_graph,
"./",
"age_gender_v1.pbtxt",
as_text=True)
kernel_initializer='VarianceScaling'))
model.add(Dropout(dropout))
model.add(
Dense(hidden4,
activation=activation,
name='dense_4',
kernel_initializer='VarianceScaling'))
model.add(
Dense(len(emotions),
activation='softmax',
trainable=trainable,
name='dense_55'))
for learn_rate in learn_rates:
sgd = SGD(lr=learn_rate,
decay=1e-6,
momentum=0.9,
nesterov=False)
rmsdrop = keras.optimizers.RMSprop(lr=learn_rate,
rho=0.9,
epsilon=None,
decay=0.0)
adagrad = keras.optimizers.Adagrad(lr=learn_rate,
epsilon=None,
decay=0.0)
adadelta = keras.optimizers.Adadelta(lr=learn_rate,
rho=0.95,
epsilon=None,
decay=0.0)
adam = keras.optimizers.Adam(lr=learn_rate,
beta_1=0.9,
beta_2=0.999,
model.add(
Convolution2D(32, 7, 7, border_mode='same', input_shape=X_train[0].shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(classes))
model.add(Activation('softmax'))
'''setting optimizer'''
learning_rate = 0.01
learning_decay = 0.01 / 32
sgd = SGD(lr=learning_rate, decay=learning_decay, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# check parameters of every layers
model.summary()
''' training'''
batch_size = 128
epoch = 32
# validation data comes from training data
# model.fit(X_train, Y_train, batch_size=batch_size,
# nb_epoch=epoch, validation_split=0.1, shuffle=True)
# validation data comes from testing data
fit_log = model.fit(X_train,
i = cake(128, 256, 2, 2)(i) # 4*4
# conv5_x
i = cake(256, 512, 2, 2)(i) # 2*2
i = BatchNormalization(axis=-1)(i)
i = relu(i)
i = AveragePooling2D(pool_size=(2, 2), padding='same')(i)
i = Flatten()(i)
i = Dense(12)(i)
i = Activation('softmax')(i)
model = Model(outputs=i, inputs=inp)
sgd = SGD(lr=0.1, decay=0.04, momentum=0.9, nesterov=False)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# # # # # # model specification scripts------------------------------------------------------------
# import the image data from the directory which stores each image under its category folder
# if you want more details about what the data should be arranged for imagedatagenerator to successfully import into the system, see "https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html"
train_datagen = ImageDataGenerator(featurewise_center=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(train_data_dir,
target_size=(img_width,
img_height),
batch_size=batch_size,
def main():
config = configparser.ConfigParser()
config_path = os.path.join("configurations", "master_configuration.ini")
config.read(config_path)
print("\nReading info from configuration:")
args = argument_parser()
if args["operation"] == "Training":
print("\nRunnning in training setting!\n")
NUM_TRAINING = int(config["TRAINING"]["NUM_TRAINING"])
NUM_VALIDATION = int(config["TRAINING"]["NUM_VALIDATION"])
NUM_EPOCHS = int(config["TRAINING"]["NUM_EPOCHS"])
elif args["operation"] == "Development":
print("\nRunnning in development setting!\n")
NUM_TRAINING = int(config["DEVELOPMENT"]["NUM_TRAINING"])
NUM_VALIDATION = int(config["DEVELOPMENT"]["NUM_VALIDATION"])
NUM_EPOCHS = int(config["DEVELOPMENT"]["NUM_EPOCHS"])
else:
print("\nError: Operation should be either 'Training' or 'Development'")
print("Exiting!\n")
sys.exit(1)
if args["epoch"] != "Default":
try:
NUM_EPOCHS = int(args["epoch"])
except ValueError:
print("\nError: Epoch should be an integer.")
print("Exiting!\n")
sys.exit(1)
BATCH_SIZE = int(config["DEFAULT"]["BATCH_SIZE"])
IMAGE_WIDTH = int(config["DEFAULT"]["IMAGE_WIDTH"])
IMAGE_HEIGHT = int(config["DEFAULT"]["IMAGE_HEIGHT"])
IMAGE_DEPTH = int(config["DEFAULT"]["IMAGE_DEPTH"])
CLASS_NAMES = config["DEFAULT"]["CLASS_NAMES"].split()
FEATURE_FILE_TRAINING = config["DEFAULT"]["FEATURE_FILE_TRAINING"]
LABEL_FILE_TRAINING = config["DEFAULT"]["LABEL_FILE_TRAINING"]
FEATURE_FILE_VALIDATION = config["DEFAULT"]["FEATURE_FILE_VALIDATION"]
LABEL_FILE_VALIDATION = config["DEFAULT"]["LABEL_FILE_VALIDATION"]
WEIGHTS = np.array(list(map(float, config["DEFAULT"]["WEIGHTS"].split())))
print("NUM_TRAINING: {}".format(NUM_TRAINING))
print("NUM_VALIDATION: {}".format(NUM_VALIDATION))
print("NUM_EPOCHS: {}".format(NUM_EPOCHS))
print("BATCH_SIZE: {}".format(BATCH_SIZE))
print("IMAGE_WIDTH: {}".format(IMAGE_WIDTH))
print("IMAGE_HEIGHT: {}".format(IMAGE_HEIGHT))
print("IMAGE_DEPTH: {}".format(IMAGE_DEPTH))
print("CLASS_NAMES: {}".format(CLASS_NAMES))
print("FEATURE_FILE_TRAINING: {}".format(FEATURE_FILE_TRAINING))
print("LABEL_FILE_TRAINING: {}".format(LABEL_FILE_TRAINING))
print("FEATURE_FILE_VALIDATION: {}".format(FEATURE_FILE_VALIDATION))
print("LABEL_FILE_VALIDATION: {}".format(LABEL_FILE_VALIDATION))
print("WEIGHTS: {}".format(WEIGHTS))
print()
datasequence_training = DataSequence(feature_file=FEATURE_FILE_TRAINING,
label_file=LABEL_FILE_TRAINING,
image_width=IMAGE_WIDTH,
image_height=IMAGE_HEIGHT,
image_depth=IMAGE_DEPTH,
num_classes=len(CLASS_NAMES),
max_index=NUM_TRAINING,
batch_size=BATCH_SIZE)
datasequence_validation = DataSequence(feature_file=FEATURE_FILE_VALIDATION,
label_file=LABEL_FILE_VALIDATION,
image_width=IMAGE_WIDTH,
image_height=IMAGE_HEIGHT,
image_depth=IMAGE_DEPTH,
num_classes=len(CLASS_NAMES),
max_index=NUM_VALIDATION,
batch_size=BATCH_SIZE)
# Note: num_filters needs to be 16 or less for batch size of 5 (for 6 GB memory)
# Compile the model
input_tensor = Input((IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_DEPTH))
model = get_unet_model(input_tensor=input_tensor, num_classes=len(CLASS_NAMES), num_filters=64,
dropout=0.25,
batchnorm=True)
model.compile(optimizer=SGD(lr=1e-5, decay=0.0),
loss=focal_loss(),
metrics=['accuracy'])
model_and_weights = os.path.join("saved_models", "model_and_weights.hdf5")
# If weights exist, load them before continuing training
continue_training = True
if(os.path.isfile(model_and_weights) and continue_training):
print("Old weights found!")
try:
model.load_weights(model_and_weights)
print("Old weights loaded successfully!")
except:
print("Old weights couldn't be loaded successfully, will continue!")
# Traing the model
history = train_model(model=model,
X=datasequence_training, y=datasequence_validation,
num_training=NUM_TRAINING, num_validation=NUM_VALIDATION,
model_path=model_and_weights, num_epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)
# Plot the history
loss_path = os.path.join("plots", "loss_vs_epoch.pdf")
plot_history(history, quantity='loss', plot_title='Loss', y_label='Loss', plot_name=loss_path)
accuracy_path = os.path.join("plots", "accuracy_vs_epoch.pdf")
plot_history(history, quantity='acc', plot_title='Accuracy', y_label='Accuracy', plot_name=accuracy_path)
# convert the labels from integers to vectors
trainY = LabelBinarizer().fit_transform(trainY)
testY = LabelBinarizer().fit_transform(testY)
# image data generator for data augmentation
augmentation = ImageDataGenerator(rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode="nearest")
# initializing model
print("[INFO] compiling model...")
optimizer = SGD(lr=0.05)
model = MiniVGGNet.build(width=64, height=64, depth=3, classes=len(classNames))
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
# train the network
print("[INFO] training network...")
H = model.fit_generator(augmentation.flow(trainX, trainY, batch_size=32),
validation_data=(testX, testY),
steps_per_epoch=len(trainX) // 32,
epochs=100,
verbose=1)
# evaluate network
print("[INFO] evaluating network...")
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(512, input_shape=(nb_data,), init='normal', name='dense1'))
# a sample is a row 28*28
model.add(Activation('relu', name='relu1'))
model.add(Dropout(0.2, name='dropout1'))
model.add(Dense(512, init='normal', name='dense2'))
model.add(Activation('relu', name='relu2'))
model.add(Dropout(0.2, name='dropout2'))
model.add(Dense(10, init='normal', name='dense3'))
model.add(Activation('softmax', name='softmax1'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.001), metrics=['accuracy'])
tb_cb = keras.callbacks.TensorBoard(log_dir=log_filepath, write_images=1, histogram_freq=1)
# 设置log的存储位置，将网络权值以图片格式保持在tensorboard中显示，设置每一个周期计算一次网络的
#权值，每层输出值的分布直方图
history = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, callbacks=[tb_cb], validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy;', score[1])
test_x = test_x.astype("float") / 255.0
# Convert the labels from integers to vectors
lb = LabelBinarizer()
train_y = lb.fit_transform(train_y)
test_y = lb.transform(test_y)
# Initialize the label names for the CIFAR-10 dataset
label_names = [
"airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse",
"ship", "truck"
]
# Initialize the SGD optimizer
print("[INFO]: Compiling model....")
optimizer = SGD(lr=0.01, decay=0.01 / 40, momentum=0.9, nesterov=True)
model = MiniVGGNet.build(width=32, height=32, depth=3, classes=10)
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
# Construct the set of callbacks
fig_path = os.path.sep.join([args["output"], "{}.png".format(os.getpid())])
json_path = os.path.sep.join([args["output"], "{}.json".format(os.getpid())])
callbacks = [TrainingMonitor(fig_path, json_path)]
# Construct the callback to save only the 'best' model to disk based on the validation loss
checkpoint = ModelCheckpoint(args['weights'],
monitor="val_loss",
mode="min",
save_best_only=True,
model = Sequential()
model.add(Dense(40, input_dim=2 * 1 * nb_classes, init='he_normal'))
model.add(Activation('relu'))
model.add(Dense(40, init='he_normal'))
model.add(Activation('relu'))
model.add(Dense(40, init='he_normal'))
model.add(Activation('relu'))
model.add(Dense(3, init='he_normal'))
model.add(Activation('softmax'))
rms = RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=1.5)
adagrad = Adagrad(clipnorm=1.5)
adam = Adam()
adadelta = Adadelta()
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.95, nesterov=True, clipnorm=1.0)
model.compile(loss='poisson', optimizer=adam)
model.fit(C,
Y,
nb_epoch=10,
batch_size=100,
show_accuracy=True,
verbose=0,
validation_data=(C, Y))
# -----------------------------------------------------
pred_class = model.predict_classes(C, batch_size=200, verbose=0)
s_nn_hat = hstack((zeros(1), pred_class, zeros(1)))
x_nn_hat = N_DUDE.denoise_with_s(z[i], s_nn_hat, 1)
resize_p = ResizePreprocessor(32, 32)
image2array_p = ImageToArrayPreprocessor()
loader = DatasetLoader(preprocessors=[resize_p, image2array_p])
(data, labels) = loader.load(imageFiles, verbose=500)
data = data.astype("float") / 255.0
(trainX, testX, trainY, testY) = train_test_split(data,
labels,
test_size=0.25,
random_state=17)
trainY = LabelBinarizer().fit_transform(trainY)
testY = LabelBinarizer().fit_transform(testY)
print("[INFO] compiling model...")
opt = SGD(lr=0.005)
model = ShallowNet.build(width=32, height=32, depth=3, classes=3)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
print("[INFO] training network...")
N_epoch = 40
H = model.fit(trainX,
trainY,
validation_data=(testX, testY),
batch_size=32,
epochs=N_epoch,
verbose=1)
print("[INFO] predict test data...")
# load dataset
from datasets import load_data
x, y = load_data(args.dataset)
n_clusters = len(np.unique(y))
init = 'glorot_uniform' #初始化器的选择
pretrain_optimizer = 'adam'
# setting parameters
if args.dataset == 'mnist' or args.dataset == 'fmnist':
update_interval = 140
pretrain_epochs = 300
init = VarianceScaling(
scale=1. / 3., mode='fan_in',
distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in)
pretrain_optimizer = SGD(lr=1, momentum=0.9)
elif args.dataset == 'reuters10k':
update_interval = 30
pretrain_epochs = 50
init = VarianceScaling(
scale=1. / 3., mode='fan_in',
distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in)
pretrain_optimizer = SGD(lr=1, momentum=0.9)
elif args.dataset == 'jd':
update_interval = 30
pretrain_epochs = 50
init = VarianceScaling(
scale=1. / 3., mode='fan_in',
distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in)
pretrain_optimizer = SGD(lr=1, momentum=0.9)
elif args.dataset == 'jdReviewWithLabel':
from keras.utils import to_categorical
batch_size = 128
epochs = 20
learning_rate = 0.5
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape([x_train.shape[0], -1]).astype('float32') / 255
y_train = to_categorical(y_train, num_classes=10)
x_test = x_test.reshape([x_test.shape[0], -1]).astype('float32') / 255
y_test = to_categorical(y_test, num_classes=10)
model = Sequential()
model.add(
Dense(10,
activation='sigmoid',
input_shape=(784, ),
kernel_initializer=RandomUniform(minval=-0.01, maxval=0.01)))
model.summary()
model.compile(loss='mean_squared_error',
optimizer=SGD(lr=learning_rate),
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test))
num_users, num_items = train.shape
print("Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d" %
(time() - t1, num_users, num_items, train.nnz, len(testRatings)))
# Build model
model = get_model(num_users, num_items, mf_dim, layers, reg_layers, reg_mf)
if learner.lower() == "adagrad":
model.compile(optimizer=Adagrad(lr=learning_rate),
loss='binary_crossentropy')
elif learner.lower() == "rmsprop":
model.compile(optimizer=RMSprop(lr=learning_rate),
loss='binary_crossentropy')
elif learner.lower() == "adam":
model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy')
else:
model.compile(optimizer=SGD(lr=learning_rate), loss='binary_crossentropy')
# Init performance
(hits, ndcgs) = evaluate_model(model, testRatings, testNegatives, topK,
evaluation_threads)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
print('Init: HR = %.4f, NDCG = %.4f' % (hr, ndcg))
best_hr, best_ndcg, best_iter = hr, ndcg, -1
if out_model > 0:
model.save_weights(model_out_file, overwrite=True)
# Training model
num_epochs = 10
for epoch in range(num_epochs):
t1 = time()
# Generate training instances
def run(args):
lr = args.lr
epochs = args.epochs
decay = args.decay
momentum = args.momentum
h5file = args.model
test_set_path = args.test
hist = args.hist
dataset = pd.read_csv(
os.path.join('/home', 'wvillegas', 'dataset-mask', 'full_masks.csv'))
from utils_fcn import DataGeneratorMobileNet
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(dataset['orig'],
dataset['mask'],
test_size=0.2,
random_state=1)
partition = {'train': list(X_train), 'test': list(X_test)}
img_list = list(X_train) + list(X_test)
mask_list = list(Y_train) + list(Y_test)
labels = dict(zip(img_list, mask_list))
img_path = os.path.join('/home', 'wvillegas', 'dataset-mask',
'dataset_resize', 'images_resize')
masks_path = os.path.join('/home', 'wvillegas', 'dataset-mask',
'dataset_resize', 'masks_resize')
batch_size = 4
train_generator = DataGeneratorMobileNet(batch_size=batch_size,
img_path=img_path,
labels=labels,
list_IDs=partition['train'],
n_channels=3,
n_channels_label=1,
shuffle=True,
mask_path=masks_path)
from keras.applications import MobileNet
from keras.layers import Conv2DTranspose, Conv2D, Add
from keras import Model
net = MobileNet(include_top=False, weights=None)
net.load_weights(
'/home/wvillegas/DLProjects/BudClassifier/cmdscripts/modelosV2/mobilenet_weights_detection.h5',
by_name=True)
for layer in net.layers:
layer.trainable = True
predict = Conv2D(filters=1, kernel_size=1, strides=1)(net.output)
deconv2 = Conv2DTranspose(filters=1,
kernel_size=4,
strides=2,
padding='same',
use_bias=False)(predict)
pred_conv_dw_11_relu = Conv2D(filters=1, kernel_size=1, strides=1)(
net.get_layer('conv_dw_11_relu').output)
fuse1 = Add()([deconv2, pred_conv_dw_11_relu])
pred_conv_pw_5_relu = Conv2D(filters=1, kernel_size=1, strides=1)(
net.get_layer('conv_pw_5_relu').output)
deconv2fuse1 = Conv2DTranspose(filters=1,
kernel_size=4,
strides=2,
padding='same',
use_bias=False)(fuse1)
fuse2 = Add()([deconv2fuse1, pred_conv_pw_5_relu])
deconv8 = Conv2DTranspose(filters=1,
kernel_size=16,
strides=8,
padding='same',
use_bias=False)(fuse2)
fcn = Model(inputs=net.input, outputs=deconv8)
from keras.optimizers import SGD
sgd = SGD(lr=lr, momentum=momentum, decay=decay)
fcn.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
history = fcn.fit_generator(generator=train_generator,
use_multiprocessing=True,
workers=6,
epochs=epochs)
fcn.save(os.path.join(h5file))
test_csv = pd.DataFrame({'x': X_test, 'y': Y_test})
test_csv.to_csv(test_set_path, header=None)
test_csv = pd.DataFrame(history.history)
test_csv.to_csv(hist)
'data_dir': ds_path + '/',
'fullds_data_dir': ds_path + '/',
#'data_dir': ds_path + '/malware_selected_1gram_mini.pkl',
#'fullds_data_dir':ds_path + '/malware_selected_1gram.pkl',
'data_target_list': [1, 2, 3, 4, 5, 6, 7, 8, 9],
#'epochs': 50,
#'epochs': 200,
'epochs': 1000,
'batch': 32,
'store_history': True,
'shuffle_batches': True,
'autoencoder_configs': {
'hidden_layer_activation': 'relu',
'output_layer_activation': 'relu',
'loss_function': 'mse',
'optimizer': SGD(lr=0.01),
'discard_decoder_function': True
},
'mlp_configs': {
'activation': 'sigmoid',
#'activation' : 'softmax',
'loss_function': 'categorical_crossentropy',
'optimizer': SGD(lr=0.01),
'use_last_dim_as_classifier': False,
'classifier_dim': 9
}
}
MAP_DIMS = {
'AE_BIGRAMA_7L_MINIDS_UNDER_01':
[9216, 5530, 4978, 4426, 3874, 3322, 2770, 2217],
def testmodels_test(X_tr,
y_tr,
X_te,
archs,
actfn='relu',
last_act='softmax',
reg_coeffs=[0.0],
num_epoch=30,
batch_size=1000,
sgd_lr=1e-5,
sgd_decays=[0.0],
sgd_moms=[0.0],
sgd_Nesterov=False,
EStop=False,
verbose=0):
[n_tr, d] = X_tr.shape
[n_te, d] = X_te.shape
call_ES = EarlyStopping(monitor='val_acc',
patience=6,
verbose=1,
mode='auto')
for arch in archs:
for reg_coeff in reg_coeffs:
for sgd_decay in sgd_decays:
for sgd_mom in sgd_moms:
# Generate Model
model = genmodel(num_units=arch,
actfn=actfn,
reg_coeff=reg_coeff,
last_act=last_act)
# Compile Model
sgd = SGD(lr=sgd_lr,
decay=sgd_decay,
momentum=sgd_mom,
nesterov=sgd_Nesterov)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# Train Model
if EStop:
model.fit(X_tr,
y_tr,
nb_epoch=num_epoch,
batch_size=batch_size,
verbose=verbose,
callbacks=[call_ES],
validation_split=0.1,
validation_data=None,
shuffle=True)
else:
model.fit(X_tr,
y_tr,
nb_epoch=num_epoch,
batch_size=batch_size,
verbose=verbose)
probability = model.predict_proba(X_te,
batch_size=batch_size,
verbose=verbose)
return probability
def vgg16_model(img_rows, img_cols, channel=1, num_classes=None):
"""VGG 16 Model for Keras
Model Schema is based on
https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3
ImageNet Pretrained Weights
https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
Parameters:
img_rows, img_cols - resolution of inputs
channel - 1 for grayscale, 3 for color
num_classes - number of categories for our classification task
"""
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(channel, img_rows, img_cols)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
# Add Fully Connected Layer
model.add(Flatten())
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1000, activation='softmax'))
# Loads ImageNet pre-trained data
model.load_weights('D:\Projeto\Datase\vgg16_weights.h5')
# Truncate and replace softmax layer for transfer learning
model.layers.pop()
model.outputs = [model.layers[-1].output]
model.layers[-1].outbound_nodes = []
model.add(Dense(num_classes, activation='softmax'))
# Uncomment below to set the first 10 layers to non-trainable (weights will not be updated)
#for layer in model.layers[:10]:
# layer.trainable = False
# Learning rate is changed to 0.001
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def train(
data,
dims,
resolution=0.2,
alpha=1.0,
tol=0.005,
init='glorot_uniform',
n_clusters=None,
n_neighbors=15,
pretrain_epochs=300,
batch_size=256,
activation='relu',
actincenter='tanh',
drop_rate_sae=0.2,
is_stacked=True,
use_early_stop=True,
use_ae_weights=False,
save_encoder_weights=False,
save_dir='tmp_result',
max_iter=1000,
epochs_fit=4,
num_cores=4,
use_gpu=False,
random_seed=1,
verbose=True,
):
if isinstance(data, AnnData):
adata = data
else:
adata = AnnData(data)
assert dims[0] == adata.shape[-1], \
'The number of columns of data needs to be equal to the first element of dims!'
random.seed(random_seed)
np.random.seed(random_seed)
tf.set_random_seed(random_seed)
total_cpu = multiprocessing.cpu_count()
num_cores = int(num_cores) if total_cpu > int(num_cores) else int(
ceil(total_cpu / 2))
print('The number of threads in your computer is', total_cpu)
print('The number of threads used is', num_cores)
if use_gpu:
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
else:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
kb.set_session(
tf.Session(
config=tf.ConfigProto(intra_op_parallelism_threads=num_cores,
inter_op_parallelism_threads=num_cores)))
if not use_ae_weights and os.path.isfile(
os.path.join(save_dir, "ae_weights.h5")):
os.remove(os.path.join(save_dir, "ae_weights.h5"))
tic = get_time() # record time
########################
desc = DescModel(dims=dims,
x=adata.X,
alpha=alpha,
tol=tol,
init=init,
n_clusters=n_clusters,
louvain_resolution=resolution,
n_neighbors=n_neighbors,
pretrain_epochs=pretrain_epochs,
batch_size=batch_size,
random_seed=random_seed,
activation=activation,
actincenter=actincenter,
drop_rate_SAE=drop_rate_sae,
is_stacked=is_stacked,
use_earlyStop=use_early_stop,
use_ae_weights=use_ae_weights,
save_encoder_weights=save_encoder_weights,
save_dir=save_dir)
desc.compile(optimizer=SGD(0.01, 0.9), loss='kld')
embedded, prob = desc.fit(maxiter=max_iter, epochs_fit=epochs_fit)
print("The desc has been trained successfully!!!!!!")
if verbose:
print("The summary of desc model is:")
desc.model.summary()
print("The runtime is ", get_time() - tic)
adata.obsm['embedded'] = embedded
adata.obsm['prob'] = prob
adata.obs['ident'] = np.argmax(prob, axis=1)
adata.obs['max_prob'] = _get_max_prob(prob)
return adata
def main(args):
# set the image parameters
img_rows = args.img_rows
img_cols = args.img_cols
img_dim_color = args.img_channels
# mix_prop = 1.0 # set the value of the mixing proportion
#############################################################################################################
################################## DEFINING MODEL ##########################################################
##############################################################################################################
model_alex = cnn_hybrid_color_single(img_rows, img_cols,
img_dim_color) # load the model
# model_final = Model(model_alex.input, model_alex.output) # specify the input and output of the model
model_final = model_alex
print(model_final.summary()) # print the model summary
plot_model(model_final,
to_file='./NIN_hybrid_bin_resnet_1x1-class',
show_shapes=True) # save the model summary as a png file
lr = args.learning_rate # set the learning rate
# set the optimizer
optimizer = SGD(lr=lr, decay=1e-6, momentum=0.9)
# model compilation
model_final.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# print the value of the learning rate
print(K.get_value(optimizer.lr))
# --------------------------------------------------
#############################################################################################################
########################## GETTING TRAINING DATA AND TESTING DATA ##########################################
##############################################################################################################
# get the training data by calling the pairs function
# read the training data
train_pairs_r, training_data_r, training_label_r = read_pairs(
args.tr_img_lab_r)
train_pairs_l, training_data_l, training_label_l = read_pairs(
args.tr_img_lab_l)
assert len(training_data_r) == len(training_data_l)
# combine the left and right image in the training data to make a X x Y x 6 tensor
training_data = []
for i in range(len(training_data_r)):
# define the stereo pair
stereo_pair = [training_data_r[i], training_data_l[i]]
training_data.append(stereo_pair)
batch_num = 0
# initialize the live samples and fake samples
live_samples_ub = 0
attack_samples_ub = 0
live_samples = []
live_labels = []
attack_samples = []
attack_labels = []
# separate the live samples and fake samples to balance the both classes, i.e. live class and fake class
assert len(training_label_r) == len(training_label_l)
for i in range(len(training_data)):
if training_label_r[i] == 0:
live_samples.append(training_data[i])
live_labels.append(training_label_r[i])
live_samples_ub += 1
elif (training_label_r[i] == 1) | (training_label_r[i] == 3) | (
training_label_r[i] == 4): # cut photo attack removed
attack_samples.append(training_data[i])
attack_labels.append(training_label_r[i])
attack_samples_ub += 1
print("Live samples are %g ,\t attack samples are %g" %
(live_samples_ub, attack_samples_ub))
# compute the difference; the live samples are always less than the fake samples in our case
diff = 0
if live_samples_ub < attack_samples_ub:
# compute the ratio
diff = np.int(attack_samples_ub / live_samples_ub)
print("The difference is :%g " % (diff))
else:
ValueError("The fake samples are less than then live samples")
# number of times the dataset has to be copied:
live_samples_b = live_samples
live_labels_b = live_labels
for i in range(diff - 1):
# print("length before balancing: %g" %len(live_samples_b))
sl_copy = live_samples.copy()
ll_copy = live_labels.copy()
live_samples_b = live_samples_b + sl_copy
live_labels_b = live_labels_b + ll_copy
# print("length after balancing: %g" % len(live_samples_b))
# balanced data
training_data_balanced = live_samples_b + attack_samples
training_label_balanced = live_labels_b + attack_labels
print("Balanced data samples: %g" % len(training_data_balanced))
# get the length of the training data
len_tr = len(training_data_balanced)
# get the number equal to the length of the training data
indices_tr = np.arange(len_tr)
np.random.shuffle(indices_tr)
# initialize the image counter
images_read = 0
train_img_data_r = []
train_img_data_l = []
for i in indices_tr:
if training_label_balanced[i] > 0:
training_label_balanced[i] = 1
train_img_data_r.append(
[training_data_balanced[i][0],
training_label_balanced[i]]) # read the right image
train_img_data_l.append(
[training_data_balanced[i][1],
training_label_balanced[i]]) # read the left image
# print(training_data_balanced[i][1])
# cv2.imshow('img1', cv2.imread(training_data_balanced[i][0]))
# cv2.waitKey()
# cv2.imshow('img2', cv2.imread(training_data_balanced[i][1]))
# cv2.waitKey()
images_read += 1
sys.stdout.write('train images read = {0}\r'.format(images_read))
sys.stdout.flush()
############################################################################################################
# read the test data
test_pairs, test_data_r, test_labels_r = read_pairs(args.tst_img_lab_r)
test_pairs, test_data_l, test_labels_l = read_pairs(args.tst_img_lab_l)
assert len(test_data_r) == len(test_data_l)
# combine the left and right image in the training data to make a X x Y x 6 tensor
test_data = []
for i in range(len(test_data_r)):
# define the stereo pair
stereo_pair_t = [test_data_r[i], test_data_l[i]]
test_data.append(stereo_pair_t)
test_labels = test_labels_r
images_read = 0
# get the length of the training data
len_test = len(test_data)
# get the number equal to the length of the training data
indices_test = np.arange(len_test)
test_img_data_r = []
test_img_data_l = []
for i in indices_test:
if test_labels[i] > 0:
test_labels[i] = 1
test_img_data_r.append([test_data[i][0],
test_labels[i]]) # read the right test image
test_img_data_l.append([test_data[i][1],
test_labels[i]]) # red the left test image
images_read += 1
sys.stdout.write('test images read = {0}\r'.format(images_read))
sys.stdout.flush()
#####################################################################################################
# make all the data in panda data frame format
train_df_r = pd.DataFrame(train_img_data_r)
train_df_r.columns = ['id_r', 'label']
train_df_l = pd.DataFrame(train_img_data_l)
train_df_l.columns = ['id_l', 'label']
test_df_r = pd.DataFrame(test_img_data_r)
test_df_r.columns = ['id_r', 'label']
test_df_l = pd.DataFrame(test_img_data_l)
test_df_l.columns = ['id_l', 'label']
########################################################################################################333
datagen = image.ImageDataGenerator()
train_generator_r = datagen.flow_from_dataframe(
dataframe=train_df_r,
directory=None,
x_col='id_r',
y_col='label',
has_ext=True,
batch_size=args.batch_size,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(args.img_rows, args.img_cols),
color_mode='grayscale',
interpolation='nearest',
drop_duplicates=False)
train_generator_l = datagen.flow_from_dataframe(
dataframe=train_df_l,
directory=None,
x_col='id_l',
y_col='label',
has_ext=True,
batch_size=args.batch_size,
seed=42,
shuffle=True,
class_mode="sparse",
target_size=(args.img_rows, args.img_cols),
color_mode='grayscale',
interpolation='nearest',
drop_duplicates=False)
test_datagen = image.ImageDataGenerator()
test_generator_r = test_datagen.flow_from_dataframe(
dataframe=test_df_r,
directory=None,
x_col='id_r',
y_col='label',
has_ext=True,
batch_size=args.batch_size,
seed=42,
shuffle=False,
class_mode="sparse",
target_size=(args.img_rows, args.img_cols),
color_mode='grayscale',
interpolation='nearest')
test_generator_l = test_datagen.flow_from_dataframe(
dataframe=test_df_l,
directory=None,
x_col='id_l',
y_col='label',
has_ext=True,
batch_size=args.batch_size,
seed=42,
shuffle=False,
class_mode="sparse",
target_size=(args.img_rows, args.img_cols),
color_mode='grayscale',
interpolation='nearest')
#############################################################################################################
batch_num = 0
while batch_num < args.max_epochs:
start_time = time.time() # initialize the clock
acc = []
loss = []
sub_count = 0
total_batch = train_generator_r.n // train_generator_r.batch_size
for i in range(train_generator_r.n // train_generator_r.batch_size):
x1, y1 = next(train_generator_r)
x2, y2 = next(train_generator_l)
# only for DP-3D for comparison
# disparity_final = []
#
# for j in range(x1.shape[0]):
# img1 = np.asarray(x1[j])
# # img1 = cv2.resize(img1, (img_rows, img_cols),
# # interpolation=cv2.INTER_AREA)
#
# img2 = np.asarray(x2[j])
# # img2 = cv2.resize(img2, (img_rows, img_cols),
# # interpolation=cv2.INTER_AREA)
# #
# disparity = cv2.subtract(img1, img2)
#
# der_k = np.asarray([[1.0, 2.0, 1.0],
# [0.0, 0.0, 0.0],
# [-1.0, -2.0, -1.0]])
#
# der = cv2.filter2D(img1, -1, kernel=der_k)
#
# disparity_f = disparity / (der + 0.005)
#
# disparity_final.append(disparity_f)
#
# # disparity_final = np.asarray(disparity_final).astype('float32')
# disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1)
x1 = x1.astype('float32') / 255
x2 = x2.astype('float32') / 255
y = y1
tr_acc1 = model_final.fit([x1, x2], y, epochs=1, verbose=0)
acc.append(tr_acc1.history['acc'][0])
loss.append(tr_acc1.history['loss'][0])
sub_count += 1
sys.stdout.write('batch_count = {0} of {1} \r'.format(
sub_count, total_batch))
sys.stdout.flush()
train_acc = np.sum(np.asarray(acc)) * 100 / (
train_generator_r.n // train_generator_r.batch_size)
train_loss = np.sum(np.asarray(loss)) * 100 / (
train_generator_r.n // train_generator_r.batch_size)
print('training_acc: {0} \t training_loss: {1}'.format(
train_acc, train_loss))
print(
'______________________________________________________________________'
)
print('Running the evaluations')
test_acc = []
test_loss = []
sub_count = 0
for i in range(test_generator_r.n // test_generator_r.batch_size):
x1, y1 = next(test_generator_r)
x2, y2 = next(test_generator_l)
# only for DP-3D for comparison
# disparity_final = []
#
# for j in range(x1.shape[0]):
# img1 = np.asarray(x1[j])
# # img1 = cv2.resize(img1, (img_rows, img_cols),
# # interpolation=cv2.INTER_AREA)
#
# img2 = np.asarray(x2[j])
# # img2 = cv2.resize(img2, (img_rows, img_cols),
# # interpolation=cv2.INTER_AREA)
# #
# disparity = cv2.subtract(img1, img2)
#
# der_k = np.asarray([[1.0, 2.0, 1.0],
# [0.0, 0.0, 0.0],
# [-1.0, -2.0, -1.0]])
#
# der = cv2.filter2D(img1, -1, kernel=der_k)
#
# disparity_f = disparity / (der + 0.005)
#
# disparity_final.append(disparity_f)
#
# # disparity_final = np.asarray(disparity_final).astype('float32')
# disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1)
x1 = x1.astype('float32') / 255
x2 = x2.astype('float32') / 255
y = y1
tst_loss, tst_acc1 = model_final.evaluate([x1, x2], y, verbose=0)
test_acc.append(tst_acc1)
test_loss.append(tst_loss)
sub_count += 1
sys.stdout.write('epoch_count = {0}\r'.format(sub_count))
sys.stdout.flush()
test_acc = np.sum(np.asarray(test_acc)) * 100 / (
test_generator_r.n // test_generator_r.batch_size)
test_loss = np.sum(np.asarray(test_loss)) * 100 / (
test_generator_r.n // test_generator_r.batch_size)
print('test_acc: {0} \t test_loss: {1}'.format(test_acc, test_loss))
batch_num += 1
# **********************************************************************************************
# learning rate schedule update: if learning is done using a single learning give the batch_num below a
# high value
if (batch_num == 3) | (batch_num == 5) | (batch_num == 7):
lr = 0.1 * lr
K.set_value(optimizer.lr, lr)
print(K.get_value(optimizer.lr))
# ************************************************************************************************
# -----------------------------------------------------------------------------------------------
end_time = time.time() - start_time
print("Total time taken %f :" % end_time)
model_final.save_weights(
'/home/yaurehman2/Documents/stereo_face_liveness/stereo_ckpt/Conventional/'
+ 'dual_grayscale_input_revised_protocol_3_' +
str(args.max_epochs) + '.h5')
xtrain = np.array(xlist[:numTrainers])
ytrain = np.array(ylist[:numTrainers])
xtest = np.array(xlist[numTrainers:])
ytest = np.array(ylist[numTrainers:])
# Create the model
model = Sequential()
model.add(Dense(64, activation='relu', input_shape=(
featureVec_size, ))) #small dataset - less hidden layers needed
model.add(Dense(40, activation='relu'))
model.add(Dense(final_dim, activation='sigmoid'))
model.summary()
# Stochastic Gradient Descent
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])
history = model.fit(xtrain, ytrain, epochs=200, batch_size=batch_sz)
# Time to test!
score = model.evaluate(xtest, ytest, batch_size=batch_sz)
print "\nCompiled! Here are your results..."
print('Test loss:', score[0])
print('Test accuracy:', score[1])
all_featurevecs = xlist
all_labels = ylist
summaries = cross_validate(model, all_featurevecs, all_labels, 3)
return_sequences=True))(layer_b)
decoder = dot([layer_a, layer_b], axes=1, normalize=True)
#dense_1 = Dense(2048, activation='relu')(pool_2)
#dense_2 = Dense(1024, activation='relu')(dense_1)
conv_1 = Conv1D(128, word_length)(decoder)
pool_1 = MaxPooling1D(5)(conv_1)
drop = Dropout(0.5)(pool_1)
global_pool = GlobalMaxPooling1D()(drop)
dense_3 = Dense(512, activation='relu')(global_pool)
output = Dense(10, activation='sigmoid')(dense_3)
model = Model(inputs=[encoder_a, encoder_b], outputs=output)
adam = Adam(lr=learning_rate)
sgd = SGD(lr=learning_rate, nesterov=True, decay=1e-6, momentum=0.9)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
print('Training shapes:', x_train.shape, y_train.shape)
print('Valid shapes:', x_valid.shape, y_valid.shape)
print(model.summary())
history = model.fit_generator(
generate_batch(x_train, y_train, tokenizer),
steps_per_epoch=steps_per_epoch,
epochs=1, #len(x_train)//batch_size//steps_per_epoch,
validation_data=generate_batch(x_valid, y_valid, tokenizer),
validation_steps=steps_per_epoch)
# Save the weights
model.save_weights('model_weights.h5')
#SGD is an optimizer that uses a fixed learning rate
#Start by using a function that creates a new model
#create models in a for loops
#each time we loop through, we compile the model using SGD with a different learning rate
#We have created a function get_new_model() that creates an unoptimized model to optimize.
from keras.optimizers import SGD
def get_new_model(input_shape=input_shape):
model = Sequential()
model.add(Dense(100, activation='relu', input_shape=input_shape))
model.add(Dense(100, activation='relu'))
model.add(Dense(2, activation='softmax'))
return (model)
#Create a list of learning rates
lr_to_test = [.000001, 0.01, 1]
#Loop over the learning rates
for lr in lr_to_test:
#Build new model to test, unaffected by previous models
model = get_new_model()
#Create SGD optimizer with specified learning rate: my_optimizer
my_optimizer = SGD(lr=lr)
#Compile the model
model.compile(optimizer=my_optimizer, loss='categorical_crossentropy')
#Fit the model
model.fit(predictors, target)
from possum.datagen import datagen
from keras.optimizers import SGD
from keras.utils import to_categorical, plot_model
from keras.callbacks import ModelCheckpoint, CSVLogger
import numpy as np
layers = 3
logger = CSVLogger("../log/r{:d}layer.log".format(layers))
X_valid = np.load("../data/valid/X.npy")
y_valid = to_categorical(np.load("../data/valid/label.npy"))
nu = ASKAP12()
generator = datagen(nu, batch=64)
sgd = SGD(lr=0.005, decay=0.001, momentum=0.9, nesterov=True)
cnn = classifier(width=201)
for i in range(2):
cnn.conv(filters=64)
cnn.batchNorm()
cnn.activation('relu')
cnn.pool(max_pool=True)
for i in range(layers):
cnn.residual(conv=3, activation='relu', batchNorm=True)
cnn.globalPooling(max_pool=True)
cnn.dropout(0.3)
cnn.compile(optimizer=sgd)
plot_model(cnn.model, 'residual.png')
def run_optimizer(n, iters):
current_pop, current_pars = generate_custompop(n)
score_list = []
print("Iteration 1")
for i in range(n):
print("Population " + str(i + 1))
#evaluate
cnn_n = train(current_pop[i])
scores = cnn_n.evaluate(x_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))
score_list.append(scores[1] * 100)
m = max(score_list)
ind = score_list.index(m)
for ite in range(iters - 1):
print("Iteration " + str(ite + 1))
current_pop, current_pars = get_next_generation(current_pars[ind], n)
score_list = []
for i in range(n):
#evaluate
print("Population " + str(i + 1))
cnn_n = train(current_pop[i])
scores = cnn_n.evaluate(x_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))
score_list.append(scores[1] * 100)
m = max(score_list)
ind = score_list.index(m)
print('\n.\n.\n.\n.\nGetting best model')
cnn_n = train(current_pop[ind])
sgd = SGD(lr=current_pars[ind][0],
decay=current_pars[ind][1],
momentum=current_pars[ind][0],
nesterov=True)
#save and retrieve
with open("/home/aswin/proj/assets/model_num.txt", 'r') as f:
model_num = int(f.read())
model_json = cnn_n.to_json()
with open(
"/home/aswin/proj/assets/models/model" + str(model_num) + ".json",
"w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
cnn_n.save_weights("/home/aswin/proj/assets/model" + str(model_num) +
".h5")
print("Saved model to disk")
# load json and create model
json_file = open(
"/home/aswin/proj/assets/models/model" + str(model_num) + ".json", 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("/home/aswin/proj/assets/model" +
str(model_num) + ".h5")
print("Loaded model from disk")
# evaluate loaded model on test data
loaded_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
score = loaded_model.evaluate(x_test, y_test, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100))
with open("/home/aswin/proj/assets/model_num.txt", 'r+') as f:
f.truncate(0)
f.write(str(model_num + 1))
import numpy as np
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.optimizers import SGD
from keras.utils import np_utils
from make_tensorboard import make_tensorboard
np.random.seed(1671) # for reproducibility
# network and training
NB_EPOCH = 20
BATCH_SIZE = 128
VERBOSE = 1
NB_CLASSES = 10 # number of outputs = number of digits
OPTIMIZER = SGD() # optimizer, explained later in this chapter
N_HIDDEN = 128 # 隠れ層のニューロン数
VALIDATION_SPLIT = 0.2 # how much TRAIN is reserved for VALIDATION
# data: shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784
RESHAPED = 784
#
X_train = X_train.reshape(60000, RESHAPED)
X_test = X_test.reshape(10000, RESHAPED)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
# normalize
def __init__(self):
# Variables to hold the description of the experiment
self.config_description = "This is the template config file."
# System dependent variable
self._workers = 5
self._multiprocessing = True
# Variables for comet.ml
self._project_name = "jpeg_deep"
self._workspace = "ssd"
# Network variables
self._weights = "/dlocal/home/2017018/bdegue01/weights/jpeg_deep/classification_dct/resnet_deconv/classification_dct_jpeg-deep_GYftBmXMdjdxoMksyI3e9VqB5IriBC9T/checkpoints/epoch-87_loss-0.7459_val_loss-1.5599.h5"
self._network = SSD300_resnet(backbone="deconv_rfa",
dct=True,
image_shape=(38, 38))
# Training variables
self._epochs = 240
self._batch_size = 32
self._steps_per_epoch = 1000
self.optimizer_parameters = {"lr": 0.001, "momentum": 0.9}
self._optimizer = SGD(**self.optimizer_parameters)
self._loss = SSDLoss(neg_pos_ratio=3, alpha=1.0).compute_loss
self._metrics = None
dataset_path = environ["DATASET_PATH"]
images_2007_path = join(dataset_path, "VOC2007/JPEGImages")
self.train_sets = [(images_2007_path,
join(dataset_path,
"VOC2007/ImageSets/Main/train.txt"))]
self.validation_sets = [(images_2007_path,
join(dataset_path,
"VOC2007/ImageSets/Main/val.txt"))]
self.test_sets = [(images_2007_path,
join(dataset_path,
"VOC2007/ImageSets/Main/test.txt"))]
# Keras stuff
self.model_checkpoint = None
self.reduce_lr_on_plateau = ReduceLROnPlateau(patience=5, verbose=1)
self.terminate_on_nan = TerminateOnNaN()
self.early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=15)
self._callbacks = [
self.reduce_lr_on_plateau, self.early_stopping,
self.terminate_on_nan
]
self.input_encoder = SSDInputEncoder()
self.train_tranformations = [SSDDataAugmentation()]
self.validation_transformations = [
ConvertTo3Channels(),
Resize(height=300, width=300)
]
self.test_transformations = [
ConvertTo3Channels(),
Resize(height=300, width=300)
]
self._train_generator = None
self._validation_generator = None
self._test_generator = None
self._horovod = None
def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python=False):
if not graph and not edge_f:
raise Exception('graph/edge_f needed')
if not graph:
graph = graph_util.loadGraphFromEdgeListTxt(edge_f)
if self._subsample:
S = graph_util.randwalk_DiGraph_to_adj(graph, node_frac=self._node_frac, n_walks_per_node=self._n_walks_per_node, len_rw=self._len_rw)
else:
S = graph_util.transform_DiGraph_to_adj(graph)
if not np.allclose(S.T, S):
print "SDNE only works for symmetric graphs! Making the graph symmetric"
t1 = time()
S = (S + S.T)/2 # enforce S is symmetric
S -= np.diag(np.diag(S)) # enforce diagonal = 0
self._node_num = S.shape[0]
n_edges = np.count_nonzero(S) # Double counting symmetric edges deliberately to maintain autoencoder symmetry
# Create matrix B
B = np.ones(S.shape)
B[S != 0] = self._beta
# compute degree of each node
deg = np.sum(S!=0, 1)
# Generate encoder, decoder and autoencoder
self._num_iter = self._n_iter
# If cannot use previous step information, initialize new models
self._encoder = get_encoder(self._node_num, self._d, self._K, self._n_units, self._nu1, self._nu2, self._actfn)
self._decoder = get_decoder(self._node_num, self._d, self._K, self._n_units, self._nu1, self._nu2, self._actfn)
self._autoencoder = get_autoencoder(self._encoder, self._decoder)
# Initialize self._model
# Input
x_in = Input(shape=(2*self._node_num,), name='x_in')
x1 = Lambda(lambda x: x[:,0:self._node_num], output_shape=(self._node_num,))(x_in)
x2 = Lambda(lambda x: x[:,self._node_num:2*self._node_num], output_shape=(self._node_num,))(x_in)
# Process inputs
[x_hat1, y1] = self._autoencoder(x1)
[x_hat2, y2] = self._autoencoder(x2)
# Outputs
x_diff1 = merge([x_hat1, x1], mode=lambda (a,b): a - b, output_shape=lambda L: L[1])
x_diff2 = merge([x_hat2, x2], mode=lambda (a,b): a - b, output_shape=lambda L: L[1])
y_diff = merge([y2, y1], mode=lambda (a,b): a - b, output_shape=lambda L: L[1])
# Objectives
def weighted_mse_x(y_true, y_pred):
''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them.
y_pred: Contains x_hat - x
y_true: Contains [b, deg]
'''
return KBack.sum(KBack.square(y_pred * y_true[:,0:self._node_num]), axis=-1)/y_true[:,self._node_num]
def weighted_mse_y(y_true, y_pred):
''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them.
y_pred: Contains y2 - y1
y_true: Contains s12
'''
min_batch_size = KBack.shape(y_true)[0]
return KBack.reshape(KBack.sum(KBack.square(y_pred), axis=-1),[min_batch_size, 1]) * y_true
# Model
self._model = Model(input=x_in, output=[x_diff1, x_diff2, y_diff])
sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
# adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
self._model.compile(optimizer=sgd, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha])
# self._model.compile(optimizer=adam, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha])
# Structure data in the correct format for the SDNE model
# InData format: [x1, x2]
# OutData format: [b1, b2, s12, deg1, deg2]
data_chunk_size = 100000
InData = np.zeros((data_chunk_size, 2*self._node_num))
OutData = np.zeros((data_chunk_size, 2*self._node_num + 3))
# Train the model
for epoch_num in range(self._num_iter):
print 'EPOCH %d/%d' % (epoch_num, self._num_iter)
e = 0
k = 0
for i in range(self._node_num):
for j in range(self._node_num):
if(S[i,j] != 0):
temp = np.append(S[i,:], S[j,:])
InData[k,:] = temp
temp = np.append(np.append(np.append(np.append(B[i,:], B[j,:]), S[i,j]), deg[i]), deg[j])
OutData[k,:] = temp
e += 1
k += 1
if k == data_chunk_size:
self._model.fit(InData, [ np.append(OutData[:,0:self._node_num], np.reshape(OutData[:,2*self._node_num+1], [data_chunk_size, 1]), axis=1),
np.append(OutData[:,self._node_num:2*self._node_num], np.reshape(OutData[:,2*self._node_num+2], [data_chunk_size, 1]), axis=1),
OutData[:,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1)
k = 0
if k > 0:
self._model.fit(InData[:k, :], [ np.append(OutData[:k,0:self._node_num], np.reshape(OutData[:k,2*self._node_num+1], [k, 1]), axis=1),
np.append(OutData[:k,self._node_num:2*self._node_num], np.reshape(OutData[:k,2*self._node_num+2], [k, 1]), axis=1),
OutData[:k,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1)
# Get embedding for all points
_, self._Y = self._autoencoder.predict(S, batch_size=self._n_batch)
t2 = time()
# Save the autoencoder and its weights
if(self._weightfile is not None):
saveweights(self._encoder, self._weightfile[0])
saveweights(self._decoder, self._weightfile[1])
if(self._modelfile is not None):
savemodel(self._encoder, self._modelfile[0])
savemodel(self._decoder, self._modelfile[1])
if(self._savefilesuffix is not None):
saveweights(self._encoder, 'encoder_weights_'+self._savefilesuffix+'.hdf5')
saveweights(self._decoder, 'decoder_weights_'+self._savefilesuffix+'.hdf5')
savemodel(self._encoder, 'encoder_model_'+self._savefilesuffix+'.json')
savemodel(self._decoder, 'decoder_model_'+self._savefilesuffix+'.json')
# Save the embedding
np.savetxt('embedding_'+self._savefilesuffix+'.txt', self._Y)
return self._Y, (t2-t1)
def cross_validate_inmemory(model_name, **kwargs):
"""
StateFarm competition:
Training set has 26 unique drivers. We do 26 fold CV where
a driver is alternatively singled out to be the validation set
Load the whole train data in memory for faster operations
args: model (keras model)
**kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
nb_classes = kwargs["nb_classes"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
prob = kwargs["prob"]
do_plot = kwargs["do_plot"]
data_file = kwargs["data_file"]
semi_super_file = kwargs["semi_super_file"]
pretr_weights_file = kwargs["pretr_weights_file"]
normalisation_style = kwargs["normalisation_style"]
weak_labels = kwargs["weak_labels"]
objective = kwargs["objective"]
experiment = kwargs["experiment"]
start_fold = kwargs["start_fold"]
# Load env variables in (in .env file at the root of the project)
load_dotenv(find_dotenv())
# Load env variables
model_dir = os.path.expanduser(os.environ.get("MODEL_DIR"))
data_dir = os.path.expanduser(os.environ.get("DATA_DIR"))
# Output path where we store experiment log and weights
model_dir = os.path.join(model_dir, model_name)
# Create if it does not exist
general_utils.create_dir(model_dir)
# Automatically determine experiment name
list_exp = glob.glob(model_dir + "/*")
# Create the experiment dir and weights dir
if experiment:
exp_dir = os.path.join(model_dir, experiment)
else:
exp_dir = os.path.join(model_dir, "Experiment_%s" % len(list_exp))
general_utils.create_dir(exp_dir)
# Compile model.
# opt = RMSprop(lr=5E-6, rho=0.9, epsilon=1e-06)
opt = SGD(lr=5e-4, decay=1e-6, momentum=0.9, nesterov=True)
# opt = Adam(lr=1E-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Batch generator
DataAug = batch_utils.AugDataGenerator(data_file,
batch_size=batch_size,
prob=prob,
dset="train",
maxproc=4,
num_cached=60,
random_augm=False,
hdf5_file_semi=semi_super_file)
DataAug.add_transform("h_flip")
# DataAug.add_transform("v_flip")
# DataAug.add_transform("fixed_rot", angle=40)
DataAug.add_transform("random_rot", angle=40)
# DataAug.add_transform("fixed_tr", tr_x=40, tr_y=40)
DataAug.add_transform("random_tr", tr_x=40, tr_y=40)
# DataAug.add_transform("fixed_blur", kernel_size=5)
DataAug.add_transform("random_blur", kernel_size=5)
# DataAug.add_transform("fixed_erode", kernel_size=4)
DataAug.add_transform("random_erode", kernel_size=3)
# DataAug.add_transform("fixed_dilate", kernel_size=4)
DataAug.add_transform("random_dilate", kernel_size=3)
# DataAug.add_transform("fixed_crop", pos_x=10, pos_y=10, crop_size_x=200, crop_size_y=200)
DataAug.add_transform("random_crop", min_crop_size=140, max_crop_size=160)
# DataAug.add_transform("hist_equal")
# DataAug.add_transform("random_occlusion", occ_size_x=100, occ_size_y=100)
epoch_size = n_batch_per_epoch * batch_size
general_utils.pretty_print("Load all data...")
with h5py.File(data_file, "r") as hf:
X = hf["train_data"][:, :, :, :]
y = hf["train_label"][:].astype(np.uint8)
y = np_utils.to_categorical(y, nb_classes=nb_classes) # Format for keras
try:
for fold in range(start_fold, 8):
# for fold in np.random.permutation(26):
min_valid_loss = 100
# Save losses
list_train_loss = []
list_valid_loss = []
# Load valid data in memory for fast error evaluation
idx_valid = hf["valid_fold%s" % fold][:]
idx_train = hf["train_fold%s" % fold][:]
X_valid = X[idx_valid]
y_valid = y[idx_valid]
# Normalise
X_valid = normalisation(X_valid, normalisation_style)
# Compile model
general_utils.pretty_print("Compiling...")
model = models.load(model_name,
nb_classes,
X_valid.shape[-3:],
pretr_weights_file=pretr_weights_file)
model.compile(optimizer=opt, loss=objective)
# Save architecture
json_string = model.to_json()
with open(os.path.join(data_dir, '%s_archi.json' % model.name), 'w') as f:
f.write(json_string)
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
l_train_loss = []
start = time.time()
for X_train, y_train in DataAug.gen_batch_inmemory(X, y, idx_train=idx_train):
if do_plot:
general_utils.plot_batch(X_train, np.argmax(y_train, 1), batch_size)
# Normalise
X_train = normalisation(X_train, normalisation_style)
train_loss = model.train_on_batch(X_train, y_train)
l_train_loss.append(train_loss)
batch_counter += 1
progbar.add(batch_size, values=[("train loss", train_loss)])
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
y_valid_pred = model.predict(X_valid, verbose=0, batch_size=16)
train_loss = float(np.mean(l_train_loss)) # use float to make it json saveable
valid_loss = log_loss(y_valid, y_valid_pred)
print("Train loss:", train_loss, "valid loss:", valid_loss)
list_train_loss.append(train_loss)
list_valid_loss.append(valid_loss)
# Record experimental data in a dict
d_log = {}
d_log["fold"] = fold
d_log["nb_classes"] = nb_classes
d_log["batch_size"] = batch_size
d_log["n_batch_per_epoch"] = n_batch_per_epoch
d_log["nb_epoch"] = nb_epoch
d_log["epoch_size"] = epoch_size
d_log["prob"] = prob
d_log["optimizer"] = opt.get_config()
d_log["augmentator_config"] = DataAug.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
json_file = os.path.join(exp_dir, 'experiment_log_fold%s.json' % fold)
general_utils.save_exp_log(json_file, d_log)
# Only save the best epoch
if valid_loss < min_valid_loss:
min_valid_loss = valid_loss
trained_weights_path = os.path.join(exp_dir, '%s_weights_fold%s.h5' % (model.name, fold))
model.save_weights(trained_weights_path, overwrite=True)
except KeyboardInterrupt:
pass
from sklearn.preprocessing import normalize
# from sklearn.model_selection import StratifiedKFold, cross_val_score
from itertools import combinations_with_replacement, product
from datetime import datetime
from configparser import ConfigParser
import matplotlib.pyplot as plt
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
reading_index = slice(None, None, None)
optimizer_dict = {
'adam': Adam(),
'nadam': Nadam(),
'sgd': SGD(lr=0.0000001, decay=0, momentum=0.6, nesterov=True),
'adadelta': Adadelta(clipnorm=1.)
}
# seed for reproducibility
np.random.seed(1337)
pair_slice_choices = [[2], [2, 6], [0, 2, 4, 6], slice(None, None, None)]
triplet_slice_choices = [[8], [8, 9], [8, 9, 26, 27],
[0, 1, 8, 9, 18, 19, 26, 27],
[
0, 1, 4, 5, 8, 9, 12, 13, 18, 19, 22, 23, 26, 27,
30, 31
],
slice(None, None, None)]
