def train_vgg16():
"""
This is the VGG16 model, which is also mentioned in our report.
:return: A VGG16 model in .pt format
"""
datapath = os.path.join("data", "CK_data.h5")
f = h5py.File(datapath, "r")
images = np.array(f['data_samples'])
labels = np.array(f['data_labels'])
dataset = load_data_vgg(images, labels)
np.random.shuffle(dataset)
len = dataset.shape[0]
spliter = np.split(dataset, [int(np.floor(len * 0.7)), int(len)])
model = VGG16(alpha=0.01,
epochs=25,
batch_size=1,
dataset=spliter,
num_classes=7)
model._train()
dst_path = "model_data/vgg16_ck.pt"
if not os.path.exists(os.path.dirname(dst_path)):
os.makedirs(os.path.dirname(dst_path))
torch.save(model, dst_path)
def init() -> None:
"""
The main function of the project used to initialise all the required classes
used when training the model
"""
# get input arguments
args = get_args()
# get static config information
config = process_config()
# combine both into dictionary
config = {**config, **args}
# initialise model
model = VGG16(config)
# create your data generators for each mode
train_data = TFRecordShardLoader(config, mode="train")
val_data = TFRecordShardLoader(config, mode="val")
test_data = TFRecordShardLoader(config, mode="test")
# initialise the estimator
trainer = Trainer(config, model, train_data, val_data, test_data)
# start training
trainer.run()
def build_graph(self, image, label):
image = image / 128.0 - 1.0
if self.args.name == 'VGG16':
logit, recon = VGG16(image, classes=self.args.types)
elif self.args.name == 'ShuffleNet':
logit = ShuffleNet(image, classes=self.args.types)
elif self.args.name == 'ResNet101':
logit, recon = ResNet101(image,
mode=self.args.mode,
classes=self.args.types)
elif self.args.name == 'DenseNet121':
logit, recon = DenseNet121(image, classes=self.args.types)
elif self.args.name == 'DenseNet169':
logit, recon = DenseNet169(image, classes=self.args.types)
elif self.args.name == 'DenseNet201':
logit, recon = DenseNet201(image, classes=self.args.types)
elif self.args.name == 'InceptionBN':
logit = InceptionBN(image, classes=self.args.types)
else:
pass
estim = tf.sigmoid(logit, name='estim')
loss_xent = class_balanced_sigmoid_cross_entropy(logit,
label,
name='loss_xent')
# loss_dice = tf.identity(1.0 - dice_coe(estim, label, axis=[0,1], loss_type='jaccard'),
# name='loss_dice')
# # Reconstruction
# with argscope([Conv2D, Conv2DTranspose], use_bias=False,
# kernel_initializer=tf.random_normal_initializer(stddev=0.02)), \
# argscope([Conv2D, Conv2DTranspose, InstanceNorm], data_format='channels_first'):
# recon = (LinearWrap(recon)
# .Conv2DTranspose('deconv0', 64 * 8, 3, strides=2)
# .Conv2DTranspose('deconv1', 64 * 8, 3, strides=2)
# .Conv2DTranspose('deconv2', 64 * 4, 3, strides=2)
# .Conv2DTranspose('deconv3', 64 * 2, 3, strides=2)
# .Conv2DTranspose('deconv4', 64 * 1, 3, strides=2)
# .tf.pad([[0, 0], [0, 0], [3, 3], [3, 3]], mode='SYMMETRIC')
# .Conv2D('recon', 1, 7, padding='VALID', activation=tf.tanh, use_bias=True)())
# recon = tf.transpose(recon, [0, 2, 3, 1])
# loss_mae = tf.reduce_mean(tf.abs(recon-image), name='loss_mae')
# Visualization
visualize_tensors('image', [image],
scale_func=lambda x: x * 128.0 + 128.0,
max_outputs=max(64, self.args.batch))
# Regularize the weight of model
wd_w = tf.train.exponential_decay(2e-4, get_global_step_var(), 80000,
0.7, True)
wd_cost = tf.multiply(wd_w,
regularize_cost('.*/W', tf.nn.l2_loss),
name='wd_cost')
add_param_summary(('.*/W', ['histogram'])) # monitor W
cost = tf.add_n([loss_xent, wd_cost], name='cost')
add_moving_summary(loss_xent)
add_moving_summary(wd_cost)
add_moving_summary(cost)
return cost
def get_model():
# model = AlexNet()
model = VGG16()
# model = VGG19()
model.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
metrics=['accuracy'])
return model
def _load_model(model_name, include_top=True, weights='imagenet'):
if model_name == 'resnet50':
from models.resnet50 import ResNet50
model = ResNet50(weights=weights, include_top=include_top)
elif model_name == 'vgg16':
from models.vgg16 import VGG16
model = VGG16(weights=weights, include_top=include_top)
if weights == 'hybrid1365':
model.load_weights(join(weights_dir,
'vgg16_hybrid1365_weights.h5'))
elif weights == 'places365':
model.load_weights(join(weights_dir, 'vgg16_places365_weights.h5'))
elif model_name == 'vgg19':
from models.vgg19 import VGG19
model = VGG19(weights=weights, include_top=include_top)
elif model_name == 'inception':
from models.inception_v3 import InceptionV3
model = InceptionV3(weights=weights, include_top=include_top)
elif model_name == 'vgg16_hybrid1365':
from models.vgg16 import VGG16
model = VGG16(weights=weights, include_top=include_top)
model.load_weights(join(weights_dir, 'vgg16_hybrid1365_weights.h5'),
by_name=True)
elif model_name == 'vgg16_places365':
from models.vgg16 import VGG16
model = VGG16(weights=weights, include_top=include_top)
model.load_weights(join(weights_dir, 'vgg16_places365_weights.h5'),
by_name=True)
elif model_name == 'resnet152_hybrid1365':
from models.resnet import ResnetBuilder
model = ResnetBuilder.build_resnet_152((3, 224, 224), 100)
print(join(weights_dir, 'resnet152_hybrid1365.h5'))
model.load_weights(join(weights_dir, 'resnet152_hybrid1365.h5'),
by_name=True)
else:
print(
'Not a valid model. Valid models are vgg16, vgg19, resnet50, inception, vgg16_hybrid1365, and vgg16_places365'
)
sys.exit(1)
return model
def classify(img_in, img_out):
images = list()
# load the VGG16 network
print("[INFO] loading network...")
model = VGG16(weights="imagenet")
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='categorical_crossentropy')
for item in os.listdir(img_in):
if not item.startswith('.') and os.path.isfile(
os.path.join(img_in, item)):
print(item)
img = cv2.imread(os.path.join(img_in, item))
if img is not None:
images.append(img)
print("[INFO] loading and preprocessing image...")
image = cv2.resize(img, (224, 224))
image = image_utils.img_to_array(image)
# our image is now represented by a NumPy array of shape (3, 224, 224),
# but we need to expand the dimensions to be (1, 3, 224, 224) so we can
# pass it through the network -- we'll also preprocess the image by
# subtracting the mean RGB pixel intensity from the ImageNet dataset
image = np.expand_dims(image, axis=0)
image = preprocess_input(image)
# classify the image
print("[INFO] classifying image...")
preds = model.predict(image)
P = decode_predictions(preds)
(imagenetID, label, prob) = P[0][0]
#print('Predicted:', decode_predictions(preds,1))
print(decode_predictions(preds, 3))
# display the predictions to our screen
print("ImageNet ID: {}, Output label: {}, Prob: {}".format(
imagenetID, label, round(prob, 4)))
cv2.putText(img, "Output label: {}".format(label), (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.9, (102, 0, 255), 2)
cv2.putText(img, "Prob: {0:.0f}%".format(prob * 100), (10, 90),
cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 0, 0), 2)
#create a new folder
foldoutcheck = os.path.exists(img_out)
if foldoutcheck == False:
os.makedirs(img_out)
foldcheck = os.path.exists(img_out + "/" + label)
if foldcheck == True:
cv2.imwrite(img_out + "/" + label + "/" + item, img)
else:
os.makedirs(img_out + "/" + label)
cv2.imwrite(img_out + "/" + label + "/" + item, img)
def main(model, weights_file, classes_file, alpha, image_file, target_file):
if model.lower() == 'vgg16':
vgg16 = VGG16(weights_file, classes_file)
model = vgg16.build_model(1, alpha)
image = vgg16.load_image(image_file)
classes = vgg16.classes
logits, relevances = propagate(image, model)
prediction = classes[str(np.argmax(logits[0]))]
print('predicted class: {}'.format(prediction))
if not os.path.exists('results'):
os.makedirs('results')
plt.imshow(heatmap(relevances[0]))
plt.axis('off')
plt.savefig(os.path.join('results', target_file), bbox_inches='tight', pad_inches=0)
print('heatmap saved as {}'.format(os.path.join('results', target_file)))
def __init__(self):
super(RPN, self).__init__()
self.features = VGG16(bn=False)
self.conv1 = Conv2d(512, 512, 3, same_padding=True)
self.score_conv = Conv2d(512,
len(self.anchor_scales) * 3 * 2,
1,
relu=False,
same_padding=False)
self.bbox_conv = Conv2d(512,
len(self.anchor_scales) * 3 * 4,
1,
relu=False,
same_padding=False)
# loss
self.cross_entropy = None
self.los_box = None
def multi_vgg(name, input_shape, time, classes, dropout, l2_reg, weights,
include_top):
model = VGG16(name=name,
input_shape=input_shape,
classes=classes,
dropout=dropout,
l2_reg=l2_reg,
weights=weights,
include_top=False)
distributed_shape = (time, ) + input_shape
time_model = Sequential()
time_model.add(TimeDistributed((model), input_shape=distributed_shape))
# Freeze all layers
for layer in time_model.layers:
layer.trainable = False
return time_model
def load_model(model_name):
print("Loading {} model...".format(model_name))
if model_name == "resnet50":
model = ResNet50(weights='imagenet')
elif model_name == "inceptionv3":
model = InceptionV3(weights='imagenet')
elif model_name == "vgg19":
model = VGG19(weights='imagenet')
elif model_name == "vgg16":
model = VGG16(weights='imagenet')
elif model_name == "alexnet":
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model = convnet('alexnet',
weights_path="../alexnet/alexnet_weights.h5",
heatmap=False)
model.compile(optimizer=sgd, loss='mse')
else:
raise ValueError("Wrong model name")
print("Loaded!")
return model, model_name
def feather_loss(syn_celar, clear, syn_blur, blur):
cuda_avail = torch.cuda.is_available()
model = VGG16()
for p in model.parameters():
p.requires_grad = False
if cuda_avail:
model.cuda()
# load model
path = 'models/VGG16model_99'
model_dict = torch.load(path)
model.load_state_dict(model_dict['model'])
model.eval()
feather_syn_c, output_c = model(syn_celar)
feather_c, output_cc = model(clear)
# print(feather_syn_c.size())
similarity = torch.cosine_similarity(feather_c, feather_syn_c, dim=1)
loss_c = torch.mean(1 - similarity, dim=0)
# loss_c=l1loss(feather_syn_c,feather_c)
feather_syn_b, output_b = model(syn_blur)
feather_b, output_bb = model(blur)
similarity2 = torch.cosine_similarity(feather_b, feather_syn_b, dim=1)
loss_b = torch.mean(1 - similarity2, dim=0)
# loss_b=l1loss(feather_syn_b,feather_b)
_, predictionb = torch.max(output_b.data, 1) #0
_, predictionbb = torch.max(output_bb.data, 1) #
_, predictionc = torch.max(output_c.data, 1) #2
nb = 0
for i in range(0, len(predictionb)):
if predictionb[i] == 0:
nb += 1
nc = 0
for i in range(0, len(predictionc)):
if predictionc[i] == 2:
nc += 1
return loss_c, loss_b, nc, nb
def build_graph(self, image, label):
image = image / 128.0 - 1.0
if self.args.name == 'VGG16':
logit, recon = VGG16(image, classes=self.args.types)
elif self.args.name == 'ShuffleNet':
logit = ShuffleNet(image, classes=self.args.types)
elif self.args.name == 'ResNet101':
logit, recon = ResNet101(image, mode=self.args.mode, classes=self.args.types)
elif self.args.name == 'DenseNet121':
logit, recon = DenseNet121(image, classes=self.args.types)
elif self.args.name == 'DenseNet169':
logit, recon = DenseNet169(image, classes=self.args.types)
elif self.args.name == 'DenseNet201':
logit, recon = DenseNet201(image, classes=self.args.types)
elif self.args.name == 'InceptionBN':
logit = InceptionBN(image, classes=self.args.types)
else:
pass
estim = tf.sigmoid(logit, name='estim')
loss_xent = class_balanced_sigmoid_cross_entropy(logit, label, name='loss_xent')
# Visualization
visualize_tensors('image', [image], scale_func=lambda x: x * 128.0 + 128.0,
max_outputs=max(64, self.args.batch))
# Regularize the weight of model
wd_w = tf.train.exponential_decay(2e-4, get_global_step_var(),
80000, 0.7, True)
wd_cost = tf.multiply(wd_w, regularize_cost('.*/W', tf.nn.l2_loss), name='wd_cost')
add_param_summary(('.*/W', ['histogram'])) # monitor W
cost = tf.add_n([loss_xent, wd_cost], name='cost')
add_moving_summary(loss_xent)
add_moving_summary(wd_cost)
add_moving_summary(cost)
return cost
split='train',
augmentation=train_datagen)
val_set = SpatialDataGenerator(
r'D:\Mestrado\databases\UCF101\test',
r'D:\Mestrado\databases\UCF101\test\frame_info.csv',
r'D:\Mestrado\databases\UCF101\classes_index.csv',
10,
16, (224, 224, 3),
split='val',
augmentation=train_datagen)
model = VGG16('spatial',
input_shape=(224, 224, 3),
classes=3,
dropout=0.9,
l2_reg=1e-5,
weights='imagenet',
include_top=True)
model.summary()
learning_rate = 1e-5
print('Learning rate: %.10f' % learning_rate)
model.compile(keras.optimizers.Adam(learning_rate=learning_rate),
keras.losses.CategoricalCrossentropy(),
metrics=['acc'])
history = model.fit_generator(train_set,
validation_data=val_set,
verbose=1,
import tensorflow as tf
from models.vgg16 import VGG16
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
from keras import models, layers, optimizers, backend
from keras.preprocessing.image import ImageDataGenerator
backend.set_session(session=tf.Session(config=tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True))))
vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=(224, 224, 3))
vgg16.summary()
for layer in vgg16.layers:
layer.trainable = False
model = models.Sequential()
model.add(vgg16)
model.add(layers.Flatten())
model.add(layers.Dense(4096, activation='relu', name='fc1'))
model.add(layers.Dense(4096, activation='relu', name='fc2'))
model.add(layers.Lambda(lambda x: backend.l2_normalize(x, axis=1)))
model.add(layers.Dense(2, activation='softmax', name='predictions'))
model.summary()
for layer in model.layers:
print(layer, layer.trainable)
train_dir = "../../data/data_a2/train"
validation_dir = "../../data/data_a2/validation"
coord.request_stop()
coord.join(threads)
if __name__ == '__main__':
with tf.Graph().as_default():
with tf.device('/cpu:0'):
train_batch, train_label_batch = cifar10_input.read_cifar10(
data_path=data_path,
is_train=True,
batch_size=batch_size,
shuffle=True)
val_batch, val_label_batch = cifar10_input.read_cifar10(
data_path=data_path,
is_train=False,
batch_size=batch_size,
shuffle=False)
xs = tf.placeholder(tf.float32, shape=[batch_size, img_h, img_w, 3])
ys = tf.placeholder(tf.int32, shape=[batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
model = VGG16(xs, num_classes, keep_prob)
logits = model.logits
loss = tools.losses(logits, ys)
acc = tools.accuracy(logits, ys)
train_op = tools.optimizer(loss, learning_rate)
train_running()
import numpy as np
from models.vgg16 import VGG16
from keras.preprocessing import image
from models.imagenet_utils import preprocess_input ,decode_predictions
model=VGG16(include_top=True,weights='imagenet')
img=image.load_img("elephent.jpeg",target_size=(224,224))
x=image.img_to_array(img)
print(type(x))
x = np.expand_dims(x,axis=0)
x=preprocess_input(x)
predict=model.predict(x)
print("prediction",decode_predictions(predict))
model.summary()
model.layers[-1].get_config()
//////////////////////////////////////////////
model=VGG16(weights='imagenet',include_top=False)
model.summary()
def __init__(self):
self.model = VGG16()
self.get_classes()
# define the params updating function using SGD
# loss will become nan if init lr = 0.01
optimizer = optim.SGD(
net.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=0.0005,
)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
"min",
factor=0.1,
)
elif model_name == "vgg16":
# instantiate the neural network
net = VGG16()
# define the loss function using CrossEntropyLoss
criterion = nn.CrossEntropyLoss()
# "The batch size was set to 256, momentum to 0.9. The training was regularised by
# weight decay (the L2 penalty multiplier set to 5*10^4) and dropout regularisation
# for the first two fully-connected layers (dropout ratio set to 0.5).
# The learning rate was initially set to 10−2" vgg16.[1]
# "Multi-GPU training exploits data parallelism, and is carried out by splitting each batch of
# training images into several GPU batches, processed in parallel on each GPU." vgg16[1]
# However, since I'm training on one GPU, to avoid "CUDA out of memory" issue, I have to reduce the
# batch size here
# using lr=0.01 with kaiming init will result in loss explosion
# 35.06133270263672
# 115.53974151611328
if args.train == "true":
date = str(datetime.now().date())
base_save_dir = os.path.join(args.save_path, date)
os.makedirs(base_save_dir)
shutil.copyfile(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "models",
"vgg16.py"), os.path.join(base_save_dir, "vgg16.py"))
final_train_perfs = {}
final_val_perfs = {}
for lr in args.lrs:
print("LR: {}".format(lr))
print("Building model")
model = VGG16(args.vgg_weights, default_arch_weights)
model.compile(optimizer=Adam(lr=lr), loss="binary_crossentropy")
print("Model built")
save_path = os.path.join(base_save_dir, "lr{}".format(lr))
os.makedirs(save_path)
train_generator = RandomBatchGenerator(args.batch_size, ["train"],
args.imdir,
args.augment == "true",
True)
val_generator = RandomBatchGenerator(args.batch_size, ["val"],
args.imdir,
args.augment == "true", True)
batch_hist_clbk = BatchLossHistory()
def init_vggnet(self):
from models.vgg16 import VGG16
self.base_model = VGG16(weights='imagenet', include_top=False)
self.input_shape = 224
self.feature_dim = 4096
self.output_shape = self.base_model.output_shape
'inital_lr': model_config.get('lr'),
'lr_power': 1,
}
else:
lr_schedule_config = None
model_config['lr_schedule_config'] = lr_schedule_config
if args.verbose:
print(model_config)
# Create model
if args.model == 'vgg16':
model = VGG16(model_config)
elif args.model == 'inception_resnetv2':
model = InceptionResnetV2(model_config)
elif args.model == 'inceptionv3':
model = InceptionV3(model_config)
elif args.model == 'custom':
model = Custom(model_config)
elif args.model == 'nasnetlarge':
model = NASNetLarge(model_config)
else:
print('Unknown/No model selected!')
sys.exit(1)
model.compile()
if args.verbose:
from models.vgg16 import VGG16
# Setting the main parameters
train_dir = "data/small_dataset/cropped/train"
validation_dir = "data/small_dataset/cropped/validation"
classes = len([
dir for dir in os.listdir(train_dir)
if os.path.isdir(os.path.join(train_dir, dir))
])
train_batchsize = 64
val_batchsize = 16
# Building the model
vgg_no_fc = VGG16(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
for layer in vgg_no_fc.layers:
layer.trainable = False
# for layer in vgg_no_fc.layers:
# print(layer, layer.trainable)
model = models.Sequential()
model.add(vgg_no_fc)
model.add(layers.Flatten(name='flatten'))
model.add(layers.Dense(4096, activation='relu', name='fc1'))
model.add(layers.Dense(4096, activation='relu', name='fc2'))
def get_pairs():
file_path = "../../data/data_a3/testPairs"
with open(file_path) as f:
content = f.readlines()
pairs = []
for line in content:
index = line.find(' ')
pairs.append([line[:index], line[index + 1:-1]])
return pairs
vgg16 = VGG16(include_top=False, weights=None, input_shape=(224, 224, 3))
middle = models.Sequential()
middle.add(vgg16)
middle.add(layers.Flatten())
middle.add(layers.Dense(4096, activation='relu', name='fc1'))
middle.add(layers.Dense(4096, activation='relu', name='fc2'))
middle.add(layers.Lambda(lambda x: backend.l2_normalize(x, axis=1)))
middle.load_weights("../../weights/a3_best_model.hdf5", by_name=True)
cropped_test_dir_path = "../../data/data_a3/cropped_test"
pairs = get_pairs()
for name1, name2 in tqdm(pairs):
rotation_range=25,
width_shift_range=.25,
height_shift_range=.25,
channel_shift_range=.35,
brightness_range=[.5, 1.5]
)
# Loads cnn model
if args.type == 's':
input_shape = (224, 224, 3)
model = VGG16(
'spatial',
input_shape=input_shape,
classes=configs['n_classes'],
dropout=configs['dropout'],
l2_reg=configs['l2_reg'],
weights=configs['weights'],
train_only_last=configs['train_only_last'],
include_top=True
)
train_set = SpatialDataGenerator(
src=configs['dataset'],
annotations=configs['annotations'],
classes_info=configs['classes'],
nb_frames=10,
batch_size=configs['batch_size'],
input_shape=(224, 224, 3),
split='train',
augmentation=data_aug
)
uvs = uvf.read().replace("'", '"')
univerb_data = json.loads(uvs)
print("Loading univerb model...", end="")
sys.stdout.flush()
univerb_model = GRU2(univerb_data["vocab_size"],
univerb_data["embedding_size"], univerb_data["max_feats"],
univerb_data["hidden_layer_size"],
univerb_data["dropout_prob"])
univerb_model.compile(optimizer=Adam(),
loss="binary_crossentropy",
class_mode="binary")
print("done")
print("Loading uniface model...", end="")
sys.stdout.flush()
uniface_model = VGG16(args.uniface_weights, False)
uniface_model.compile(optimizer=Adam(),
loss="binary_crossentropy",
class_mode="binary")
print("done")
print("Loading unikey model...", end="")
sys.stdout.flush()
unikey_model = VGG16(args.unikey_weights, False)
unikey_model.compile(optimizer=Adam(),
loss="binary_crossentropy",
class_mode="binary")
print("done")
accs = {}
for typ in args.evaluate_on:
split='train',
augmentation=train_datagen)
val_set = MotionFlowDataGenerator(
r'/home/coala/mestrado/datasets/UCF101/split01/',
r'/home/coala/mestrado/datasets/UCF101/split01/frame_info.csv',
r'/home/coala/mestrado/datasets/UCF101/classes_index.csv',
10,
16, (224, 224, 20),
split='val',
augmentation=train_datagen)
model = VGG16('temporal',
input_shape=(224, 224, 20),
classes=101,
dropout=0.9,
l2_reg=1e-5,
weights='imagenet',
include_top=True,
train_only_last=False)
model.summary()
learning_rate = 1e-3
print('Learning rate: %.10f' % learning_rate)
model.compile(keras.optimizers.Adam(learning_rate=learning_rate),
keras.losses.CategoricalCrossentropy(),
metrics=['acc'])
history = model.fit_generator(train_set,
validation_data=val_set,
verbose=1,