def train(model_label, nb_steps_toplayer, nb_steps_finetune, training_data_dir,
validation_data_dir, img_width, img_height, batch, nb_classes,
shuffle_flag, learning_rate, nb_training_samples,
nb_validation_samples, directory_location):
os.makedirs(directory_location, exist_ok=True)
if nb_classes == 2:
classmode, losstype = "binary", "binary_crossentropy"
elif nb_classes > 2:
classmode, losstype = "categorical", "categorical_crossentropy"
train, validate = build_data(training_data_dir, validation_data_dir,
img_width, img_height, batch, nb_classes,
shuffle_flag, classmode)
early_stop = EarlyStopping(monitor='val_loss', patience=3)
tbCallBack = keras.callbacks.TensorBoard(log_dir='Graph_finetune',
histogram_freq=0,
write_graph=True,
write_images=True)
if model_label == "vgg16":
vgg16(train, validate, losstype, nb_steps_toplayer, nb_steps_finetune,
learning_rate, tbCallBack, early_stop, nb_classes,
nb_training_samples, nb_validation_samples, directory_location)
elif model_label == "vgg19":
vgg19(train, validate, losstype, nb_steps_toplayer, nb_steps_finetune,
learning_rate, tbCallBack, early_stop, nb_classes,
nb_training_samples, nb_validation_samples, directory_location)
def model_config(net_type, num_classes, OOD_num_classes):
if net_type == "resnet50":
model = models.resnet50(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
elif net_type == "resnet34":
model = models.resnet34(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
elif net_type == "vgg19":
model = models.vgg19(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
elif net_type == "vgg16":
model = models.vgg16(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
elif net_type == "vgg19_bn":
model = models.vgg19_bn(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
elif net_type == "vgg16_bn":
model = models.vgg16_bn(num_c=num_classes,
num_cc=OOD_num_classes,
pretrained=True)
return model
def main(args):
print('\nPreparing {} data'.format(args.dataset))
if args.dataset == 'mnist':
(X_train, y_train), (X_test, y_test), (X_valid, y_valid) = load_mnist()
elif args.dataset == 'cifar10':
(X_train, y_train), (X_test, y_test), (X_valid, y_valid) = load_cifar10()
elif args.dataset == 'cifar100':
(X_train, y_train), (X_test, y_test), (X_valid, y_valid) = load_cifar100()
print('\nConstruction graph')
if args.model == 'cnn_1':
env = cnn_1(args)
elif args.model == 'cnn_2':
env = cnn_2(args)
elif args.model == 'vgg16':
env = vgg16(args)
elif args.model == 'vgg19':
env = vgg19(args)
print('\nInitializing graph')
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
print('\nTraining')
name = '{0}_{1}'.format(args.model, args.dataset)
train(sess, env, X_train, y_train, X_valid, y_valid, batch_size=args.batch_size,
epochs=args.epochs, name=name)
print('\nEvaluating on clean data')
evaluate(sess, env, X_test, y_test)
def __init__(self, config):
weights_path_2_cls = config['weights_path_2_cls']
weights_path_4_cls = config['weights_path_4_cls']
self.img_height = config['img_height']
self.img_width = config['img_width']
self.graph = tf.get_default_graph()
self.model_2_cls = vgg16(input_shape=(self.img_height, self.img_width,
3),
class_num=2,
weights_path=weights_path_2_cls)
self.model_4_cls = vgg16(input_shape=(self.img_height, self.img_width,
3),
class_num=4,
weights_path=weights_path_4_cls)
def initModel(args):
# Setup Model
if args.arch == "resnet50":
model = models.resnet50(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "resnet101":
model = models.resnet101(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "resnet152":
model = models.resnet152(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "vgg16":
model = models.vgg16(pretrained=True, num_classes=18)
elif args.arch == "googlenet":
model = models.googlenet(pretrained=True, num_classes=18)
for param in model.parameters():
param.requires_grad = False
#model = model.cuda()
if args.resume is not None:
if os.path.isfile(args.resume):
print(("Loading model and optimizer from checkpoint '{}'".format(
args.resume)))
checkpoint = torch.load(args.resume,
map_location=torch.device('cpu'))
# model.load_state_dict(checkpoint['state_dict'])
d = collections.OrderedDict()
for key, value in list(checkpoint['state_dict'].items()):
tmp = key[7:]
d[tmp] = value
model.load_state_dict(d)
print(("Loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch'])))
sys.stdout.flush()
else:
print(("No checkpoint found at '{}'".format(args.resume)))
sys.stdout.flush()
model.eval()
summary(model, (3, 640, 640))
return model
def get_network(args, depth=10, width=10):
""" return given network
"""
if args.task == 'cifar10':
nclass = 10
elif args.task == 'cifar100':
nclass = 100
#Yang added none bn vggs
if args.net == 'vgg11':
if args.batch_norm:
net = vgg11_bn(num_classes=nclass)
else:
net = vgg11(num_classes=nclass)
elif args.net == 'vgg13':
if args.batch_norm:
net = vgg13_bn(num_classes=nclass)
else:
net = vgg13(num_classes=nclass)
elif args.net == 'vgg16':
if args.batch_norm:
net = vgg16_bn(num_classes=nclass)
else:
net = vgg16(num_classes=nclass)
elif args.net == 'vgg19':
if args.batch_norm:
net = vgg19_bn(num_classes=nclass)
else:
net = vgg19(num_classes=nclass)
elif args.net == 'resnet':
net = resnet(num_classes=nclass, depth=depth, width=width)
# elif args.net == 'resnet34':
# net = resnet34(num_classes=nclass)
# elif args.net == 'resnet50':
# net = resnet50(num_classes=nclass)
# elif args.net == 'resnet101':
# net = resnet101(num_classes=nclass)
# elif args.net == 'resnet152':
# net = resnet152(num_classes=nclass)
else:
print('the network name you have entered is not supported yet')
sys.exit()
if args.gpu: #use_gpu
net = net.cuda()
return net
def main(args):
print('\nPreparing {} data'.format(args.dataset))
if args.dataset == 'mnist':
(X_train, y_train), (X_test, y_test), (X_valid, y_valid) = load_mnist()
elif args.dataset == 'cifar10':
(X_train, y_train), (X_test, y_test), (X_valid,
y_valid) = load_cifar10()
elif args.dataset == 'cifar100':
(X_train, y_train), (X_test, y_test), (X_valid,
y_valid) = load_cifar100()
print('\nConstruction graph')
if args.model == 'cnn_1':
env = cnn_1(args)
elif args.model == 'cnn_2':
env = cnn_2(args)
elif args.model == 'vgg16':
env = vgg16(args)
elif args.model == 'vgg19':
env = vgg19(args)
print('\nInitializing graph')
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
print('\nLoading saved model')
name = '{0}_{1}'.format(args.model, args.dataset)
env.saver.restore(sess, 'models/{0}/{1}'.format(name, name))
print('\nEvaluating on clean data')
evaluate(sess, env, X_test, y_test)
print('\nExcluding misclassification samples')
# mnist 1000 samples -> 0:1010
# cifar10 1000 samples -> 0:
(X_test, y_test) = exclude_miss(sess, env, X_test, y_test, 0, 12)
evaluate(sess, env, X_test, y_test)
print('\nGenerating adversarial data')
X_adv = make_adv(args, sess, env, X_test, y_test)
print('\nEvaluating on adversarial data')
evaluate(sess, env, X_adv, y_test)
print('\nResults')
def init_pixel_link(model_path):
model = models.vgg16(pretrained=True, num_classes=18)
model = model.cuda(device)
if os.path.isfile(model_path):
print(("Loading model and optimizer from checkpoint '{}'".format(
model_path)))
checkpoint = torch.load(model_path)
# model.load_state_dict(checkpoint['state_dict'])
d = collections.OrderedDict()
for key, value in list(checkpoint['state_dict'].items()):
tmp = key[7:]
d[tmp] = value
model.load_state_dict(d)
print(("Loaded checkpoint '{}' (epoch {})".format(
model_path, checkpoint['epoch'])))
else:
print(("No checkpoint found at '{}'".format(args.resume)))
return model
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
num_workers=0,
pin_memory=True,
shuffle=False)
features = compute_features(dataloader, model, len(dataset))
return features
if __name__ == '__main__':
# global args
# args = parser.parse_args()
modelpth = '/home/CUSACKLAB/annatruzzi/deepcluster_models/vgg16/'
checkpoint = torch.load(modelpth +
'checkpoint_vgg16.pth.tar')['state_dict']
checkpoint_new = OrderedDict()
for k, v in checkpoint.items():
name = k.replace(".module", '') # remove 'module.' of dataparallel
checkpoint_new[name] = v
model = models.vgg16(sobel=True, bn=True, out=10000)
model.load_state_dict(checkpoint_new)
model.cuda()
image_pth = '/home/CUSACKLAB/annatruzzi/cichy2016/algonautsChallenge2019/Training_Data/92_Image_Set/92images'
act = get_activations(image_pth)
with open(
'/home/CUSACKLAB/annatruzzi/cichy2016/niko92_activations_pretrained_vgg16.pickle',
'wb') as handle:
pickle.dump(act, handle)
def main(args):
# ee = tf.data.Dataset()
num_classes = 8
size = (224, 224, 3) # size of images
learning_rate = args.learning_rate
# Rruntime initialization will not allocate all memory on GPU
physical_devices = tf.config.list_physical_devices('GPU')
try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
optimizer = optimizers.SGD(learning_rate=learning_rate, momentum=0.9)
loss = keras.losses.SparseCategoricalCrossentropy(from_logits=False)
metrics = [keras.metrics.SparseCategoricalAccuracy()]
model = models.vgg16(input_shape=size,
num_classes=num_classes,
classifier_activation='softmax')
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
model.summary()
# if os.path.exists('isic_class.h5'):
# # load 3rd epoch weights
# model.load_weights('isic_class.h5')
train_aug = iaa.Sequential([
iaa.Resize(size=size[:-1], interpolation='cubic'),
iaa.Fliplr(p=0.5),
iaa.Flipud(p=0.5),
iaa.Rotate(rotate=(-180, 180)),
iaa.AdditivePoissonNoise(lam=(0, 10)),
iaa.GammaContrast(gamma=(.8, 1.5)),
iaa.GaussianBlur(sigma=(.0, .8)),
iaa.CoarseDropout(p=(.02, .1), size_px=(0.02, 0.05), size_percent=0.5),
])
val_aug = iaa.Sequential(
[iaa.Resize(size=size[:-1], interpolation='cubic')])
training_dataset = ISICClassification(args.dataset, 'training',
args.batch_size, train_aug)
validation_dataset = ISICClassification(args.dataset,
'validation',
args.batch_size,
val_aug,
shuffle=False)
print('Loading validation images...', end='')
val_samples = validation_dataset.load_samples()
print('done')
# Save checkpoints
checkpoint = ModelCheckpoint("isic_class_vgg.h5",
monitor='val_sparse_categorical_accuracy',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
save_freq='epoch')
# stop training after 20 epochs of no improvement
early = EarlyStopping(monitor='val_sparse_categorical_accuracy',
min_delta=0,
patience=args.epochs // 4,
verbose=1,
mode='auto')
# Train the model
history = model.fit(
x=training_dataset,
epochs=args.epochs,
verbose=1,
callbacks=[checkpoint, early],
validation_data=val_samples,
steps_per_epoch=len(training_dataset),
validation_steps=len(validation_dataset),
use_multiprocessing=True,
workers=8,
)
return
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.alexnet(num_classes=num_classes, pretrained='imagenet')
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.densenet121(num_classes=num_classes, pretrained='imagenet')
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.densenet169(num_classes=num_classes, pretrained='imagenet')
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.densenet201(num_classes=num_classes, pretrained='imagenet')
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.densenet201(num_classes=num_classes, pretrained='imagenet')
assert test(inputs).size()[1] == num_classes
print('ok')
test = models.inceptionv3(num_classes=num_classes, pretrained='imagenet')
assert test(torch.rand([2, 3, 299, 299]))[0].size()[1] == num_classes
print('ok')
assert test(torch.rand([2, 3, 299, 299]))[1].size()[1] == num_classes
print('ok')
test = models.vgg16(num_classes=num_classes, pretrained='imagenet')
assert test(torch.rand([1, 3, 224, 224])).size()[1] == num_classes
print('ok')
test = models.vgg16_bn(num_classes=num_classes, pretrained=None)
assert test(torch.rand([1, 3, 224, 224])).size()[1] == num_classes
print('ok')
args = parser.parse_args()
with open(args.train_config_path, 'r', encoding='utf-8') as f:
train_config = json.load(f)
model_type = train_config['model_type']
if os.path.exists(train_config['weights_path']):
weights_path = train_config['weights_path']
else:
weights_path = None
if model_type == 'vgg16':
model = vgg16(input_shape=(train_config['img_height'],
train_config['img_width'], 3),
class_num=train_config['class_num'],
weights_path=weights_path)
# 冻结不训练的层
for layer in model.layers:
if layer.name.find('output') == 0:
# 不冻结输出层
continue
# train_config['train_layers'] 可选 ["block1", "block2", "block3", "block4", "block5"]
is_match = False
for block_name in train_config['train_layers']:
if layer.name.find(block_name) == 0:
# 不冻结train_layers中设置的block
is_match = True
break
if is_match:
continue
def main():
output_dir = "./save_fig"
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Hyper-parameters
eps = 1e-8
### data config
test_dataset = load_data.Dog_dataloader(image_dir=image_dir,
num_class=args.num_classes,
mode="test")
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=1,
shuffle=False,
num_workers=2)
### novelty data
out_test_dataset = load_data.Dog_dataloader(image_dir=image_dir,
num_class=args.num_classes,
mode="OOD")
out_test_loader = torch.utils.data.DataLoader(out_test_dataset,
batch_size=1,
shuffle=False,
num_workers=2)
##### model, optimizer config
if args.net_type == "resnet50":
model = models.resnet50(num_c=args.num_classes, pretrained=True)
elif args.net_type == "resnet34":
model = models.resnet34(num_c=args.num_classes,
num_cc=args.OOD_num_classes,
pretrained=True)
elif args.net_type == "vgg19":
model = models.vgg19(num_c=args.num_classes,
num_cc=args.OOD_num_classes,
pretrained=True)
elif args.net_type == "vgg16":
model = models.vgg16(num_c=args.num_classes,
num_cc=args.OOD_num_classes,
pretrained=True)
elif args.net_type == "vgg19_bn":
model = models.vgg19_bn(num_c=args.num_classes,
num_cc=args.OOD_num_classes,
pretrained=True)
elif args.net_type == "vgg16_bn":
model = models.vgg16_bn(num_c=args.num_classes,
num_cc=args.OOD_num_classes,
pretrained=True)
print("load checkpoint_last")
checkpoint = torch.load(args.model_path)
##### load model
model.load_state_dict(checkpoint["model"])
start_epoch = checkpoint["epoch"]
optimizer = optim.SGD(model.parameters(), lr=checkpoint["init_lr"])
#### create folder
Path(output_dir).mkdir(exist_ok=True, parents=True)
model = model.to(device).eval()
# Start grad-CAM
bp = BackPropagation(model=model)
inv_normalize = transforms.Normalize(
mean=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.255],
std=[1 / 0.229, 1 / 0.224, 1 / 0.255])
target_layer = "layer4"
stime = time.time()
gcam = GradCAM(model=model)
grad_cam = GradCAMmodule(target_layer, output_dir)
grad_cam.model_config(model)
for j, test_data in enumerate(test_loader):
#### initialized
org_image = test_data['input'].to(device)
target_class = test_data['label'].to(device)
target_class = int(target_class.argmax().cpu().detach())
result = model(org_image).argmax()
print("number: {} pred: {} target: {}".format(j, result, target_class))
result = int(result.cpu().detach())
grad_cam.saveGradCAM(org_image, result, j)
opt = TrainOptions().parse()
if not os.path.exists(opt.checkpoints_dir):
os.mkdir(opt.save_dir)
dataset = create_dataset(
opt) # create a dataset given opt.dataset_mode and other options
dataset_size = len(dataset) # get the number of images in the dataset.
print('The number of training images = %d' % dataset_size)
rec_ = VggRecModel(opt)
print(rec_.netrec.parameters)
rec_.setup(opt)
#print(rec_.netrec.parameters)
print('set up reconstuction model')
vgg_ = vgg16().cuda()
for param in vgg_.parameters():
param.requires_grad = False
vgg_.eval()
vgg_.classifier = None
vgg_.top_layer = None
if os.path.isfile(opt.encoder):
print("=> loading encoder '{}'".format(opt.encoder))
checkpoint = torch.load(opt.encoder)
# remove top_layer and classifier parameters from checkpoint
for key in list(checkpoint['state_dict']):
if 'top_layer' in key:
del checkpoint['state_dict'][key]
if 'classifier' in key:
del checkpoint['state_dict'][key]
from collections import OrderedDict
net = tflearn.residual_block(net, 1, 512, downsample=True)
net = tflearn.residual_block(net, 2, 512)
net = tflearn.global_avg_pool(net)
fully_connected = tflearn.fully_connected(net, HIDDEN, activation="relu")
result = tflearn.fully_connected(fully_connected, LEN_OUT, activation='softmax')
mom = tflearn.Momentum(0.01, lr_decay=0.1, decay_step=int(395000 / BATCH_SIZE) * DECAY_STEP, staircase=True)
net = tflearn.regression(result, optimizer=mom, loss="categorical_crossentropy")
model = tflearn.DNN(net,checkpoint_path='models/{}'.format(MODEL_FILE),session=sess,max_checkpoints=100,
tensorboard_verbose=0)
elif NET_TYPE == 'vgg16':
with tf.device("/gpu:{}".format(GPU_CORE)):
import tflearn
from models import vgg16
input_data = tflearn.input_data(shape=[None, IMG_SIZE, IMG_SIZE, 3])
hidden = vgg16(input_data)
softmax = tflearn.fully_connected(hidden, LEN_OUT, activation='softmax', scope='fc8',restore=False)
#momem = tflearn.optimizers.SGD(learning_rate=0.01)#ecay_step=200,lr_decay=0.1)
momem = tflearn.Momentum(0.01)
net = tflearn.regression(softmax, optimizer=momem,
loss='categorical_crossentropy')
model = tflearn.DNN(net, checkpoint_path='models/{}'.format(MODEL_FILE),session=sess,max_checkpoints=100,tensorboard_verbose=0)
elif NET_TYPE == 'inception-resnet-v2':
with tf.device("/gpu:{}".format(GPU_CORE)):
from inception_resnet import *
input_dat = tflearn.input_data(shape=[None,IMG_SIZE,IMG_SIZE,3])
fully_connected,end_points = inception_resnet_v1(input_dat)
result = tflearn.fully_connected(fully_connected, 7211, activation='softmax')
# [7211]
mom = tflearn.Momentum(0.01,lr_decay=0.1,decay_step=int(395000 / BATCH_SIZE) * 10,staircase=True)
reg = tflearn.regression(result,optimizer=mom,loss='categorical_crossentropy')
def store_styleft():
vgg_ = vgg16().cuda()
for param in vgg_.parameters():
param.requires_grad = False
vgg_.top_layer = None
encoder_path = '/home/visiting/Projects/levishery/checkpoint.pth.tar'
data_path = '/home/visiting/datasets/crop_vangogh_original'
if os.path.isfile(encoder_path):
print("=> loading encoder '{}'".format(encoder_path))
checkpoint = torch.load(encoder_path)
# remove top_layer and classifier parameters from checkpoint
for key in list(checkpoint['state_dict']):
if 'top_layer' in key:
del checkpoint['state_dict'][key]
from collections import OrderedDict
new_state_dict = OrderedDict()
for k, v in checkpoint['state_dict'].items():
if 'feature' in k:
name = k[:8] + k[15:] # remove `module.`
new_state_dict[name] = v
else:
new_state_dict[k] = v
vgg_.load_state_dict(new_state_dict)
print("=> loaded checkpoint '{}' (epoch {})".format(
encoder_path, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(encoder_path))
# preprocessing of data
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
tra = [transforms.Resize(240), transforms.ToTensor(), normalize]
# load the data
end = time.time()
dataset = datasets.ImageFolder(data_path,
transform=transforms.Compose(tra))
print('Load dataset: {0:.2f} s'.format(time.time() - end))
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=16,
num_workers=4)
for i, (input_tensor, _) in enumerate(dataloader):
input_var = torch.autograd.Variable(input_tensor.cuda(), volatile=True)
features = vgg_.features(input_var)
PATCH_NUM = 10
features = features.unfold(2, 3, 3).unfold(3, 3, 3)
features = features.permute(0, 2, 3, 1, 4, 5)
x = features.reshape(features.size(0) * PATCH_NUM * PATCH_NUM, -1)
x = vgg_.classifier(x).cpu().numpy()
features = features.cpu().numpy()
if i == 0:
store_features = np.zeros(
(len(dataset.imgs), features.shape[1], features.shape[2],
features.shape[3], features.shape[4],
features.shape[5])).astype('float32')
store_linear = np.zeros((len(dataset.imgs) * PATCH_NUM * PATCH_NUM,
x.shape[1])).astype('float32')
if i < len(dataloader) - 1:
store_features[i * 16:(i + 1) * 16] = features.astype('float32')
store_linear[i * 16 * PATCH_NUM * PATCH_NUM:(i + 1) * 16 *
PATCH_NUM * PATCH_NUM] = x.astype('float32')
else:
# special treatment for final batch
store_features[i * 16:] = features.astype('float32')
store_linear[i * 16 * PATCH_NUM * PATCH_NUM:] = x.astype('float32')
small_ft, pca = index_features(store_linear)
faiss.write_VectorTransform(pca, "vangogh.pca")
small_ft = small_ft.tolist()
store_features = store_features.tolist()
file_name = '/home/visiting/Projects/levishery/reconstruction/vangogh_index.json'
print('start writing index')
with open(file_name, 'w') as file_object:
json.dump(small_ft, file_object)
file_name = '/home/visiting/Projects/levishery/reconstruction/vangogh_features.json'
print('start writing features')
with open(file_name, 'w') as file_object:
json.dump(store_features, file_object)
def main(args):
if args.checkpoint == '':
args.checkpoint = "checkpoints/ic15_%s_bs_%d_ep_%d"%(args.arch, args.batch_size, args.n_epoch)
if args.pretrain:
if 'synth' in args.pretrain:
args.checkpoint += "_pretrain_synth"
else:
args.checkpoint += "_pretrain_s1280"
print(('checkpoint path: %s'%args.checkpoint))
print(('init lr: %.8f'%args.lr))
print(('schedule: ', args.schedule))
sys.stdout.flush()
if not os.path.isdir(args.checkpoint):
os.makedirs(args.checkpoint)
writer=SummaryWriter(args.checkpoint)
kernel_num=18
start_epoch = 0
#####
#
#
#
#####
data_loader = IC15Loader(is_transform=True, img_size=args.img_size)
train_loader = torch.utils.data.DataLoader(
data_loader,
batch_size=args.batch_size,
shuffle=True,
num_workers=3,
drop_last=False,
pin_memory=True)
if args.arch == "resnet50":
model = models.resnet50(pretrained=True, num_classes=kernel_num)
elif args.arch == "resnet101":
model = models.resnet101(pretrained=True, num_classes=kernel_num)
elif args.arch == "resnet152":
model = models.resnet152(pretrained=True, num_classes=kernel_num)
elif args.arch == "vgg16":
model = models.vgg16(pretrained=False,num_classes=kernel_num)
model = torch.nn.DataParallel(model).cuda()
model.train()
if hasattr(model.module, 'optimizer'):
optimizer = model.module.optimizer
else:
# NOTE 这个地方的momentum对训练影响相当之大，使用0.99时训练crossentropy无法收敛.
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
title = 'icdar2015'
if args.pretrain:
print('Using pretrained model.')
assert os.path.isfile(args.pretrain), 'Error: no checkpoint directory found!'
checkpoint = torch.load(args.pretrain)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss','Train Acc.', 'Train IOU.'])
elif args.resume:
print('Resuming from checkpoint.')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
print('Training from scratch.')
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss','Train Acc.', 'Train IOU.'])
images_loss = {}
# data_plot = images_loss.values()
# import matplotlib.pyplot as plt
# plt.plot(data_plot)
# plt.ylabel('Loss plot')
# plt.show()
for epoch in range(start_epoch, args.n_epoch):
adjust_learning_rate(args, optimizer, epoch)
print(('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.n_epoch, optimizer.param_groups[0]['lr'])))
train_loss, train_te_acc, train_te_iou = train(train_loader,images_loss, model, dice_loss, optimizer, epoch,writer)
if epoch %5 == 0 and epoch != 0:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'lr': args.lr,
'optimizer' : optimizer.state_dict(),
}, checkpoint=args.checkpoint,filename='checkpoint_%d.pth'%epoch)
logger.append([optimizer.param_groups[0]['lr'], train_loss, train_te_acc, train_te_iou])
logger.close()
writer.flush()
writer.close()
"""
n = 1000
X_train = X_train[:n]
Y_train = Y_train[:n]
X_test = X_test[:n]
Y_test = Y_test[:n]
"""
"""
m = X_train.min()
s = X_train.max()
X_train -= m
X_train = X_train/s
X_test -= m
X_test = X_test/s
"""
print("Training . . . ")
model = models.vgg16(lr=1e-3, mbs=50, pred_mbs=500, seed=456)
model.start_session()
accs = model.train(X_train,
Y_train,
eval_set=(X_test, Y_test),
epochs=100,
early_stopping=5)
model.stop_session()
print("Saving results . . . ")
np.save("results/vgg16_denoised_accs.npy", accs)
print("Done!")
def test(args):
data_loader = IC15TestLoader(long_size=args.long_size)
test_loader = torch.utils.data.DataLoader(data_loader,
batch_size=1,
shuffle=False,
num_workers=2,
drop_last=True)
# Setup Model
if args.arch == "resnet50":
model = models.resnet50(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "resnet101":
model = models.resnet101(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "resnet152":
model = models.resnet152(pretrained=True,
num_classes=18,
scale=args.scale)
elif args.arch == "vgg16":
model = models.vgg16(pretrained=True, num_classes=18)
for param in model.parameters():
param.requires_grad = False
model = model.cuda()
if args.resume is not None:
if os.path.isfile(args.resume):
print(("Loading model and optimizer from checkpoint '{}'".format(
args.resume)))
checkpoint = torch.load(args.resume)
# model.load_state_dict(checkpoint['state_dict'])
d = collections.OrderedDict()
for key, value in list(checkpoint['state_dict'].items()):
tmp = key[7:]
d[tmp] = value
model.load_state_dict(d)
print(("Loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch'])))
sys.stdout.flush()
else:
print(("No checkpoint found at '{}'".format(args.resume)))
sys.stdout.flush()
model.eval()
total_frame = 0.0
total_time = 0.0
for idx, (org_img, img) in enumerate(test_loader):
print(('progress: %d / %d' % (idx, len(test_loader))))
sys.stdout.flush()
img = img.cuda()
org_img = org_img.numpy().astype('uint8')[0]
text_box = org_img.copy()
torch.cuda.synchronize()
start = time.time()
cls_logits, link_logits = model(img)
outputs = torch.cat((cls_logits, link_logits), dim=1)
shape = outputs.shape
pixel_pos_scores = F.softmax(outputs[:, 0:2, :, :], dim=1)[:, 1, :, :]
# pixel_pos_scores=torch.sigmoid(outputs[:,1,:,:])
# FIXME the dimention should be changed
link_scores = outputs[:, 2:, :, :].view(shape[0], 2, 8, shape[2],
shape[3])
link_pos_scores = F.softmax(link_scores, dim=1)[:, 1, :, :, :]
mask, bboxes = to_bboxes(org_img,
pixel_pos_scores.cpu().numpy(),
link_pos_scores.cpu().numpy())
score = pixel_pos_scores[0, :, :]
score = score.data.cpu().numpy().astype(np.float32)
torch.cuda.synchronize()
end = time.time()
total_frame += 1
total_time += (end - start)
print(('fps: %.2f' % (total_frame / total_time)))
sys.stdout.flush()
for bbox in bboxes:
cv2.drawContours(text_box, [bbox.reshape(4, 2)], -1, (0, 255, 0),
2)
image_name = data_loader.img_paths[idx].split('/')[-1].split('.')[0]
write_result_as_txt(image_name, bboxes, 'outputs/submit_ic15/')
text_box = cv2.resize(text_box, (org_img.shape[1], org_img.shape[0]))
score_s = cv2.resize(
np.repeat(score[:, :, np.newaxis] * 255, 3, 2).astype(np.uint8),
(org_img.shape[1], org_img.shape[0]))
mask = cv2.resize(
np.repeat(mask[:, :, np.newaxis], 3, 2).astype(np.uint8),
(org_img.shape[1], org_img.shape[0]))
link_score = (link_pos_scores[0, 0, :, :]).cpu().numpy() * (
score > 0.5).astype(np.float)
link_score = cv2.resize(
np.repeat(link_score[:, :, np.newaxis] * 255, 3,
2).astype(np.uint8),
(org_img.shape[1], org_img.shape[0]))
debug(idx, data_loader.img_paths,
[[text_box, score_s], [link_score, mask]], 'outputs/vis_ic15/')
cmd = 'cd %s;zip -j %s %s/*' % ('./outputs/', 'submit_ic15.zip',
'submit_ic15')
print(cmd)
sys.stdout.flush()
util.cmd.cmd(cmd)
cmd_eval = 'cd eval;sh eval_ic15.sh'
sys.stdout.flush()
util.cmd.cmd(cmd_eval)
height_shift_range=0.1,
fill_mode='nearest',
zca_whitening=True # ZCA白色化
)
# 学習
# model = vgg16(imgsize)
model = None
if model_name == "v2_model":
model = v2_model()
elif model_name == "v3_model":
model = v3_model()
elif model_name == "v4_model":
model = v4_model()
elif model_name == "vgg":
model = vgg16(imgsize)
elif model_name == "rmforest":
model = RFR(n_jobs=-1, random_state=seed_value)
# checkpointの設定
model_path = 'models/' + model_name_prefix + '_' + "fold" + str(
i) + "_best_model.hdf5"
checkpoint = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
save_best_only=True,
period=1)
# 動的学習率変化
history = None
if lrateflag:
print("Leaning Rate Scheduling")
def training(args):
# DIRECTORY FOR CKPTS and META FILES
# ROOT_DIR = '/neuhaus/movie/dataset/tf_records'
ROOT_DIR = '/media/data/movie/dataset/tf_records'
TRAIN_REC_PATH = os.path.join(
ROOT_DIR,
args.experiment_name,
'train.tfrecords')
VAL_REC_PATH = os.path.join(
ROOT_DIR,
args.experiment_name,
'val.tfrecords')
CKPT_PATH = os.path.join(
ROOT_DIR,
args.experiment_name,
args.ckpt_folder_name,
'/')
# SCOPING BEGINS HERE
with tf.Session().as_default() as sess:
global_step = tf.train.get_global_step()
train_queue = tf.train.string_input_producer(
[TRAIN_REC_PATH], num_epochs=None)
train_fFrames, train_lFrames, train_iFrames, train_mfn =\
read_and_decode(
filename_queue=train_queue,
is_training=True,
batch_size=args.batch_size)
val_queue = tf.train.string_input_producer(
[VAL_REC_PATH], num_epochs=None)
val_fFrames, val_lFrames, val_iFrames, val_mfn = \
read_and_decode(
filename_queue=val_queue,
is_training=False,
batch_size=args.batch_size)
with tf.variable_scope('bipn'):
print('TRAIN FRAMES (first):')
train_rec_iFrames = bipn.build_bipn(
train_fFrames,
train_lFrames,
use_batch_norm=True,
is_training=True)
with tf.variable_scope('bipn', reuse=tf.AUTO_REUSE):
print('VAL FRAMES (first):')
val_rec_iFrames = bipn.build_bipn(
val_fFrames,
val_lFrames,
use_batch_norm=True,
is_training=False)
print('Model parameters:{}'.format(
count_parameters()))
# Weights should be kept locally ~ 500 MB space
with tf.variable_scope('vgg16'):
train_iFrames_features = vgg16.build_vgg16(
train_iFrames, end_point='conv4_3').features
with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE):
train_rec_iFrames_features = vgg16.build_vgg16(
train_rec_iFrames, end_point='conv4_3').features
if args.perceptual_loss_weight:
# Weights should be kept locally ~ 500 MB space
with tf.variable_scope('vgg16'):
train_iFrames_features = vgg16(
train_iFrames,
end_point='conv4_3')
with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE):
train_rec_iFrames_features = vgg16(
train_rec_iFrames,
end_point='conv4_3')
# DEFINE METRICS
if args.loss_id == 0:
train_loss = huber_loss(
train_iFrames, train_rec_iFrames,
delta=1.)
val_loss = huber_loss(
val_iFrames, val_rec_iFrames,
delta=1.)
elif args.loss_id == 1:
train_loss = l2_loss(
train_iFrames, train_rec_iFrames)
val_loss = l2_loss(
val_iFrames, val_rec_iFrames)
total_train_loss = train_loss
tf.summary.scalar('train_l2_loss', train_loss)
tf.summary.scalar('total_val_l2_loss', val_loss)
if args.perceptual_loss_weight:
train_perceptual_loss = perceptual_loss(
train_iFrames_features,
train_rec_iFrames_features)
tf.summary.scalar('train_perceptual_loss',\
train_perceptual_loss)
total_train_loss += train_perceptual_loss\
* args.perceptual_loss_weight
# SUMMARIES
tf.summary.scalar('total_train_loss',\
total_train_loss)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
CKPT_PATH + 'train',
sess.graph)
# DEFINE OPTIMIZER
optimizer = get_optimizer(
train_loss,
optim_id=args.optim_id,
learning_rate=args.learning_rate,
use_batch_norm=True)
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver()
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(
coord=coord)
# START TRAINING HERE
try:
for iteration in range(args.train_iters):
_, t_summ, t_loss = sess.run(
[optimizer, merged, total_train_loss])
train_writer.add_summary(t_summ, iteration)
print('Iter:{}/{}, Train Loss:{}'.format(
iteration,
args.train_iters,
t_loss))
if iteration % args.val_every == 0:
v_loss = sess.run(val_loss)
print('Iter:{}, Val Loss:{}'.format(
iteration,
v_loss))
if iteration % args.save_every == 0:
saver.save(
sess,
CKPT_PATH + 'iter:{}_val:{}'.format(
str(iteration),
str(round(v_loss, 3))))
if iteration % args.plot_every == 0:
start_frames, end_frames, mid_frames,\
rec_mid_frames = sess.run(
[train_fFrames, train_lFrames,\
train_iFrames,\
train_rec_iFrames])
visualize_frames(
start_frames,
end_frames,
mid_frames,
rec_mid_frames,
iteration=iteration,
save_path=os.path.join(
CKPT_PATH,
'plots/'))
except Exception as e:
coord.request_stop(e)
def main():
start_epoch = 0
pretrained_model = os.path.join("./pre_trained", args.dataset,
args.net_type + ".pth.tar")
save_model = "./save_model_dis/pre_training"
tensorboard_dir = "./tensorboard/OOD_dis/pre_training" + args.dataset
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Hyper-parameters
eps = 1e-8
### data config
train_dataset = load_data.Dog_metric_dataloader(image_dir=image_dir,
num_class=args.num_classes,
mode="train",
soft_label=args.soft_label)
if args.custom_sampler:
MySampler = load_data.customSampler(train_dataset, args.batch_size,
args.num_instances)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=MySampler,
num_workers=2)
else:
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
test_dataset = load_data.Dog_dataloader(image_dir=image_dir,
num_class=args.num_classes,
mode="test")
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=8,
shuffle=False,
num_workers=2)
out_test_dataset = load_data.Dog_dataloader(image_dir=image_dir,
num_class=args.num_classes,
mode="OOD")
out_test_loader = torch.utils.data.DataLoader(out_test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=2)
if args.transfer:
### perfectly OOD data
OOD_dataset = load_data.Dog_dataloader(image_dir=OOD_dir,
num_class=args.OOD_num_classes,
mode="OOD")
OOD_loader = torch.utils.data.DataLoader(OOD_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
##### model, optimizer config
if args.net_type == "resnet50":
model = models.resnet50(num_c=args.num_classes, pretrained=True)
elif args.net_type == "resnet34":
model = models.resnet34(num_c=args.num_classes, pretrained=True)
elif args.net_type == "vgg19":
model = models.vgg19(num_c=args.num_classes, pretrained=True)
elif args.net_type == "vgg16":
model = models.vgg16(num_c=args.num_classes, pretrained=True)
elif args.net_type == "vgg19_bn":
model = models.vgg19_bn(num_c=args.num_classes, pretrained=True)
elif args.net_type == "vgg16_bn":
model = models.vgg16_bn(num_c=args.num_classes, pretrained=True)
if args.transfer:
extra_fc = nn.Linear(2048, args.num_classes + args.OOD_num_classes)
if args.load == True:
print("loading model")
checkpoint = torch.load(pretrained_model)
##### load model
model.load_state_dict(checkpoint["model"])
batch_num = len(
train_loader) / args.batch_size if args.custom_sampler else len(
train_loader)
optimizer = optim.SGD(model.parameters(),
lr=args.init_lr,
momentum=0.9,
nesterov=args.nesterov)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.num_epochs * batch_num)
#### loss config
criterion = nn.BCEWithLogitsLoss()
#### create folder
Path(os.path.join(save_model, env, args.net_type)).mkdir(exist_ok=True,
parents=True)
if args.board_clear == True:
files = glob.glob(tensorboard_dir + "/*")
for f in files:
shutil.rmtree(f)
i = 0
while True:
if Path(os.path.join(tensorboard_dir, str(i))).exists() == True:
i += 1
else:
Path(os.path.join(tensorboard_dir, str(i))).mkdir(exist_ok=True,
parents=True)
break
summary = SummaryWriter(os.path.join(tensorboard_dir, str(i)))
# Start training
j = 0
best_score = 0
score = 0
membership_loss = torch.tensor(0)
transfer_loss = torch.tensor(0)
for epoch in range(start_epoch, args.num_epochs):
running_loss = 0
running_membership_loss = 0
running_transfer_loss = 0
running_class_loss = 0
train_acc = 0
test_acc = 0
stime = time.time()
# for i, (train_data, OOD_data) in enumerate(zip(train_loader, OOD_loader)):
for i, train_data in enumerate(train_loader):
#### initialized
org_image = train_data['input'] + 0.01 * torch.randn_like(
train_data['input'])
org_image = org_image.to(device)
gt = train_data['label'].type(torch.FloatTensor).to(device)
model = model.to(device).train()
optimizer.zero_grad()
#### forward path
out1, out2 = model.pendis_forward(org_image)
if args.membership:
membership_loss = (
Membership_loss(out2, gt, args.num_classes) +
Membership_loss(out1, gt, args.num_classes))
running_membership_loss += membership_loss.item()
if args.transfer:
extra_fc = extra_fc.to(device).train()
OOD_image = (
OOD_data['input'] +
0.01 * torch.randn_like(OOD_data['input'])).to(device)
OOD_gt = torch.cat(
(torch.zeros(args.batch_size, args.num_classes),
OOD_data['label'].type(torch.FloatTensor)),
dim=1).to(device)
#### forward path
_, feature = model.gen_forward(OOD_image)
OOD_output = extra_fc(feature)
transfer_loss = criterion(OOD_output, OOD_gt)
running_transfer_loss += transfer_loss.item()
#### calc loss
class1_loss = criterion(out1, gt)
class2_loss = criterion(out2, gt)
class_loss = (class1_loss + class2_loss)
total_loss = class_loss + membership_loss * 0.3 + transfer_loss
#### calc accuracy
train_acc += sum(
torch.argmax(out1, dim=1) == torch.argmax(
gt, dim=1)).cpu().detach().item()
train_acc += sum(
torch.argmax(out2, dim=1) == torch.argmax(
gt, dim=1)).cpu().detach().item()
total_loss.backward()
optimizer.step()
scheduler.step()
running_class_loss += class_loss.item()
running_loss += total_loss.item()
with torch.no_grad():
for i, test_data in enumerate(test_loader):
org_image = test_data['input'].to(device)
model = model.to(device).eval()
gt = test_data['label'].type(torch.FloatTensor).to(device)
#### forward path
out1, out2 = model.pendis_forward(org_image)
score_1 = nn.functional.softmax(out1, dim=1)
score_2 = nn.functional.softmax(out2, dim=1)
dist = torch.sum(torch.abs(score_1 - score_2), dim=1).reshape(
(org_image.shape[0], -1))
if i == 0:
dists = dist
labels = torch.zeros((org_image.shape[0], ))
else:
dists = torch.cat((dists, dist), dim=0)
labels = torch.cat(
(labels, torch.zeros((org_image.shape[0]))), dim=0)
test_acc += sum(
torch.argmax(torch.sigmoid(out1), dim=1) == torch.argmax(
gt, dim=1)).cpu().detach().item()
test_acc += sum(
torch.argmax(torch.sigmoid(out2), dim=1) == torch.argmax(
gt, dim=1)).cpu().detach().item()
for i, out_org_data in enumerate(out_test_loader):
out_org_image = out_org_data['input'].to(device)
out1, out2 = model.pendis_forward(out_org_image)
score_1 = nn.functional.softmax(out1, dim=1)
score_2 = nn.functional.softmax(out2, dim=1)
dist = torch.sum(torch.abs(score_1 - score_2), dim=1).reshape(
(out_org_image.shape[0], -1))
dists = torch.cat((dists, dist), dim=0)
labels = torch.cat((labels, torch.ones(
(out_org_image.shape[0]))),
dim=0)
roc = evaluate(labels.cpu(), dists.cpu(), metric='roc')
print('Epoch{} AUROC: {:.3f}, test accuracy : {:.4f}'.format(
epoch, roc, test_acc / test_dataset.num_image / 2))
print(
'Epoch [{}/{}], Step {}, total_loss = {:.4f}, class = {:.4f}, membership = {:.4f}, transfer = {:.4f}, exe time: {:.2f}, lr: {:.4f}*e-4'
.format(epoch, args.num_epochs, i + 1, running_loss / batch_num,
running_class_loss / batch_num,
running_membership_loss / batch_num,
running_transfer_loss / batch_num,
time.time() - stime,
scheduler.get_last_lr()[0] * 10**4))
print('exe time: {:.2f}, lr: {:.4f}*e-4'.format(
time.time() - stime,
scheduler.get_last_lr()[0] * 10**4))
print("train accuracy total : {:.4f}".format(
train_acc / train_dataset.num_image / 2))
print("test accuracy total : {:.4f}".format(
test_acc / test_dataset.num_image / 2))
summary.add_scalar('loss/total_loss', running_loss / batch_num, epoch)
summary.add_scalar('loss/class_loss', running_class_loss / batch_num,
epoch)
summary.add_scalar('loss/membership_loss',
running_membership_loss / batch_num, epoch)
summary.add_scalar('acc/train_acc',
train_acc / train_dataset.num_image / 2, epoch)
summary.add_scalar('acc/test_acc',
test_acc / test_dataset.num_image / 2, epoch)
summary.add_scalar("learning_rate/lr",
scheduler.get_last_lr()[0], epoch)
time.sleep(0.001)
torch.save(
{
'model': model.state_dict(),
'epoch': epoch,
'init_lr': scheduler.get_last_lr()[0]
},
os.path.join(save_model, env, args.net_type,
'checkpoint_last_pre.pth.tar'))
import planner as pln
import hardware as hw
import dataset
import models
import torch.nn
import torch
import time
simd_cfg_path = '../../hwcfg/simd.json'
hw_spec = hw.HardwareSpec(simd_cfg_path)
data = dataset.imagenet()
vgg16 = models.vgg16()
pnn = pln.Planner()
start_time = time.time()
for name, module in vgg16.named_modules():
if isinstance(module, torch.nn.Sequential):
continue
pnn.set_data(data=data, module=module, hw_spec=hw_spec, layer_name=name)
data = pnn.run('../../build')
elapsed_time = time.time() - start_time
print('[Front-end] Elapsed time: ' +
time.strftime('%H hours %M min %S sec', time.gmtime(elapsed_time)))
