def __init__(self, csv_path, dataset_path, bag_refer_list, val_refer_list, logUtil=None, cuda_device=2, description=""):
self.log = logUtil
self.printlog("Current PID: " + str(os.getpid()))
self.device = cuda_device
self.description = description
TrainDataset = DataUtils.DatasetLoader(csv_path, dataset_path, refer_list=np.load(bag_refer_list),
mode="Train", up_size=self.UP_SIZE)
ValDataset = DataUtils.DatasetLoader(csv_path, dataset_path, refer_list=np.load(val_refer_list),
mode="Valid", up_size=self.UP_SIZE)
self.trainloader = torch.utils.data.DataLoader(TrainDataset, batch_size=self.BATCH_SIZE, num_workers=2, shuffle=True)
self.validloader = torch.utils.data.DataLoader(ValDataset, batch_size=self.BATCH_SIZE, num_workers=2, shuffle=True)
self.max_accu = 0
# deprecation._PRINT_DEPRECATION_WARNINGS = False
# tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.WARNING)
# tf.get_logger().setLevel('INFO')
# tf.autograph.set_verbosity(1)
import numpy as np
# import tensorflow.contrib.eager as tfe
# Enable eager execution
# tfe.enable_eager_execution()
# import logging
# logging.disable(logging.WARNING)
import resnet
import vgg_preprocessing
import utils_imagenet
from configparser import ConfigParser
from Utils import DataUtils
utils = DataUtils()
##################################################
if len(sys.argv) != 2: # We have to give 1 arg
print('Arguments: config')
sys.exit(-1)
cp = ConfigParser()
cp.read(sys.argv[1])
cp = cp[os.path.basename(__file__)]
#######################################################
_DEFAULT_IMAGE_SIZE = 224
_NUM_CHANNELS = 3
_BIAS_EPOCHS = 2
_NUM_TRAIN_FILES = 1024
from bagging.MergeResults import BaggingResult
from Utils import DataUtils
csv_path = "q1_data/train2.csv"
dataset_path = "q1_data/train.npy"
val_refer_list = "bagging/val.npy"
BATCH_SIZE = 20
CUDA_DEVICE = 2
CLASS_NUM = 100
UP_SIZE = (224,224)
bag_pkl_paths=["./pklmodels/Class100_A_epoch_40.pkl",
"./pklmodels/Class100_B_epoch_40.pkl",
"./pklmodels/Class100_C_epoch_40.pkl"]
ValDataset = DataUtils.DatasetLoader(csv_path, dataset_path, refer_list=np.load(val_refer_list),
mode="Valid", up_size=UP_SIZE)
validloader = torch.utils.data.DataLoader(ValDataset, batch_size=BATCH_SIZE, num_workers=2, shuffle=True)
results = BaggingResult(CUDA_DEVICE, bag_pkl_paths=bag_pkl_paths, class_num=CLASS_NUM)
merge_accuracy = []
split_accuracy = [[] for i in range(len(bag_pkl_paths))]
for i, data in enumerate(validloader):
_, val_x, val_label = data
merge_res, split_res = results.pred(val_x)
merge_accuracy.append((val_label==merge_res).numpy().mean())
print(i*BATCH_SIZE, " - ", (i+1)*BATCH_SIZE)
for j in range(split_res.shape[0]):
res = split_res[j]
split_accuracy[j].append((val_label==res).numpy().mean())
from utils_incremental.train_eval_MS import train_eval_MS
from utils_incremental.train_eval_LF import train_eval_LF
from utils_incremental.train_eval_MR_LF import train_eval_MR_LF
######### Modifiable Settings ##########
from configparser import ConfigParser
from Utils import DataUtils
import warnings, socket, os
if len(sys.argv) != 2: # We have to give 1 arg
print('Arguments: config')
sys.exit(-1)
with warnings.catch_warnings(record=True) as warn_list:
######### Modifiable Settings ##########
utils = DataUtils()
# loading configuration file
cp = ConfigParser()
cp.read(sys.argv[1])
cp = cp[os.path.basename(__file__)]
########################################
train_batch_size = int(cp['train_batch_size']) # Batch size for train
test_batch_size = int(cp['test_batch_size']) # Batch size for test
eval_batch_size = int(cp['eval_batch_size']) # Batch size for eval
base_lr = float(cp['base_lr']) # Initial learning rate
lr_strat = utils.from_str_to_list(
cp['lr_strat'], 'int') # Epochs where learning rate gets decreased
lr_factor = float(cp['lr_factor']) # Learning rate decrease factor
custom_weight_decay = float(cp['custom_weight_decay']) # Weight Decay
custom_momentum = float(cp['custom_momentum']) # Momentum
normalization_dataset_name = cp['normalization_dataset_name'] # Momentum
from __future__ import division
import numpy as np
import torch as th
import AverageMeter as AverageMeter
import sys, os, copy
from Utils import DataUtils
utils = DataUtils()
if len(sys.argv) != 9:
print(
'Arguments: images_list_files_path, exemplars_files_path, scores_path, b1_scores_path, K, P, S, dataset'
)
sys.exit(-1)
#Parameters###############################
batch_size = 256
images_list_files_path = sys.argv[1]
exemplars_files_path = sys.argv[2]
scores_path = sys.argv[3]
b1_scores_path = sys.argv[4]
K = int(sys.argv[5]) #size of the memory for past class exemplars
P = int(sys.argv[6]) #number of classes per state
S = int(
sys.argv[7]) #number of states, including the first non-incremental one
dataset = sys.argv[8]
###########################################
print('Dataset name = ' + dataset)
top1_acc_ft = []
top1_acc_mc = []
top5_acc_ft = []
'Arguments: ft_feat_scores_path, ft_weights_path, K, P, S, Dataset_name [vgg_faces|ilsvrc|google_landmarks]'
)
sys.exit(-1)
#Parameters###############################
batch_size = 256
scores_path = sys.argv[1]
weights_path = sys.argv[2]
memory_size = sys.argv[3]
P = int(sys.argv[4])
S = int(sys.argv[5])
dataset = sys.argv[6]
#Parameters#############################################
print('Dataset name = ' + dataset)
utils = DataUtils()
top1_accuracies = []
rectified_top1_accuracies = []
top5_accuracies = []
rectified_top5_accuracies = []
batch_initial_weight_matrix = {}
batch_initial_bias_vector = {}
#get first batch weights and bias
first_model_weights_path = os.path.join(
weights_path,
dataset + '/S~' + str(S) + '/K~' + memory_size + '/b1_weight_bias.tf')
with open(first_model_weights_path, 'rb') as fp:
intermediate_models_save_dir = os.path.join(cp['intermediate_models_save_dir'],
algo_name)
saving_intermediate_models = cp['saving_intermediate_models'] == 'True'
datasets_mean_std_file_path = cp['datasets_mean_std_file_path']
if saving_intermediate_models:
if not os.path.exists(intermediate_models_save_dir):
os.makedirs(intermediate_models_save_dir)
print('Loading train images from ' + train_file_path)
print('Loading val images from ' + val_file_path)
print('Dataset name for normalization = ' + normalization_dataset_name)
#catching warnings
with warnings.catch_warnings(record=True) as warn_list:
utils = DataUtils()
dataset_mean, dataset_std = utils.get_dataset_mean_std(
normalization_dataset_name, datasets_mean_std_file_path)
print('dataset mean = ' + str(dataset_mean))
print('dataset std = ' + str(dataset_std))
# Data loading code
normalize = transforms.Normalize(mean=dataset_mean, std=dataset_std)
train_dataset = ImagesListFileFolder(
train_file_path,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
import numpy as np
import os, math
import sys, socket
from configparser import ConfigParser
try:
import cPickle
except:
import _pickle as cPickle
# Syspath for the folder with the utils files
# sys.path.insert(0, "/media/data/srebuffi")
import utils_resnet
import utils_icarl
import utils_data
from Utils import DataUtils
utils = DataUtils()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
if len(sys.argv) != 2: # We have to give 1 arg
print('Arguments: config')
sys.exit(-1)
# loading configuration file
cp = ConfigParser()
cp.read(sys.argv[1])
cp = cp[os.path.basename(__file__)]
######### Modifiable Settings ##########
batch_size = int(cp['batch_size']) # Batch size
nb_groups = int(cp['nb_groups']) # Number of groups
nb_cl = int(cp['nb_cl']) # Number of groups
#!/usr/bin/env python
# coding=utf-8
import torch
import torch.nn as nn
import torch.nn.functional as F
from utils_pytorch import *
from Utils import DataUtils
import AverageMeter as AverageMeter
utils = DataUtils()
#code for baseline1 : FT+standard distillation
def train_eval(epochs, tg_model, ref_model, tg_optimizer, tg_lr_scheduler, \
trainloader, testloader, \
iteration, start_iteration, \
T, beta, \
fix_bn=False, weight_per_class=None, device=None):
if device is None:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if iteration > start_iteration:
ref_model.eval()
num_old_classes = ref_model.fc.out_features
top = min(5, tg_model.fc.out_features)
for epoch in range(epochs):
#train
tg_model.train()
if fix_bn:
for m in tg_model.modules():
# First batch parameters
first_model_load_path = cp['first_model_load_path']
# Incremental batches dataset
normalization_dataset_name = cp['normalization_dataset_name']
dataset_files_dir = cp['dataset_files_dir']
data_output_dir = cp['data_output_dir']
# Semi-supervised labelisation settings
B = float(cp['B'])
classical_AF = cp['classical_AF']
rerun = cp['rerun'] == "True"
apply_th_train = cp['apply_th_train'] == "True"
apply_th_val_al = cp['apply_th_val_al'] == "True"
train_files_dir = os.path.join(dataset_files_dir, 'separated/train')
path_val_batch1 = os.path.join(dataset_files_dir, 'batch1/val.lst')
path_train_batch1 = os.path.join(dataset_files_dir, 'batch1/train.lst')
full_paths_suffix = ''
################ Global variables
utils = DataUtils()
dataset_mean, dataset_std = utils.get_dataset_mean_std(
normalization_dataset_name, datasets_mean_std_file_path)
normalize = transforms.Normalize(mean=dataset_mean, std=dataset_std)
print("Running on " + str(socket.gethostname()) + " | gpu " + str(gpu))
utils.print_parameters(cp)
assert (mode in ['il', 'il_al'])
# start the main program
main(I)
#!/usr/bin/nohup python
# -*- coding: utf-8 -*-
__author__ = "Eden Belouadah & Adrian Popescu"
import sys, copy
import numpy as np
import torch as th
import AverageMeter as AverageMeter
from Utils import DataUtils
utils = DataUtils()
from sklearn import preprocessing
dataset = sys.argv[1] # PARAM - name of the dataset
list_root_dir = sys.argv[2] # root dir for the list of images
local_root_dir = sys.argv[
3] # root dir on node for the features and the classification masks
Z = int(sys.argv[4]) # PARAM - number of incremental Z for the dataset
B = int(sys.argv[5]) # PARAM - size of the allowed B for past classes
P = int(sys.argv[6]) # PARAM - size of each incremental batch
last_batch_number = int(sys.argv[7]) # PARAM - max number of incremental Z
top_rewinded = int(
sys.argv[8]) # PARAM - number of past classes used for rewinding
batch_size = 256
''' USAGE
######### CIFAR-100 ######### S=10 : K={1000,500,250} | S=20, S=50 : K=250
nohup python ./codes/scail.py cifar100 ./data/images_list_files /path/to/scail/data 10 1000 10 10 10 2>&1 | tee /path/to/your/logs/scail/cifar100/s10/k1000/scail_cifar100_s10_1k.log &
nohup python ./codes/scail.py cifar100 ./data/images_list_files /path/to/scail/data 10 500 10 10 10 2>&1 | tee /path/to/your/logs/scail/cifar100/s10/k500/scail_cifar100_s10_0,5k.log &
if len(sys.argv) != 8:
print(
'Arguments: fine_tuning_scores_path, K, P, S, Dataset_name [vgg_faces|ilsvrc|google_landmarks], first_batch_number [2-, last_batch_number'
)
sys.exit(-1)
#Parameters###############################
batch_size = 256
scores_path = sys.argv[1]
memory_size = sys.argv[2]
P = int(sys.argv[3])
S = int(sys.argv[4])
dataset = sys.argv[5]
first_batch = int(sys.argv[6])
last_batch = int(sys.argv[7])
utils = DataUtils()
#Parameters###############################
images_list_files_path = '/home/eden/images_list_files/'
###########################################
print('Dataset name = ' + dataset)
print('scores_path = ' + scores_path)
print('K = ' + memory_size)
print('P = ' + str(P))
print('S = ' + str(S))
print('first_batch = ' + str(first_batch))
print('last_batch= ' + str(last_batch))
def softmax(x):
e_x = np.exp(x - np.max(x))
return e_x / e_x.sum()
save_csv_path = "q1_data/samplesummision_class100.csv"
testset_path = "q1_data/test.npy"
BATCH_SIZE = 20
CUDA_DEVICE = 2
CLASS_NUM = 100
UP_SIZE = (224, 224)
csvheader = ["image_id", "fine_label"]
bag_pkl_paths = [
"./pklmodels/Class20_A_epoch_40.pkl", "./pklmodels/Class20_B_epoch_40.pkl",
"./pklmodels/Class20_C_epoch_40.pkl"
]
testDataset = DataUtils.DatasetLoader("q1_data/samplesummission1.csv",
testset_path,
mode="Test",
up_size=UP_SIZE)
setsize = len(testDataset)
testloader = torch.utils.data.DataLoader(testDataset,
batch_size=BATCH_SIZE,
num_workers=2,
shuffle=False)
results = BaggingResult(CUDA_DEVICE,
bag_pkl_paths=bag_pkl_paths,
class_num=CLASS_NUM)
resdata = -np.ones(setsize) # Default label -1
for i, data in enumerate(testloader):
index, val_x, val_label = data
print(np.min(index.numpy()), " - ", np.max(index.numpy()))
kernel_initializer=init)(d)
# define model
model = Model(in_image, patch_out)
return model
if __name__ == '__main__':
path2TR1 = os.path.join(os.getcwd(), 'Tr1', 'TrainT1')
path2TR2 = os.path.join(os.getcwd(), 'Tr2', 'TrainT2')
checkpoint_path = os.path.join(os.getcwd(), 'Trained_Model4')
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
batch_size = 4
epochs = 100
# load images from
images_x = DataUtils(path2TR1, (220, 184)).get_data(batch_size)
images_y = DataUtils(path2TR2, (220, 184)).get_data(batch_size)
sample_x_data = next(iter(images_x))
sample_y_data = next(iter(images_y))
cycleGan = CycleGAN_TR1_TR2((220, 184, 1), checkpoint_path)
cycleGan.train(images_x,
images_y,
epochs,
plot_results=True,
sample_data=(sample_x_data, sample_y_data))
