def set_seed(seed: int) -> None:
torch.manual_seed(seed)
cuda.manual_seed(seed)
cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)
cudnn.benchmark = False
cudnn.deterministic = True
def set_random_seed(seed):
np.random.seed(seed)
cudnn.deterministic = True
cudnn.benchmark = False
torch.manual_seed(seed)
if cuda.is_available():
cuda.manual_seed_all(seed)
print('Random seed:', seed)
def set_random_seed(seed: int):
if seed == -1:
seed = random.randint(0, 99999)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
cuda.manual_seed_all(seed)
return seed
def __init__(self, args, crition: nn.CrossEntropyLoss, optimzer):
seed = random.randint(0, 1000)
random.seed(seed)
torch.manual_seed(seed)
cd.manual_seed(seed)
cd.manual_seed_all(seed)
self.args = args
self.model_name = args.net
self.config = self._parse_args(args.config)
net_module = importlib.import_module(f"net.{self.model_name}")
self.model_class = getattr(net_module, self.model_name)
self.model = self.model_class(**self._parse_model_args())
self.crition = crition
self.base_lr = self.config.get("lr", 0.01)
self.optimizer = self._get_optimizer(optimzer)
self.iters = self.config.get("iter", 5000)
self.power = self.config.get("power", 0.9)
self.numclass = self.config['numclass']
self.batch_size = self.config['batch_size']
self.print_freq = self.config['print_freq']
self.save_freq = self.config['save_freq']
self.gpu = self.config.get('gpus')
print(f"gpus: {self.gpu}")
if self.gpu:
self.gpu = [self.gpu] if isinstance(self.gpu, int) else list(
self.gpu)
else:
self.device = torch.device("cpu")
self.train_dataloader = get_data_loader(
self.config['train_data_path'],
self.config['train_annot_path'],
self.numclass,
img_size=self.config['img_size'],
batch_size=self.batch_size,
name=self.config['dataset_name'])
self.val_dataloader = get_data_loader(self.config['val_data_path'],
self.config['val_annot_path'],
self.numclass,
img_size=self.config['img_size'],
batch_size=self.batch_size,
name=self.config['dataset_name'],
mode='eval')
self.metricer = Metrics(self.numclass)
logdir = self._get_log_dir()
self.writer = SummaryWriter(log_dir=logdir)
if self.gpu:
print(torch.cuda.device_count())
self.model = nn.DataParallel(self.model,
device_ids=self.gpu).cuda(self.gpu[0])
# self.crition = self.crition.cuda(self.gpu[0])
cudnn.benchmark = False # 加速1
cudnn.deterministic = True
def set_seed(args):
"""
Set the random seed to ensure reproducible result for the random generator of python, numpy, pytorch, and cuda
Parameters:
args : int
Random seed to set
"""
random.seed(args)
np.random.seed(args)
manual_seed(args)
manual_seed_all(args)
def fix_random_state(seed_value):
"""
fix the random seed of each library
"""
random.seed(seed_value)
np.random.seed(seed_value)
if torch.cuda.is_available():
cuda.manual_seed(seed_value)
cuda.manual_seed_all(seed_value)
torch.manual_seed(seed_value)
torch.random.manual_seed(seed_value)
def set_seed(self, seed):
'''
Set the seed across all different necessary platforms
to allow for comparison of different models and runs
:param seed: random seed to set for all random num generators
:type seed: int
'''
torch.manual_seed(seed)
cuda.manual_seed_all(seed)
# Not totally sure what these two do!
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
random.seed(seed)
def set_seed(seed: int,
cudnn_deterministic: bool = False,
cudnn_benchmark: bool = True):
"""
Set all relevant seeds for torch, numpy and python
Args:
seed: int seed
cudnn_deterministic: set True for deterministic training..
cudnn_benchmark: set False for deterministic training.
"""
th.manual_seed(seed)
cuda.manual_seed(seed)
cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)
if cudnn_deterministic:
cudnn.deterministic = True
cudnn.benchmark = cudnn_benchmark
def set_seed(seed: int, set_deterministic: bool = True):
"""
Set all relevant seeds for torch, numpy and python
Args:
seed: int seed
set_deterministic: Guarantee deterministic training, possibly at the cost of performance.
"""
th.manual_seed(seed)
cuda.manual_seed(seed)
cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)
if set_deterministic:
cudnn.benchmark = False
cudnn.deterministic = True
elif cudnn.benchmark or not cudnn.deterministic:
print(
f"WARNING: Despite fixed seed {seed}, training may not be deterministic with {cudnn.benchmark=} "
f"(must be False for deterministic training) and {cudnn.deterministic=} (must be True for deterministic "
f"training)")
help="Path to the validation target data")
opt = parser.parse_args()
if opt.computational_batch_size <= 0:
opt.computational_batch_size = opt.batch_size
print(opt)
print '__PYTORCH VERSION:', torch.__version__
if torch.cuda.is_available() and not opt.gpus:
print("WARNING: You have a CUDA device, should run with -gpus 0")
if opt.gpus:
cuda.set_device(opt.gpus[0])
cuda.manual_seed_all(opt.seed)
if not opt.disable_cudnn:
print '__CUDNN VERSION:', torch.backends.cudnn.version()
else:
torch.backends.cudnn.enabled = False
import rnnlib.SRU
torch.manual_seed(opt.seed)
def NMTCriterion(vocabSize):
weight = torch.ones(vocabSize)
weight[nmt.Constants.PAD] = 0
crit = nn.NLLLoss(weight, size_average=False)
def fit(self, model: "TorchModel", data: Union[DataUtil, List[DataUtil],
TorchDataSetProvider]):
self.log.info(f"Preparing parameter learning of {model} via {self}")
def toDataSetProvider(d) -> TorchDataSetProvider:
if isinstance(d, TorchDataSetProvider):
return d
elif isinstance(d, DataUtil):
return TorchDataSetProviderFromDataUtil(d, self.cuda)
if type(data) != list:
dataSetProviders = [toDataSetProvider(data)]
else:
dataSetProviders = [toDataSetProvider(item) for item in data]
# initialise data to be generated
self.trainingLog = []
self.bestEpoch = None
def trainingLog(s):
self.log.info(s)
self.trainingLog.append(s)
self._init_cuda()
# Set the random seed manually for reproducibility.
seed = 42
torch.manual_seed(seed)
if self.cuda:
torchcuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
# obtain data, splitting it into training and validation set(s)
validationSets = []
trainingSets = []
outputScalers = []
self.log.info("Obtaining input/output training instances")
for idxDataSetProvider, dataSetProvider in enumerate(dataSetProviders):
outputScalers.append(dataSetProvider.getOutputTensorScaler())
trainS, valS = dataSetProvider.provideSplit(self.trainFraction)
trainingLog(
f"Data set {idxDataSetProvider+1}/{len(dataSetProviders)}: #train={trainS.size()}, #validation={valS.size()}"
)
validationSets.append(valS)
trainingSets.append(trainS)
trainingLog("Number of validation sets: %d" % len(validationSets))
torchModel = model.createTorchModule()
if self.cuda:
torchModel.cuda()
model.setTorchModule(torchModel)
nParams = sum([p.nelement() for p in torchModel.parameters()])
self.log.info(f"Learning parameters of {model} via {self}")
trainingLog('Number of parameters: %d' % nParams)
criterion = self.lossEvaluator.getTrainingCriterion()
if self.cuda:
criterion = criterion.cuda()
best_val = 1e9
best_epoch = 0
optim = _Optimiser(torchModel.parameters(),
method=self.optimiser,
lr=self.optimiserLR,
max_grad_norm=self.optimiserClip,
lr_decay=self.optimiserLRDecay,
start_decay_at=self.startLRDecayAtEpoch,
use_shrinkage=self.useShrinkage,
**self.optimiserArgs)
bestModelBytes = model.getModuleBytes()
self.lossEvaluatorState = self.lossEvaluator.startTraining(self.cuda)
validationMetricName = self.lossEvaluator.getValidationMetricName()
try:
self.log.info('Begin training')
self.log.info('Press Ctrl+C to end training early')
for epoch in range(1, self.epochs + 1):
epoch_start_time = time.time()
# perform training step, processing all the training data once
train_loss = self._train(trainingSets, torchModel, criterion,
optim, self.batchSize, self.cuda,
outputScalers)
# perform validation, computing the mean metrics across all validation sets (if more than one)
metricsSum = None
metricsKeys = None
for i, (validationSet, outputScaler) in enumerate(
zip(validationSets, outputScalers)):
metrics = self._evaluate(validationSet, torchModel,
outputScaler)
metricsArray = np.array(list(metrics.values()))
if i == 0:
metricsSum = metricsArray
metricsKeys = metrics.keys()
else:
metricsSum += metricsArray
metricsSum /= len(validationSets) # mean results
metrics = dict(zip(metricsKeys, metricsSum))
# check for new best result according to validation results
current_val = metrics[
self.lossEvaluator.getValidationMetricName()]
isNewBest = current_val < best_val
if isNewBest:
best_val = current_val
best_epoch = epoch
bestStr = "best {:s} {:5.6f} from this epoch".format(
validationMetricName, best_val)
else:
bestStr = "best {:s} {:5.6f} from epoch {:d}".format(
validationMetricName, best_val, best_epoch)
trainingLog(
'Epoch {:3d}/{} completed in {:5.2f}s | train loss {:5.4f} | validation {:s} | {:s}'
.format(
epoch, self.epochs, (time.time() - epoch_start_time),
train_loss,
", ".join(["%s %5.4f" % e
for e in metrics.items()]), bestStr))
if isNewBest:
bestModelBytes = model.getModuleBytes()
trainingLog("Training complete")
except KeyboardInterrupt:
trainingLog('Exiting from training early')
trainingLog(
'Best model is from epoch %d with %s %f on validation set' %
(best_epoch, validationMetricName, best_val))
self.bestEpoch = best_epoch
# reload best model
model.setModuleBytes(bestModelBytes)
import torch.optim as optim
from sklearn.model_selection import StratifiedKFold
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from torchvision import models
from tqdm import tqdm as tqdm
from datasets.gtzan import GTZAN_MELSPEC as GTZAN
# In[2]:
# Random seeds
torch.manual_seed(1234)
cuda.manual_seed_all(1234)
np.random.seed(1234)
SEGMENTS = None
BATCH_SIZE = 4
EPOCHS = 300
LR = 1e-3
NUM_CLASSES = 10
NUM_KFOLD = 10
NUM_REDUCE_LR_PATIENCE = 3
NUM_EARLY_STOPPING_PATIENCE = 10
DEVICE = torch.device('cuda:0')
# In[3]:
def get_arguments():
parser = argparse.ArgumentParser()
# Dataset
parser.add_argument('--dataset',
default='celebA',
choices=['celebA', 'TinyImagenet'])
parser.add_argument(
'--imageSize',
type=int,
default=64,
help='the height / width of the input image to network')
parser.add_argument('--nz',
type=int,
default=100,
help='size of the latent z vector')
parser.add_argument('--ngf',
type=int,
default=64,
help='Size of the output image from the generator')
parser.add_argument('--ndf',
type=int,
default=64,
help='Size of the input image to the discriminator')
# Optimization
parser.add_argument('--niter',
type=int,
default=150,
help='number of epochs to train for')
parser.add_argument('--lr',
type=float,
default=0.0002,
help='learning rate, default=0.0002')
parser.add_argument('--beta1',
type=float,
default=0.5,
help='beta1 for adam. default=0.5')
parser.add_argument('--batchSize',
type=int,
default=64,
help='input batch size')
parser.add_argument('--workers',
type=int,
help='number of data loading workers',
default=2)
parser.add_argument('--ngpu',
type=int,
default=1,
help='number of GPUs to use')
parser.add_argument('--manual-seed', type=int, help='manual seed')
# GAN architecture
parser.add_argument(
'--mode',
type=str,
default='mmgan',
metavar='N',
help='Type of GAN: minimax, non-saturating, least-square, Wasserstein.',
choices=['mmgan', 'nsgan', 'lsgan', 'wgan'])
parser.add_argument(
'--upsample',
type=str,
default='nearest',
choices=['convtranspose', 'nearest', 'bilinear'],
help='Method used in the generator to up-sample images.')
parser.add_argument('--critic_iter',
type=int,
default=1,
help='number of critic iterations')
parser.add_argument('--gen_iter',
type=int,
default=1,
help='number of generator iterations')
parser.add_argument('--lanbda',
type=float,
default=1,
help='Regularization factor for the gradient penalty.')
parser.add_argument(
'--penalty',
type=str,
default='both',
choices=['real', 'fake', 'both', 'uniform', 'midinterpol', 'grad_g'],
help='Distribution on which to apply gradient penalty.')
parser.add_argument(
'--spectral-norm',
type=bool,
default=False,
help='If True, use spectral normalization in the discriminator.')
# Checkpoints
parser.add_argument('--netG',
default='',
help="path to netG (to continue training)")
parser.add_argument('--netD',
default='',
help="path to netD (to continue training)")
parser.add_argument('--outf',
default='gan/logs/dcgan/',
help='folder to output images and model checkpoints')
parser.add_argument('--name',
type=str,
default='',
metavar='N',
help='Name of the session appended to folder name.')
opt = parser.parse_args()
print(opt)
# DATASET
if opt.dataset.lower() == 'celeba':
opt.dataroot = '/data/lisa/data/celeba'
elif opt.dataset.lower() == 'tinyimagenet':
opt.dataroot = '/data/lisa/data/tiny-imagenet-200/train'
else:
raise NotImplementedError
# Number of input channels (RGB)
opt.nc = 3
# CUDA
cudnn.benchmark = True
opt.cuda = cuda.is_available()
# SEED
if opt.manual_seed is None:
opt.seed = random.randint(1, 10000)
else:
opt.seed = opt.manual_seed
print("Random Seed: ", opt.seed)
random.seed(opt.seed)
torch.manual_seed(opt.seed)
if opt.cuda:
cuda.manual_seed_all(opt.seed)
# OUT FOLDER
root_path = f'/data/milatmp1/{getpass.getuser()}'
now = datetime.datetime.now()
strpenalty = '0' if opt.lanbda <= 0 else f'{opt.lanbda}{opt.penalty}'
strspectral = '_SN' if opt.spectral_norm else ''
folder_path = (
f'{opt.dataset}/{now.month}_{now.day}'
f'/{now.hour}_{now.minute}_{opt.mode}_{opt.name}'
f'_gp={strpenalty}{strspectral}_citer={opt.critic_iter}_giter={opt.gen_iter}'
f'_beta1={opt.beta1}_upsample={opt.upsample}_seed={opt.seed}')
opt.outf = os.path.join(root_path, opt.outf, folder_path)
print('Outfile: ', opt.outf)
os.makedirs(opt.outf)
return opt
import torch
import torch.nn as nn
from torch import cuda
from sentence_transformers import SentenceTransformer
torch.manual_seed(0)
cuda.manual_seed_all(0)
class Encoder(nn.Module):
def __init__(self, emb_size, hidden_size, dropout_rate):
super(Encoder, self).__init__()
self.hidden_layer = nn.Linear(emb_size, hidden_size)
self.dropout = nn.Dropout(p=dropout_rate)
self.tanh = nn.Tanh()
def forward(self, emb):
hidden = self.tanh(self.hidden_layer(self.dropout(emb)))
return hidden
class Classifier(nn.Module):
def __init__(self, hidden_size):
super(Classifier, self).__init__()
self.output_layer = nn.Linear(hidden_size, 1)
self.sigmoid = nn.Sigmoid()
def forward(self, hidden):
output = self.output_layer(hidden)
pre = self.sigmoid(output)
def fit(self, model: "TorchModel", data: Union[DataUtil, List[DataUtil], TorchDataSetProvider, List[TorchDataSetProvider],
TorchDataSet, List[TorchDataSet], Tuple[TorchDataSet, TorchDataSet], List[Tuple[TorchDataSet, TorchDataSet]]],
createTorchModule=True) -> "TrainingInfo":
"""
Fits the parameters of the given model to the given data, which can be a list of or single instance of one of the following:
* a `DataUtil` or `TorchDataSetProvider` (from which a training set and validation set will be obtained according to
the `trainFraction` parameter of this object)
* a `TorchDataSet` which shall be used as the training set (for the case where no validation set shall be used)
* a tuple with two `TorchDataSet` instances, where the first shall be used as the training set and the second as
the validation set
:param model: the model to be fitted
:param data: the data to use (see variants above)
:param createTorchModule: whether to newly create the torch module that is to be trained from the model's factory.
If False, (re-)train the existing module.
"""
self.cuda = model.cuda
self.log.info(f"Preparing parameter learning of {model} via {self} with cuda={self.cuda}")
useValidation = self.params.trainFraction != 1.0
def toDataSetProvider(d) -> TorchDataSetProvider:
if isinstance(d, TorchDataSetProvider):
return d
elif isinstance(d, DataUtil):
return TorchDataSetProviderFromDataUtil(d, self.cuda)
else:
raise ValueError(f"Cannot create a TorchDataSetProvider from {d}")
trainingLogEntries = []
def trainingLog(s):
self.log.info(s)
trainingLogEntries.append(s)
self._init_cuda()
# Set the random seed manually for reproducibility.
seed = 42
torch.manual_seed(seed)
if self.cuda:
torchcuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
# obtain data, splitting it into training and validation set(s)
validationSets = []
trainingSets = []
outputScalers = []
if type(data) != list:
data = [data]
self.log.info("Obtaining input/output training instances")
for idxDataItem, dataItem in enumerate(data):
if isinstance(dataItem, TorchDataSet):
if useValidation:
raise ValueError("Passing a TorchDataSet instance is not admissible when validation is enabled (trainFraction != 1.0). Pass a TorchDataSetProvider or another representation that supports validation instead.")
trainingSet = dataItem
validationSet = None
outputScaler = TensorScalerIdentity()
elif type(dataItem) == tuple:
trainingSet, validationSet = dataItem
outputScaler = TensorScalerIdentity()
else:
dataSetProvider = toDataSetProvider(dataItem)
trainingSet, validationSet = dataSetProvider.provideSplit(self.params.trainFraction)
outputScaler = dataSetProvider.getOutputTensorScaler()
trainingSets.append(trainingSet)
if validationSet is not None:
validationSets.append(validationSet)
outputScalers.append(outputScaler)
trainingLog(f"Data set {idxDataItem+1}/{len(data)}: #train={trainingSet.size()}, #validation={validationSet.size() if validationSet is not None else 'None'}")
trainingLog("Number of validation sets: %d" % len(validationSets))
torchModel = model.createTorchModule() if createTorchModule else model.getTorchModule()
if self.cuda:
torchModel.cuda()
model.setTorchModule(torchModel)
nParams = sum([p.nelement() for p in torchModel.parameters()])
self.log.info(f"Learning parameters of {model}")
trainingLog('Number of parameters: %d' % nParams)
trainingLog(f"Starting training process via {self}")
lossEvaluator = self.params.lossEvaluator
totalEpochs = None
best_val = 1e9
best_epoch = 0
optim = _Optimiser(torchModel.parameters(), method=self.params.optimiser, lr=self.params.optimiserLR,
max_grad_norm=self.params.shrinkageClip, use_shrinkage=self.params.useShrinkage, **self.params.optimiserArgs)
bestModelBytes = model.getModuleBytes()
lossEvaluation = lossEvaluator.startEvaluation(self.cuda)
validationMetricName = lossEvaluator.getValidationMetricName()
trainingLossValues = []
validationMetricValues = []
try:
self.log.info(f'Begin training with cuda={self.cuda}')
self.log.info('Press Ctrl+C to end training early')
for epoch in range(1, self.params.epochs + 1):
lossEvaluation.startEpoch()
epoch_start_time = time.time()
# perform training step, processing all the training data once
train_loss = self._train(trainingSets, torchModel, optim, lossEvaluation, self.params.batchSize, outputScalers)
trainingLossValues.append(train_loss)
# perform validation, computing the mean metrics across all validation sets (if more than one),
# and check for new best result according to validation results
isNewBest = False
if useValidation:
metricsSum = None
metricsKeys = None
for i, (validationSet, outputScaler) in enumerate(zip(validationSets, outputScalers)):
metrics = self._evaluate(validationSet, torchModel, lossEvaluation, outputScaler)
metricsArray = np.array(list(metrics.values()))
if i == 0:
metricsSum = metricsArray
metricsKeys = metrics.keys()
else:
metricsSum += metricsArray
metricsSum /= len(validationSets) # mean results
metrics = dict(zip(metricsKeys, metricsSum))
current_val = metrics[lossEvaluator.getValidationMetricName()]
validationMetricValues.append(current_val)
isNewBest = current_val < best_val
if isNewBest:
best_val = current_val
best_epoch = epoch
bestStr = "best {:s} {:5.6f} from this epoch".format(validationMetricName, best_val)
else:
bestStr = "best {:s} {:5.6f} from epoch {:d}".format(validationMetricName, best_val, best_epoch)
valStr = f' | validation {", ".join(["%s %5.4f" % e for e in metrics.items()])} | {bestStr}'
else:
valStr = ""
trainingLog(
'Epoch {:3d}/{} completed in {:5.2f}s | train loss {:5.4f}{:s}'.format(
epoch, self.params.epochs, (time.time() - epoch_start_time), train_loss, valStr))
totalEpochs = epoch
if useValidation:
if isNewBest:
bestModelBytes = model.getModuleBytes()
# check for early stopping
numEpochsWithoutImprovement = epoch - best_epoch
if self.params.earlyStoppingEpochs is not None and numEpochsWithoutImprovement >= self.params.earlyStoppingEpochs:
trainingLog(f"Stopping early: {numEpochsWithoutImprovement} epochs without validation metric improvement")
break
trainingLog("Training complete")
except KeyboardInterrupt:
trainingLog('Exiting from training early because of keyboard interrupt')
# reload best model according to validation results
if useValidation:
trainingLog(f'Best model is from epoch {best_epoch} with {validationMetricName} {best_val} on validation set')
self.bestEpoch = best_epoch
model.setModuleBytes(bestModelBytes)
return TrainingInfo(bestEpoch=best_epoch if useValidation else None, log=trainingLogEntries, totalEpochs=totalEpochs,
trainingLossSequence=trainingLossValues, validationMetricSequence=validationMetricValues)
def main(json_path: str = 'options/train_denoising.json'):
parser = argparse.ArgumentParser()
parser.add_argument('-opt',
type=str,
default=json_path,
help='Path to option JSON file.')
opt = option.parse(parser.parse_args().opt, is_train=True)
util.makedirs(
[path for key, path in opt['path'].items() if 'pretrained' not in key])
current_step = 0
option.save(opt)
# logger
logger_name = 'train'
utils_logger.logger_info(
logger_name, os.path.join(opt['path']['log'], logger_name + '.log'))
logger = logging.getLogger(logger_name)
logger.info(option.dict2str(opt))
# seed
seed = opt['train']['manual_seed']
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
cuda.manual_seed_all(seed)
# data
opt_data_train: Dict[str, Any] = opt["data"]["train"]
train_set: DatasetDenoising = select_dataset(opt_data_train, "train")
train_loader: DataLoader[DatasetDenoising] = DataLoader(
train_set,
batch_size=opt_data_train['batch_size'],
shuffle=True,
num_workers=opt_data_train['num_workers'],
drop_last=True,
pin_memory=True)
opt_data_test = opt["data"]["test"]
test_sets: List[DatasetDenoising] = select_dataset(opt_data_test, "test")
test_loaders: List[DataLoader[DatasetDenoising]] = []
for test_set in test_sets:
test_loaders.append(
DataLoader(test_set,
batch_size=1,
shuffle=False,
num_workers=1,
drop_last=True,
pin_memory=True))
# model
model = Model(opt)
model.init()
# train
start = time.time()
for epoch in range(1000000): # keep running
for train_data in tqdm(train_loader):
current_step += 1
model.feed_data(train_data)
model.train()
model.update_learning_rate(current_step)
if current_step % opt['train']['checkpoint_log'] == 0:
model.log_train(current_step, epoch, logger)
if current_step % opt['train']['checkpoint_test'] == 0:
avg_psnrs: Dict[str, List[float]] = {}
avg_ssims: Dict[str, List[float]] = {}
tags: List[str] = []
test_index = 0
for test_loader in tqdm(test_loaders):
test_set: DatasetDenoising = test_loader.dataset
avg_psnr = 0.
avg_ssim = 0.
for test_data in tqdm(test_loader):
test_index += 1
model.feed_data(test_data)
model.test()
psnr, ssim = model.cal_metrics()
avg_psnr += psnr
avg_ssim += ssim
if current_step % opt['train'][
'checkpoint_saveimage'] == 0:
model.save_visuals(test_set.tag)
avg_psnr = round(avg_psnr / len(test_loader), 2)
avg_ssim = round(avg_ssim * 100 / len(test_loader), 2)
name = test_set.name
if name in avg_psnrs:
avg_psnrs[name].append(avg_psnr)
avg_ssims[name].append(avg_ssim)
else:
avg_psnrs[name] = [avg_psnr]
avg_ssims[name] = [avg_ssim]
if test_set.tag not in tags:
tags.append(test_set.tag)
header = ['Dataset'] + tags
t = PrettyTable(header)
for key, value in avg_psnrs.items():
t.add_row([key] + value)
logger.info(f"Test PSNR:\n{t}")
t = PrettyTable(header)
for key, value in avg_ssims.items():
t.add_row([key] + value)
logger.info(f"Test SSIM:\n{t}")
logger.info(f"Time elapsed: {time.time() - start:.2f}")
start = time.time()
model.save(logger)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data', default='./data/squad/')
parser.add_argument('--epoch', type=int, default=100)
parser.add_argument('--batch_size', type=int, default=15)
parser.add_argument('--dev_batch_size', type=int, default=15)
parser.add_argument('--lr', type=float, default=0.5)
parser.add_argument('--char_emb_dim', type=int, default=16)
parser.add_argument('--word_emb_dim', type=int, default=300)
parser.add_argument('--char_filter_dim', type=int, default=100)
parser.add_argument('--n_gram_sizes', type=Tuple[int, ...], default=[5])
parser.add_argument('--hidden_dim', type=int, default=100)
parser.add_argument('--use_glove', type=bool, default=True)
parser.add_argument('--train_emb', default=False)
parser.add_argument('--dropout', default=0.2)
parser.add_argument('--bidirectional', default=True)
parser.add_argument('--log', default='original')
parser.add_argument('--gpu_device', default=3)
parser.add_argument('--size', default='')
parser.add_argument('--glove', default='../data/glove/glove.840B.300d.txt')
parser.add_argument('--decay_rate', default=0.999)
parser.add_argument('--checkpoint', default=200)
parser.add_argument('--cuda_set', default=True)
parser.add_argument('--seed', default=133)
parser.add_argument('--log_output', default=True)
parser.add_argument('--max_passage_len', default=None)
parser.add_argument('--log_file_name', default="revised_sent_attention")
parser.add_argument('--sent_attention', default=True)
args = parser.parse_args()
cuda.set_device(args.gpu_device)
cuda.manual_seed_all(args.seed)
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = False
train_dataset, dev_dataset = Dataset(), Dataset()
filename = ['lower_train_dataset.json', 'lower_dev_dataset.json']
train_dataset.load_dataset(args.data + filename[0])
dev_dataset.load_dataset(args.data + filename[1])
vocab = Vocabulary(file_path=args.data + "lower_vocabulary.json")
char_vocab = vocab.create_char_vocabulary(min_counts=7)
char_vocab.prepare_dataset()
pre_trained_embedding = _read_pretrained_embedding_file(
args.glove, args.word_emb_dim, vocab)
pre_trained_embedding = pre_trained_embedding.cuda()
print("Preparing batch loader...")
print("============= Train ===============")
train_loader = BatchLoader(train_dataset,
vocab,
char_vocab=char_vocab,
max_passage_len=args.max_passage_len,
cuda=args.cuda_set,
batch_size=args.batch_size)
print("============= Valid ===============")
dev_loader = BatchLoader(dev_dataset,
vocab,
char_vocab=char_vocab,
max_passage_len=args.max_passage_len,
cuda=args.cuda_set,
batch_size=args.dev_batch_size)
train_loader.save_global_sent_len_dist("train")
dev_loader.save_global_sent_len_dist("dev")
vocab_size, char_size = vocab.get_vocab_size(), char_vocab.get_vocab_size()
model = Bidaf(char_size,
vocab_size,
args.char_emb_dim,
args.word_emb_dim,
args.char_filter_dim,
args.n_gram_sizes,
args.hidden_dim,
dropout=args.dropout,
word_emb=pre_trained_embedding,
sent_attention=args.sent_attention).cuda()
criterion = nn.NLLLoss().cuda()
#optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, betas=[0.9, 0.9])
optimizer = optim.Adadelta(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
avg_params = flatten_params(model)
print("#" * 15, "Model Info", "#" * 15)
print("Model: ", model)
print("Criterion: ", criterion)
print("Optimizer: ", optimizer)
print("")
_logger = logging.getLogger('mylogger')
file_name = args.log_file_name
fileHandler = logging.FileHandler('./' + file_name + '.log')
streamHandler = logging.StreamHandler()
_logger.addHandler(fileHandler)
_logger.addHandler(streamHandler)
_logger.setLevel(logging.DEBUG)
_logger.info(args)
print(
"\n=================== Start Train ======================"
)
for epoch in range(args.epoch):
if epoch > 2:
lr = args.lr * 0.8
for param_group in optimizer.param_groups:
print("learning rate change")
param_group['lr'] = lr
train_loss = AverageMeter()
start_acc = AverageMeter()
end_acc = AverageMeter()
span_acc = BooleanAccuracy()
squad_em_f1 = SquadEmAndF1()
start_time = time.time()
for i, data in enumerate(tqdm(train_loader)):
model.train()
passage, query = data["passage"], data["query"]
char_passage, char_query = data["char_passage"], data["char_query"]
passage_mask, query_mask = torch.clamp(passage, 0,
1).long(), torch.clamp(
query, 0, 1).long()
sent_start, sent_end, sent_mask = data["sent_start"], data[
"sent_end"], data["sent_mask"]
optimizer.zero_grad()
output = model(passage,
query,
char_paragraph=char_passage,
char_query=char_query,
dm=passage_mask,
qm=query_mask,
sent_start=sent_start,
sent_end=sent_end,
sent_mask=sent_mask)
loss1 = criterion(output["span_start_log"], data["span_start"])
loss2 = criterion(output["span_end_log"], data["span_end"])
loss = loss1 + loss2
_, val_pred1 = output["span_start_probs"].data.cpu().topk(1)
_, val_pred2 = output["span_end_probs"].data.cpu().topk(1)
batch_size = output["span_start_probs"].size(0)
best_spans = get_best_span(output["span_start_logits"],
output["span_end_logits"])
span_acc(
best_spans.data,
torch.stack([data["span_start"], data["span_end"]], -1).data)
best_predict_strings = get_best_predict_string(
batch_size, best_spans, data["original_passage"],
data["token_offset"])
for b_i in range(batch_size):
squad_em_f1(best_predict_strings[b_i],
data["answer_texts"][b_i])
start_acc.update(
index_accuracy(val_pred1.numpy(),
tensor2np(data["span_start"])))
end_acc.update(
index_accuracy(val_pred2.numpy(), tensor2np(data["span_end"])))
loss.backward()
optimizer.step()
avg_params = exp_moving_avg(model,
avg_params,
decay_rate=args.decay_rate)
train_loss.update(loss.data[0])
if i % args.checkpoint == 0 and i != 0:
em, f1 = squad_em_f1.get_metric(reset=False)
_span_acc = span_acc.get_metric(reset=False)
print("")
message = "Train epoch: %d iter: %d start_acc: %1.2f end_acc: %1.2f train_loss: %1.2f span_acc: %1.2f em: %1.2f f1: %1.2f elapsed: %1.2f " \
% (epoch + 1, i, start_acc.avg, end_acc.avg, train_loss.avg, _span_acc,
em, f1, time.time() - start_time)
_logger.info(message)
print("")
em, f1 = squad_em_f1.get_metric(reset=False)
_span_acc = span_acc.get_metric(reset=False)
print("")
message = "Train epoch: %d iter: %d start_acc: %1.2f end_acc: %1.2f train_loss: %1.2f span_acc: %1.2f em: %1.2f f1: %1.2f elapsed: %1.2f " \
% (epoch + 1, i, start_acc.avg, end_acc.avg, train_loss.avg, _span_acc,
em, f1, time.time() - start_time)
_logger.info(message)
print("")
print("\n" + str(epoch + 1) + "Epoch Done!")
print(
"\n==================== Evaluation ======================"
)
model.eval()
val_loss = AverageMeter()
val_start_acc = AverageMeter()
val_end_acc = AverageMeter()
val_span_acc = BooleanAccuracy()
val_squad_em_f1 = SquadEmAndF1()
original_param = flatten_params(model)
model = load_params(model, avg_params)
for j, data in enumerate(tqdm(dev_loader)):
passage, query = data["passage"], data["query"]
char_passage, char_query = data["char_passage"], data["char_query"]
passage_mask, query_mask = torch.clamp(passage, 0,
1).long(), torch.clamp(
query, 0, 1).long()
sent_start, sent_end, sent_mask = data["sent_start"], data[
"sent_end"], data["sent_mask"]
dev_output = model(passage,
query,
char_paragraph=char_passage,
char_query=char_query,
dm=passage_mask,
qm=query_mask,
sent_start=sent_start,
sent_end=sent_end,
sent_mask=sent_mask)
_, val_pred1 = dev_output["span_start_probs"].data.cpu().topk(1)
_, val_pred2 = dev_output["span_end_probs"].data.cpu().topk(1)
loss1 = criterion(dev_output["span_start_log"], data["span_start"])
loss2 = criterion(dev_output["span_end_log"], data["span_end"])
loss = loss1 + loss2
val_loss.update(loss.data[0])
batch_size = dev_output["span_start_probs"].size(0)
ground_truths = torch.stack([data["span_start"], data["span_end"]],
-1)
best_spans = get_best_span(dev_output["span_start_logits"],
dev_output["span_end_logits"])
val_span_acc(best_spans.data, ground_truths.data)
best_predict_strings = get_best_predict_string(
batch_size, best_spans, data["original_passage"],
data["token_offset"])
em_match = []
for b_i in range(batch_size):
em_match.append(
val_squad_em_f1(best_predict_strings[b_i],
data["answer_texts"][b_i]))
val_start_acc.update(
index_accuracy(val_pred1.numpy(),
tensor2np(data["span_start"])))
val_end_acc.update(
index_accuracy(val_pred2.numpy(), tensor2np(data["span_end"])))
model = load_params(model, original_param)
em, f1 = val_squad_em_f1.get_metric(reset=True)
_span_acc = val_span_acc.get_metric(reset=True)
print("")
message = "val_start: %1.5f val_end: %1.5f val loss: %1.5f span acc: %1.5f em: %1.5f f1: %1.5f" % \
(val_start_acc.avg, val_end_acc.avg, val_loss.avg, _span_acc, em, f1)
_logger.info(message)
print("")
from math import ceil
import random
from pre_train_autoencoder import pre_train_autoencoder
from pre_train_classifier import pre_train_classifier
import optim
import model
sys.path.append('./hyperparams/')
if sys.argv[1] == 'glove':
from hyperparams_glove import Hyperparams as hp
elif sys.argv[1] == 'gn':
from hyperparams_gn_glove import Hyperparams as hp
if hp.gpu:
cuda.set_device(hp.gpu)
cuda.manual_seed_all(hp.seed)
torch.manual_seed(hp.seed)
random.seed(hp.seed)
np.random.seed(hp.seed)
def shuffle_data(words, tags=None, limit_size=None, sampling=None):
perm = torch.randperm(len(words))
words = [words[idx.item()] for idx in perm]
if tags != None:
tags = [tags[idx.item()] for idx in perm]
if sampling == 'under_sampling':
words = words[:limit_size]
if tags != None:
tags = tags[:limit_size]
elif sampling == 'over_sampling':
def train(self,
model,
train_dataloader,
validation_dataloader,
optimizer,
scheduler,
epochs,
gpu_device):
'''
Below is our training loop. There's a lot going on, but fundamentally
for each pass in our loop we have a trianing phase and a validation phase.
**Training:**
- Unpack our data inputs and labels
- Load data onto the GPU for acceleration
- Clear out the gradients calculated in the previous pass.
- In pytorch the gradients accumulate by default
(useful for things like RNNs) unless you explicitly clear them out.
- Forward pass (feed input data through the network)
- Backward pass (backpropagation)
- Tell the network to update parameters with optimizer.step()
- Track variables for monitoring progress
**Evalution:**
- Unpack our data inputs and labels
- Load data onto the GPU for acceleration
- Forward pass (feed input data through the network)
- Compute loss on our validation data and track variables for monitoring progress
Pytorch hides all of the detailed calculations from us,
but we've commented the code to point out which of the
above steps are happening on each line.
PyTorch also has some
https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py) which you may also find helpful.*
'''
datetime.datetime.now().isoformat()
# This training code is based on the `run_glue.py` script here:
# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128
# Set the seed value all over the place to make this reproducible.
seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
# From torch:
cuda.manual_seed_all(seed_val)
# We'll store a number of quantities such as training and validation loss,
# validation accuracy, and timings.
training_stats = []
# Measure the total training time for the whole run.
total_t0 = time.time()
# For each epoch...
for epoch_i in range(0, epochs):
# ========================================
# Training
# ========================================
# Perform one full pass over the training set.
print("")
print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
# Measure how long the training epoch takes.
t0 = time.time()
# Reset the total loss for this epoch.
total_train_loss = 0
# Put the model into training mode. Don't be mislead--the call to
# `train` just changes the *mode*, it doesn't *perform* the training.
# `dropout` and `batchnorm` layers behave differently during training
# vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)
model.train()
# For each batch of training data...
for step, batch in enumerate(train_dataloader):
# Progress update every 40 batches.
if step % 40 == 0 and not step == 0:
# Calculate elapsed time in minutes.
elapsed = self.format_time(time.time() - t0)
# Report progress.
print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
# Unpack this training batch from our dataloader.
#
# As we unpack the batch, we'll also copy each tensor to the GPU using the
# `to` method.
#
# `batch` contains three pytorch tensors:
# [0]: input ids
# [1]: attention masks
# [2]: labels
b_input_ids = batch[0].to(gpu_device)
b_input_mask = batch[1].to(gpu_device)
b_labels = batch[2].to(gpu_device)
# Always clear any previously calculated gradients before performing a
# backward pass. PyTorch doesn't do this automatically because
# accumulating the gradients is "convenient while training RNNs".
# (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)
model.zero_grad()
# Perform a forward pass (evaluate the model on this training batch).
# The documentation for this `model` function is here:
# https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
# It returns different numbers of parameters depending on what arguments
# arge given and what flags are set. For our useage here, it returns
# the loss (because we provided labels) and the "logits"--the model
# outputs prior to activation.
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
# Accumulate the training loss over all of the batches so that we can
# calculate the average loss at the end. `loss` is a Tensor containing a
# single value; the `.item()` function just returns the Python value
# from the tensor.
total_train_loss += loss.item()
# Perform a backward pass to calculate the gradients.
loss.backward()
# Clip the norm of the gradients to 1.0.
# This is to help prevent the "exploding gradients" problem.
# From torch:
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
# Update parameters and take a step using the computed gradient.
# The optimizer dictates the "update rule"--how the parameters are
# modified based on their gradients, the learning rate, etc.
optimizer.step()
# Update the learning rate.
scheduler.step()
# Calculate the average loss over all of the batches.
avg_train_loss = total_train_loss / len(train_dataloader)
# Measure how long this epoch took.
training_time = self.format_time(time.time() - t0)
print("")
print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
# ========================================
# Validation
# ========================================
# After the completion of each training epoch, measure our performance on
# our validation set.
print("")
print("Running Validation...")
t0 = time.time()
# Put the model in evaluation mode--the dropout layers behave differently
# during evaluation.
model.eval()
# Tracking variables
total_eval_accuracy = 0
total_eval_loss = 0
#nb_eval_steps = 0
# Evaluate data for one epoch
for batch in validation_dataloader:
# Unpack this training batch from our dataloader.
#
# As we unpack the batch, we'll also copy each tensor to the GPU using
# the `to` method.
#
# `batch` contains three pytorch tensors:
# [0]: input ids
# [1]: attention masks
# [2]: labels
b_input_ids = batch[0].to(gpu_device)
b_input_mask = batch[1].to(gpu_device)
b_labels = batch[2].to(gpu_device)
# Tell pytorch not to bother with constructing the compute graph during
# the forward pass, since this is only needed for backprop (training).
with torch.no_grad():
# Forward pass, calculate logit predictions.
# token_type_ids is the same as the "segment ids", which
# differentiates sentence 1 and 2 in 2-sentence tasks.
# The documentation for this `model` function is here:
# https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
# Get the "logits" output by the model. The "logits" are the output
# values prior to applying an activation function like the softmax.
(loss, logits) = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
# Accumulate the validation loss.
total_eval_loss += loss.item()
# Move logits and labels to CPU
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu').numpy()
# Calculate the accuracy for this batch of test sentences, and
# accumulate it over all batches.
total_eval_accuracy += self.flat_accuracy(logits, label_ids)
# Report the final accuracy for this validation run.
avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
# Calculate the average loss over all of the batches.
avg_val_loss = total_eval_loss / len(validation_dataloader)
# Measure how long the validation run took.
validation_time = self.format_time(time.time() - t0)
print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
# Record all statistics from this epoch.
training_stats.append(
{
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': avg_val_loss,
'Valid. Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time
}
)
print("")
print("Training complete!")
print("Total training took {:} (h:mm:ss)".format(self.format_time(time.time()-total_t0)))
datetime.datetime.now().isoformat()
def main(args):
config = json.load(open(args.config))
if args.gpu >= 0:
device = 'cuda:{}'.format(args.gpu)
cuda.manual_seed_all(args.seed)
else:
device = 'cpu'
logger.info(f'Device: {device}')
torch.manual_seed(args.seed)
random.seed(args.seed)
valid_size = config['valid_size']
pre_valid_size = config['pre_valid_size']
batch_size = config['batch_size']
pre_batch_size = config['pre_batch_size']
emb_size = config['emb_size']
hidden_size = config['hidden_size']
dropout_rate = config['dropout_rate']
optimizer = config['optimizer']
lr = config['lr']
lr_decay = config['lr_decay']
max_grad_norm = config['max_grad_norm']
save_prefix = Path(args.save_prefix)
save_binary = args.save_binary
logger.info('Loading word embedding')
if args.embedding.endswith('bin'):
binary = True
else:
binary = False
emb = KeyedVectors.load_word2vec_format(args.embedding, binary=binary)
logger.info('Loading dictionary')
dictionary = json.load(open(args.dictionary))
emb_words = [word for word in emb.vocab]
words = [word for word in emb_words if word in dictionary]
dictionary = preprocess_dictionary(dictionary, words, args)
words = [word for word in emb_words if word in dictionary]
random.shuffle(emb_words)
random.shuffle(words)
logger.info('Creating dataloader')
pre_train_words = emb_words[pre_valid_size:]
pre_valid_words = emb_words[:pre_valid_size]
logger.info(f'Pre train data size:{len(pre_train_words)}')
logger.info(f'Pre valid data size:{len(pre_valid_words)}')
train_words, valid_words = words[valid_size:], words[:valid_size]
logger.info(f'Train data size:{len(train_words)}')
logger.info(f'Valid data size:{len(valid_words)}')
pre_train_dataloader = DataLoader(EmbDataset(pre_train_words, emb),
batch_size=pre_batch_size,
shuffle=True)
pre_valid_dataloader = DataLoader(EmbDataset(pre_valid_words, emb),
batch_size=pre_batch_size,
shuffle=False)
train_dataloader = DataLoader(EmbDictDataset(train_words, emb, dictionary),
batch_size=batch_size,
shuffle=True)
valid_dataloader = DataLoader(EmbDictDataset(valid_words, emb, dictionary),
batch_size=batch_size,
shuffle=False)
logger.info('Creating models')
encoder = model.Encoder(emb_size, hidden_size, dropout_rate)
dictionary_decoder = model.Decoder(emb_size, hidden_size, dropout_rate)
corpus_decoder = model.Decoder(emb_size, hidden_size, dropout_rate)
encoder = encoder.to(device)
dictionary_decoder = dictionary_decoder.to(device)
corpus_decoder = corpus_decoder.to(device)
encoder_optim = make_optim(encoder, optimizer, lr, lr_decay, max_grad_norm)
dictionary_decoder_optim = make_optim(dictionary_decoder, optimizer, lr,
lr_decay, max_grad_norm)
corpus_decoder_optim = make_optim(corpus_decoder, optimizer, lr, lr_decay,
max_grad_norm)
criterion = nn.MSELoss()
trainer = Trainer(args, config, encoder, dictionary_decoder,
corpus_decoder, encoder_optim, dictionary_decoder_optim,
corpus_decoder_optim, criterion, device, save_prefix)
logger.info('Pre-training dictionary & corpus decoder')
trainer.run(pre_train_dataloader, pre_valid_dataloader, 'pre-train')
logger.info('Debiasing word embedding')
emb_dict = debiasing_emb(emb, trainer, device, batch_size)
logger.info('Saving word embedding')
save_word2vec_format(emb_dict, save_prefix, 'autoenc_emb.bin')
logger.info('Training adversarial')
trainer.run(train_dataloader, valid_dataloader, 'adversarial')
logger.info('Debiasing word embedding')
emb_dict = debiasing_emb(emb, trainer, device, batch_size)
logger.info('Saving word embedding')
save_word2vec_format(emb_dict, save_prefix, 'debiasing_emb.bin')
def prepare_seed(rand_seed):
random.seed(rand_seed)
np.random.seed(rand_seed)
torch.manual_seed(rand_seed)
cuda.manual_seed(rand_seed)
cuda.manual_seed_all(rand_seed)
def main():
# Control the random seeds
torch.manual_seed(SEED)
cuda.manual_seed(SEED)
cuda.manual_seed_all(SEED)
np.random.seed(SEED) # Numpy module.
random.seed(SEED) # Python random module.
backends.cudnn.benchmark = False
backends.cudnn.deterministic = True
print(">> Set random seed: {}".format(SEED))
# Write data filenames and labels to a txt file.
read_filenames_and_labels_to_txt(CANDIDATE_ROOT, "gt.txt")
# Conduct data augmentation first
create_patches(CANDIDATE_ROOT, PATCH_ROOT)
# Get the size of the unlabeled data pool to build a list of indices
indices = list(range(get_sample_num(PATCH_ROOT)))
# Randomly select K samples in the first cycle
random.shuffle(indices)
labeled_indices = indices[:K]
unlabeled_indices = indices[K:]
# Load training and testing data
filenames, labels = load_train_data(CANDIDATE_ROOT, PATCH_ROOT,
labeled_indices)
train_dataset = MyDataset(filenames, labels, transform=image_transform)
train_loader = DataLoader(train_dataset,
batch_size=BATCH,
shuffle=True,
pin_memory=True)
print("Current training dataset size: {}".format(len(train_dataset)))
filenames, labels = load_test_data(TEST_ROOT)
test_dataset = MyDataset(filenames, labels, transform=image_transform)
test_loader = DataLoader(test_dataset,
batch_size=BATCH,
sampler=SequentialSampler(range(
len(test_dataset))),
pin_memory=True)
dataloaders = {'train': train_loader, 'test': test_loader}
# Set the device for running the network
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Build the network structure
classifier_network = ResNet18(num_classes=23)
classifier_network.to(device)
loss_network = LossNet()
loss_network.to(device)
# Load pre-trained weight of the classifier network
classifier_dict = classifier_network.state_dict()
pretrained_dict = torch.load("resnet18.pth")
parameter_dict = {
k: v
for k, v in pretrained_dict.items() if k in classifier_dict
}
classifier_dict.update(parameter_dict)
classifier_network.load_state_dict(classifier_dict)
# Integration
model = {'classifier': classifier_network, 'module': loss_network}
# Set the loss criterion of the training procedure
criterion = nn.CrossEntropyLoss(reduction='none')
print(">> Start active learning!")
for cycle in range(CYCLES):
# for each cycle, we need new optimizers and learning rate schedulers
optim_classifier = optim.SGD(model['classifier'].parameters(),
lr=LR_classifier,
momentum=MOMENTUM,
weight_decay=WDECAY)
optim_loss = optim.SGD(model['module'].parameters(),
lr=LR_loss,
momentum=MOMENTUM,
weight_decay=WDECAY)
optimizers = {'classifier': optim_classifier, 'loss': optim_loss}
scheduler_classifier = lr_scheduler.MultiStepLR(optim_classifier,
milestones=MILESTONE)
scheduler_loss = lr_scheduler.MultiStepLR(optim_loss,
milestones=MILESTONE)
schedulers = {
'classifier': scheduler_classifier,
'module': scheduler_loss
}
# Training
train(model, criterion, optimizers, schedulers, dataloaders, EPOCH,
device)
acc = test(model, dataloaders, device, mode='test')
print('Cycle {}/{} || Label set size {}: Test acc {}'.format(
cycle + 1, CYCLES, len(labeled_indices), acc))
# Random subset sampling to explore the data pool
random.shuffle(unlabeled_indices)
subset_indices = unlabeled_indices[:SUBSET]
# Choose the active learning strategy
selected_indices = active_sampling(strategy="hybrid",
model=model,
indices=subset_indices)
# Add new labeled samples to the labeled dataset
labeled_indices.extend(selected_indices)
# Remove labeled samples from the unlabeled data pool
for i in selected_indices:
unlabeled_indices.remove(i)
# Update the training dataset
filenames, labels = load_train_data(CANDIDATE_ROOT, PATCH_ROOT,
labeled_indices)
train_dataset = MyDataset(filenames, labels, transform=image_transform)
print("Training data number: ", len(train_dataset))
dataloaders['train'] = DataLoader(train_dataset,
batch_size=BATCH,
pin_memory=True,
shuffle=True)
# Save the model of the current cycle
torch.save(model["classifier"].state_dict(),
'checkpoints/active_resnet18_cycle{}.pth'.format(cycle))
