def train(self):
args = self.args
model = self.model
logger = self.logger
epochRange = self._getEpochRange(self.nEpochs)
# init optimizer
optimizer = SGD(model.alphas(),
args.search_learning_rate,
momentum=args.search_momentum,
weight_decay=args.search_weight_decay)
# init scheduler
scheduler = ReduceLROnPlateau(optimizer,
mode='min',
factor=0.95,
patience=args.search_patience,
min_lr=args.search_learning_rate_min)
for epoch in epochRange:
print('========== Epoch:[{}/{}] =============='.format(
epoch, self.nEpochs))
# init epoch train logger
trainLogger = HtmlLogger(self.trainFolderPath, epoch)
# set loggers dictionary
loggersDict = {self.trainLoggerKey: trainLogger}
# create epoch jobs
epochDataRows = self._createEpochJobs(epoch)
# add epoch data rows
for jobDataRow in epochDataRows:
logger.addDataRow(jobDataRow, trType='<tr bgcolor="#2CBDD6">')
# train alphas
# epochLossDict, alphasDataRow = self.trainAlphas(self._getNextSearchQueueDataLoader(), optimizer, epoch, loggersDict)
epochLossDict, alphasDataRow = self.trainAlphas(
self.valid_queue, optimizer, epoch, loggersDict)
# update scheduler
scheduler.step(epochLossDict.get(self.flopsLoss.totalKey()))
# calc model choosePathAlphasAsPartition flops ratio
model.choosePathAlphasAsPartition()
# add values to alphas data row
additionalData = {
self.epochNumKey: epoch,
self.lrKey: optimizer.param_groups[0]['lr'],
self.validFlopsRatioKey: model.flopsRatio()
}
self._applyFormats(additionalData)
# add alphas data row
alphasDataRow.update(additionalData)
logger.addDataRow(alphasDataRow)
# save checkpoint
save_checkpoint(self.trainFolderPath, model, optimizer,
epochLossDict)
def train(self, trainFolderName='init_weights_train'):
args = self.getArgs()
# create train folder
folderPath = '{}/{}'.format(self.getTrainFolderPath(), trainFolderName)
if not exists(folderPath):
makedirs(folderPath)
# init optimizer
optimizer = self._initOptimizer()
# init scheduler
scheduler = ReduceLROnPlateau(optimizer,
mode='min',
factor=0.95,
patience=args.weights_patience,
min_lr=args.learning_rate_min)
epoch = 0
trainLoggerFlag = True
while not self.stopCondition(epoch):
# update epoch number
epoch += 1
# init train logger
trainLogger = None
if trainLoggerFlag:
trainLogger = HtmlLogger(folderPath, epoch)
trainLogger.addInfoTable('Learning rates', [[
'optimizer_lr', self.formats[self.lrKey](
optimizer.param_groups[0]['lr'])
]])
# update train logger condition for next epoch
trainLoggerFlag = ((epoch + 1) % args.logInterval) == 0
# set loggers dictionary
loggersDict = {self.trainLoggerKey: trainLogger}
print('========== Epoch:[{}] =============='.format(epoch))
# train
trainData = self.weightsEpoch(optimizer, epoch, loggersDict)
# validation
validData = self.inferEpoch(epoch, loggersDict)
# update scheduler
scheduler.step(self.schedulerMetric(validData.lossDict()))
self.postEpoch(epoch, optimizer, trainData, validData)
self.postTrain()
def train(scriptArgs):
# load args from file
args = loadCheckpoint(scriptArgs.json,
map_location=lambda storage, loc: storage.cuda())
# terminate if validAcc exists
_validAccKey = TrainWeights.validAccKey
if hasattr(args, _validAccKey):
print('[{}] exists'.format(_validAccKey))
exit(0)
# no need to save model random weights
args.saveRandomWeights = False
# update args parameters
args.seed = datetime.now().microsecond
# update cudnn parameters
random.seed(args.seed)
set_device(scriptArgs.gpu[0])
cudnn.benchmark = True
torch_manual_seed(args.seed)
cudnn.enabled = True
cuda_manual_seed(args.seed)
# copy scriptArgs values to args
for k, v in vars(scriptArgs).items():
setattr(args, k, v)
# load model flops
_modelFlopsPathKey = BaseNet.modelFlopsPathKey()
modelFlopsPath = getattr(args, _modelFlopsPathKey)
if modelFlopsPath and exists(modelFlopsPath):
setattr(args, BaseNet.modelFlopsKey(), loadCheckpoint(modelFlopsPath))
folderNotExists = not exists(args.save)
if folderNotExists:
create_exp_dir(args.save)
# init logger
logger = HtmlLogger(args.save, 'log')
if scriptArgs.individual:
args.width = []
alphasRegime = OptimalRegime(args, logger)
# train according to chosen regime
alphasRegime.train()
return folderNotExists
def printToFile(self, saveFolder):
fileName = 'model'
logger = HtmlLogger(saveFolder, fileName)
if exists(logger.fullPath):
return
logger.setMaxTableCellLength(1000)
layerIdxKey = 'Layer#'
nFiltersKey = 'Filters#'
widthsKey = 'Width'
layerArchKey = 'Layer Architecture'
logger.createDataTable(
'Model architecture',
[layerIdxKey, nFiltersKey, widthsKey, layerArchKey])
for layerIdx, layer in enumerate(self._layers.flops()):
widths = layer.widthList()
dataRow = {
layerIdxKey: layerIdx,
nFiltersKey: layer.outputChannels(),
widthsKey: [widths],
layerArchKey: layer
}
logger.addDataRow(dataRow)
layerIdx += 1
# log additional layers, like Linear, MaxPool2d, AvgPool2d
for layer in self.additionalLayersToLog():
dataRow = {layerIdxKey: layerIdx, layerArchKey: layer}
logger.addDataRow(dataRow)
layerIdx += 1
# log layers alphas distribution
self.logTopAlphas(len(widths),
loggerFuncs=[
lambda k, rows: logger.addInfoTable(
self._alphasDistributionKey, rows)
],
logLayer=True)
# reset table max cell length
logger.resetMaxTableCellLength()
import torch.backends.cudnn as cudnn
from torch.cuda import is_available, set_device
from torch.cuda import manual_seed as cuda_manual_seed
from torch import manual_seed as torch_manual_seed
from trainRegimes.PreTrainedRegime import PreTrainedRegime
from utils.HtmlLogger import HtmlLogger
from utils.emails import sendEmail
from utils.args import parseArgs
if __name__ == '__main__':
# load command line arguments
args = parseArgs()
# init main logger
logger = HtmlLogger(args.save, 'log')
if not is_available():
print('no gpu device available')
exit(1)
args.seed = datetime.now().microsecond
nprandom.seed(args.seed)
set_device(args.gpu[0])
cudnn.benchmark = True
torch_manual_seed(args.seed)
cudnn.enabled = True
cuda_manual_seed(args.seed)
try:
# build regime for alphas optimization