def train(scriptArgs):
# load args from file
args = loadCheckpoint(scriptArgs.json,
map_location=lambda storage, loc: storage.cuda())
# terminate if validAcc exists
_validAccKey = TrainWeights.validAccKey
if hasattr(args, _validAccKey):
print('[{}] exists'.format(_validAccKey))
exit(0)
# no need to save model random weights
args.saveRandomWeights = False
# update args parameters
args.seed = datetime.now().microsecond
# update cudnn parameters
random.seed(args.seed)
set_device(scriptArgs.gpu[0])
cudnn.benchmark = True
torch_manual_seed(args.seed)
cudnn.enabled = True
cuda_manual_seed(args.seed)
# copy scriptArgs values to args
for k, v in vars(scriptArgs).items():
setattr(args, k, v)
# load model flops
_modelFlopsPathKey = BaseNet.modelFlopsPathKey()
modelFlopsPath = getattr(args, _modelFlopsPathKey)
if modelFlopsPath and exists(modelFlopsPath):
setattr(args, BaseNet.modelFlopsKey(), loadCheckpoint(modelFlopsPath))
folderNotExists = not exists(args.save)
if folderNotExists:
create_exp_dir(args.save)
# init logger
logger = HtmlLogger(args.save, 'log')
if scriptArgs.individual:
args.width = []
alphasRegime = OptimalRegime(args, logger)
# train according to chosen regime
alphasRegime.train()
return folderNotExists
def make_results_reproducible(model_is_convolutional: bool = False) -> None:
"""
Make the subsequent instructions produce purely deterministic outputs
by fixing all the relevant seeds:
"""
random_seed(0)
_ = numpy_seed(0)
_ = torch_manual_seed(0)
# since GPU computations may introduce additional stochasticity with
# their convolutional operation optimizations:
if model_is_convolutional:
if cuda_is_available():
# disabling benchmarking and choosing among convolution operation
# implementation alternatives, which is stochastic based due to
# noise and hardware:
cudnn.benchmark = False
# ansuring deterministic algorithms for colvolutional operations
# are employed:
cudnn.deterministic = True
def seed_everything(seed=1234):
random_seed(seed)
torch_manual_seed(seed)
print('no gpu device available')
exit(1)
# update GPUs list
if type(args.gpu) is not 'None':
args.gpu = [int(i) for i in args.gpu.split(',')]
args.device = 'cuda:' + str(args.gpu[0])
# CUDA
if args.seed is None:
args.seed = random.randint(1, 10000)
random.seed(args.seed)
set_device(args.gpu[0])
cudnn.benchmark = True
torch_manual_seed(args.seed)
cudnn.enabled = True
cuda_manual_seed(args.seed)
# create folder
baseFolder = dirname(abspath(getfile(currentframe())))
args.time = time.strftime("%Y%m%d-%H%M%S")
args.folderName = '{}_{}_{}_{}'.format(args.model, args.actBitwidth,
args.weightBitwidth, args.time)
args.save = '{}/results/{}'.format(baseFolder, args.folderName)
if not os.path.exists(args.save):
os.makedirs(args.save)
# save args to JSON
saveArgsToJSON(args)
def G(scriptArgs):
# load args from file
args = loadCheckpoint(scriptArgs.json,
map_location=lambda storage, loc: storage.cuda())
# # ========================== DEBUG ===============================
# print(args)
# # extract args JSON folder path
# folderName = path.dirname(scriptArgs.json)
# # results folder is JSON filename
# jsonFileName = path.basename(scriptArgs.json)
# # set results folder path
# args.save = '{}/{}'.format(folderName, jsonFileName[:-5])
# print('====')
# print(args.save)
# create_exp_dir(args.save)
# # init logger
# logger = HtmlLogger(args.save, 'log')
# # init project base folder
# baseFolder = path.dirname(path.abspath(getfile(currentframe()))) # script directory
# # set pre-trained path
# preTrainedKey = '[(32, 32)],[{}]'.format(args.model)
# preTrainedFileName = 'model.updated_stats.pth.tar'
# args.pre_trained = '{}/../pre_trained/{}/train_portion_1.0/{}/train/{}'.format(baseFolder, args.dataset, preTrainedKey, preTrainedFileName)
# print(args.pre_trained)
#
# # build model for uniform distribution of bits
# from cnn.utils import models
# modelClass = models.__dict__[args.model]
# # init model
# model = modelClass(args)
# model = model.cuda()
# # load pre-trained full-precision model
# args.loadedOpsWithDiffWeights = model.loadPreTrained(args.pre_trained, logger, args.gpu[0])
#
# print(args)
# setattr(args, 'Validation acc', 33.4)
# setattr(args, 'Validation loss', 1.347)
#
# saveCheckpoint(args, scriptArgs.json)
# # move checkpoint to finished jobs folder
# newDestination = scriptArgs.json
# if scriptArgs.dstFolder is not None:
# newDestination = '{}/{}'.format(scriptArgs.dstFolder, jsonFileName)
# rename(scriptArgs.json, newDestination)
#
# print(args)
# print('DDone !')
# from time import sleep
# sleep(60)
# exit(0)
# # ================================================================
args.seed = datetime.now().microsecond
# update cudnn parameters
random.seed(args.seed)
set_device(scriptArgs.gpu[0])
cudnn.benchmark = True
torch_manual_seed(args.seed)
cudnn.enabled = True
cuda_manual_seed(args.seed)
# update values
args.train_portion = 1.0
args.batch_size = 250
args.gpu = scriptArgs.gpu
args.data = scriptArgs.data
# extract args JSON folder path
folderName = path.dirname(scriptArgs.json)
# results folder is JSON filename
jsonFileName = path.basename(scriptArgs.json)
# set results folder path
args.save = '{}/{}'.format(folderName, jsonFileName[:-5])
if not path.exists(args.save):
create_exp_dir(args.save)
# init logger
logger = HtmlLogger(args.save, 'log')
# log command line
logger.addInfoTable(title='Command line',
rows=[[' '.join(argv)],
['PID:[{}]'.format(getpid())],
['Hostname', gethostname()],
[
'CUDA_VISIBLE_DEVICES',
environ.get('CUDA_VISIBLE_DEVICES')
]])
# init project base folder
baseFolder = path.dirname(path.abspath(getfile(
currentframe()))) # script directory
# set pre-trained path
preTrainedKey = '[(32, 32)],[{}]'.format(args.model)
preTrainedFileName = 'model.updated_stats.pth.tar'
args.pre_trained = '{}/../pre_trained/{}/train_portion_1.0/{}/{}'.format(
baseFolder, args.dataset, preTrainedKey, preTrainedFileName)
# args.pre_trained = scriptArgs.pp
# check path exists
if path.exists(args.pre_trained):
alphasRegime = OptimalModel(args, logger)
# train according to chosen regime
alphasRegime.train()
# best_prec1, best_valid_loss are now part of args, therefore we have to save args again, the sender will be able to read these values
saveCheckpoint(args, scriptArgs.json)
# move checkpoint to finished jobs folder
newDestination = scriptArgs.json
if scriptArgs.dstFolder is not None:
newDestination = '{}/{}'.format(scriptArgs.dstFolder,
jsonFileName)
rename(scriptArgs.json, newDestination)
print(args)
logger.addInfoToDataTable(
'Saved args to [{}]'.format(newDestination))
logger.addInfoToDataTable('Done !')