def trainModel_Exp(modelBlock, resultBlock, n_epochs, log_file, result_file,
model_file):
# Function TRAIN_MODEL
# Trains all models in modelList for n_epochs
# Parameters:
# * modelBlock: Nested dictionary of models
# * n_epochs: Number of epochs for which to train
# * N: size of image to generate
# Parameters to add
# * exp flag that tells whehter or not we are hyperopt or running experiments
# * resultBlock
# * Number of previously executed epochs
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s:%(name)s]:%(message)s')
if not len(logger.handlers):
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
epochs_trained = modelBlock["Meta"]["Epochs_Trained"]
epochs_total = epochs_trained + n_epochs
print(epochs_total)
for epoch in range(n_epochs):
epoch_real = epoch + epochs_trained
# Generate training samples and iterate through all models in modelList
print('Starting epoch %d / %d' % (epoch_real + 1, epochs_total))
sampleDict = generateSamples(modelBlock["Meta"]["N"], 20000,
modelBlock["Meta"]["Layers"])
for key, val in modelBlock.items():
if (key != "Meta"):
runEpoch(modelBlock[key]["Model"],
modelBlock["Meta"]["Loss_Function"],
modelBlock[key]["Optimizer"],
modelBlock["Meta"]["Type"], modelBlock[key]["Batch"],
sampleDict)
print('Finishing epoch %d / %d' % (epoch_real + 1, epochs_total))
# Want to record test error if the total number of epochs is a multiple 50 or this is the final epoch
if (((epoch_real % 10) == 0) or (epoch == (n_epochs - 1))):
# Every 50 epochs, evaluate the performance of all the models and print summary statistics
testDict = generateSamples(modelBlock["Meta"]["N"], 40000,
modelBlock["Meta"]["Layers"])
print('')
logger.info('Finishing epoch %d / %d' %
(epoch_real + 1, epochs_total))
loss = []
accAll = []
accPath = []
accDistract = []
for key, val in modelBlock.items():
if (key != "Meta"):
model_accAll, model_accPath, model_accDistract, model_loss = checkAccuracy(
modelBlock[key]["Model"],
modelBlock["Meta"]["Loss_Function"],
modelBlock["Meta"]["Type"], modelBlock[key]["Batch"],
testDict)
modelBlock[key]["Loss"] = model_loss
modelBlock[key]["Acc_All"] = model_accAll
modelBlock[key]["Acc_Path"] = model_accPath
modelBlock[key]["Acc_Distract"] = model_accDistract
resultBlock[key][epoch_real] = {
"Loss": model_loss,
"Acc_All": model_accAll,
"Acc_Path": model_accPath,
"Acc_Distract": model_accDistract
}
loss.append(model_loss)
accAll.append(model_accAll)
accPath.append(model_accPath)
accDistract.append(model_accDistract)
loss_array = np.asarray(loss)
accAll_array = np.asarray(accAll)
accPath_array = np.asarray(accPath)
accDistract_array = np.asarray(accDistract)
print(loss_array)
logger.info('[Loss] Mean:%.6f, Median:%.6f, Best:%.6f' %
(np.mean(loss_array), np.median(loss_array),
np.min(loss_array)))
logger.info(
'[Accuracy (All pixels)] Mean:%.6f, Median:%.6f, Best:%.6f ' %
(np.mean(accAll_array), np.median(accAll_array),
np.min(accAll_array)))
logger.info(
'[Accuracy (Edge-Connected Paths)] Mean:%.6f, Median:%.6f, Best:%.6f '
% (np.mean(accPath_array), np.median(accPath_array),
np.min(accPath_array)))
logger.info(
'[Accuracy (Distractors)] Mean:%.6f, Median:%.6f, Best:%.6f ' %
(np.mean(accDistract_array), np.median(accDistract_array),
np.min(accDistract_array)))
logger.info('')
print('')
# Update the total number of epochs trained
modelBlock["Meta"]["Epochs_Trained"] = epoch_real
torch.save(resultBlock, result_file)
modelBlock_State = convertStateDict(modelBlock)
torch.save(modelBlock_State, model_file)
def main(args):
##################################################################
# Top level code for running hyperoptimization
# User specifies model type and layer number
# Code then finds optimal hyperparameters for all
# combinations of models/layers
##################################################################
# Load in arguments
n_models = args.n_models
n_epochs = args.n_epochs
hyp_epochs = args.hyp_epochs
load = args.resume
model_type = args.model
layers = args.layers
image_size = args.image_size
exp_name = args.exp_name
# Make sure the result directory exists. If not create
directory_logs = '../../PredPrey_Results/Decision/Logs'
directory_results = '../../PredPrey_Results/Decision/ResultBlock'
if not os.path.exists(directory_logs):
os.makedirs(directory_logs)
if not os.path.exists(directory_results):
os.makedirs(directory_results)
# Create name for result folders
log_file = '../../PredPrey_Results/Decision/Logs/' + exp_name + '.log'
hyperopt_file = '../../PredPrey_Results/Decision/ResultBlock/hyperparameter_' + exp_name + '.pth.tar'
result_file = '../../PredPrey_Results/Decision/ResultBlock/resultBlock_' + exp_name + '.pth.tar'
model_file = '../../PredPrey_Results/Decision/ResultBlock/modelBlock_' + exp_name + '.pth.tar'
# Initizlize Logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s:%(name)s]:%(message)s')
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
# Print experiment parameters to log
logger.info(
'Training %s models for %d hyperband epochs of %d epochs each.' %
(model_type, hyp_epochs, n_epochs))
logger.info('Initial number of models: %d' % (n_models))
# Setup network parameters
num_nodes = image_size**2
input_size = num_nodes
hidden_size = num_nodes
loss_fn = nn.BCELoss()
dtype = torch.FloatTensor
if args.use_gpu:
print('GPU is used.')
dtype = torch.cuda.FloatTensor
hyperparameter = {}
# Run hyperband epoch
modelBlock, resultBlock = generateDictionary_Hyperopt(
n_models, model_type, layers, input_size, hidden_size, image_size,
loss_fn, dtype)
torch.save(resultBlock, result_file)
modelBlock_State = convertStateDict(modelBlock)
torch.save(modelBlock_State, model_file)
for h_epoch in range(hyp_epochs):
trainModel(modelBlock, n_epochs, log_file)
pruneModel(modelBlock, resultBlock)
torch.save(resultBlock, result_file)
modelBlock_State = convertStateDict(modelBlock)
torch.save(modelBlock_State, model_file)
trainModel(modelBlock, n_epochs, log_file)
epoch_total = modelBlock["Meta"]["Epochs_Trained"]
resultBlock["Meta"]["Total_Epochs"] = epoch_total
# Find the model id with best loss and return its parameters
best_loss = 1000.0
for key, val in modelBlock.items():
if (key != "Meta"):
resultBlock[key][epoch_total] = {
"Loss": modelBlock[key]["Loss"],
"Acc_All": modelBlock[key]["Acc_All"]
}
resultBlock[key]["Hyperparameter"]["Max_Epoch"] = epoch_total
if ((modelBlock[key]["Loss"] < best_loss)):
best_loss = modelBlock[key]["Loss"]
best_key = key
# This ensures that values are returned even if none of the keys have loss >= 1000.0
# This should not happen so print an error to the log
if (best_loss >= 1000.0):
logger.warning('All models had loss greater than 1000.0')
logger.warning('Returning parameters for first remaining model')
keys = list(modelBlock.keys())
keys.remove("Meta")
best_key = next(iter(keys))
lr = modelBlock[best_key]["Learning"]
batch_size = modelBlock[best_key]["Batch"]
weight_decay = modelBlock[best_key]["Weight_Decay"]
acc = modelBlock[best_key]["Accuracy"]
avg_loss = modelBlock[best_key]["Loss"]
resultBlock["Meta"]["Learning"] = modelBlock[best_key]["Learning"]
resultBlock["Meta"]["Batch"] = modelBlock[best_key]["Batch"]
resultBlock["Meta"]["Weight_Decay"] = modelBlock[best_key]["Weight_Decay"]
resultBlock["Meta"]["Acc_All"] = modelBlock[best_key]["Acc_All"]
resultBlock["Meta"]["Loss"] = modelBlock[best_key]["Loss"]
resultBlock["Meta"]["Best_Key"] = best_key
torch.save(resultBlock, result_file)
modelBlock_State = convertStateDict(modelBlock)
torch.save(modelBlock_State, model_file)
if (not (model_type in hyperparameter)):
hyperparameter[model_type] = {}
if (not (layers in hyperparameter[model_type])):
hyperparameter[model_type][layers] = {}
hyperparameter[model_type][layers]["Learning"] = lr
hyperparameter[model_type][layers]["Batch"] = batch_size
hyperparameter[model_type][layers]["Weight_Decay"] = weight_decay
hyperparameter[model_type][layers]["Acc"] = acc
hyperparameter[model_type][layers]["Loss"] = avg_loss
torch.save(resultBlock, result_file)
torch.save(hyperparameter, hyperopt_file)
def main(args):
##################################################################
# Top level code for running hyperoptimization
# User specifies model type and layer number
# Code then finds optimal hyperparameters for all
# combinations of models/layers
##################################################################
# Load in arguments
n_models = args.n_models
n_epochs = args.n_epochs
load_experiment = args.resume
load_result = args.resume_result
hyper_path = args.hyper
model_type = args.model
layers = args.layers
image_size = args.image_size
lr = args.lr
exp_name = args.exp_name
# Make sure the result directory exists. If not create
directory_logs = '../../PredPrey_Results/Propagation/Logs'
directory_results = '../../PredPrey_Results/Propagation/ResultBlock'
if not os.path.exists(directory_logs):
os.makedirs(directory_logs)
if not os.path.exists(directory_results):
os.makedirs(directory_results)
# Create name for result folders
log_file = '../../PredPrey_Results/Propagation/Logs/'+ exp_name + '.log'
result_file = '../../PredPrey_Results/Propagation/ResultBlock/resultBlock_' + exp_name + '.pth.tar'
model_file = '../../PredPrey_Results/Propagation/ResultBlock/modelBlock_' + exp_name + '.pth.tar'
# Initizlize Logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s:%(name)s]:%(message)s')
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
# Print experiment parameters to log
logger.info('Training %s models with %i layers for %d epochs.' % (model_type, layers, n_epochs))
logger.info('Number of models: %d' % (n_models))
# Want to change this so that hyperparameter can only be loaded
if os.path.isfile(hyper_path):
print('Loading hyperparameter block.')
hyperparameter = torch.load(hyper_path)
else:
print("=> no hyperparameter block found at '{}'".format(hyper_path))
hyperparameter = {}
hyperparameter[model_type] = {}
hyperparameter[model_type][layers] = {"Learning": lr, "Batch": 32, "Weight_Decay": 0}
# Set up experiment block
num_nodes = image_size**2
loss_fn = nn.MSELoss()
dtype = torch.FloatTensor
if args.use_gpu:
print('GPU is used.')
dtype = torch.cuda.FloatTensor
if ((load_experiment) and os.path.isfile(load_experiment) and os.path.isfile(load_result)):
modelBlock = torch.load(load_experiment)
resultBlock = torch.load(load_result)
else:
print("=> Generating new result block")
modelBlock, resultBlock = generateDictionary_Exp(n_models, model_type, layers, num_nodes, num_nodes,
image_size, loss_fn, dtype, hyperparameter)
# Figure out how many epochs are left to train
epochs_remaining = n_epochs - modelBlock["Meta"]["Epochs_Trained"]
trainModel(modelBlock, epochs_remaining, log_file)
# torch.save(resultBlock, result_file)
modelBlock_State = convertStateDict(modelBlock)
torch.save(modelBlock_State, model_file)