def create_models(num_networks, hyperparameter_ranges_file):
identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
data_is_target_list = [0]
num_scat_list = [1, 2, 3]
batch_size_list = [32]
# data_noise_gaussian_list = [0, 1]
data_noise_gaussian_list = [1] # Decided on 11/22/2018 b/c better models
#dropout_input_list = [0, 0.1, 0.2]
#dropout_list = [0, 0.1, 0.2, 0.3, 0.4, 0.5]
weight_decay_list = [0]
for count in range(num_networks):
data_is_target = choice(data_is_target_list)
n_scat = choice(num_scat_list)
bs = choice(batch_size_list)
data_noise_gaussian = choice(data_noise_gaussian_list)
#dropout_input = choice(dropout_input_list)
weight_decay = choice(weight_decay_list)
# get params
model_params = choose_hyperparameters_from_file(
hyperparameter_ranges_file)
# set other params
model_params['data_is_target'] = data_is_target
home = os.path.expanduser('~')
model_params['data_train'] = os.path.join(
home, 'Downloads', '20180402_L74_70mm',
'train_' + str(n_scat) + '.h5')
model_params['data_val'] = os.path.join(home, 'Downloads',
'20180402_L74_70mm',
'val_' + str(n_scat) + '.h5')
model_params['batch_size'] = bs
model_params['data_noise_gaussian'] = data_noise_gaussian
#model_params['dropout_input'] = dropout_input
model_params['weight_decay'] = weight_decay
model_params['patience'] = 20
model_params['cuda'] = 1
model_params['save_initial'] = 0
k_list = [3, 4, 5]
for k in k_list:
model_params['k'] = k
model_params['save_dir'] = os.path.join(
'DNNs', identifier + '_' + str(count + 1) + '_created',
'k_' + str(k))
# print(model_params['save_dir'])
ensure_dir(model_params['save_dir'])
save_model_params(
os.path.join(model_params['save_dir'], model_params_fname),
model_params)
print('create_models: created model {}_{}'.format(identifier, count))
return identifier
def get_and_save_model_dict(kernel_width, num_kernels, num_layers, index):
model_dict = get_model_dict(kernel_width, num_kernels, num_layers)
for k in [3, 4, 5]:
model_dict['k'] = k
identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
model_dict['save_dir'] = os_path_join(
'DNNs', 'fcnn_v1.6.7_{}_{}_created'.format(identifier, index),
'k_{}'.format(k))
# print(model_params['save_dir'])
ensure_dir(model_dict['save_dir'])
save_model_params(
os_path_join(model_dict['save_dir'], MODEL_PARAMS_FNAME),
model_dict)
print('created ', model_dict['save_dir'])
def create_models(num_networks, hyperparameter_ranges_file):
identifier = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
for count in range(num_networks):
model_params = choose_hyperparameters_from_file(hyperparameter_ranges_file)
model_params['train_features'] = os.path.join('data', 'train-images-idx3-ubyte.gz')
model_params['train_labels'] = os.path.join('data', 'train-labels-idx3-ubyte.gz')
model_params['valid_features'] = os.path.join('data', 't10k-images-idx3-ubyte.gz')
model_params['valid_labels'] = os.path.join('data', 't10k-labels-idx3-ubyte.gz')
model_params['save_dir'] = os.path.join(trained_models_folder, identifier + '_' + str(count+1) + '_created')
ensure_dir(model_params['save_dir'])
save_model_params(os.path.join(model_params['save_dir'], 'model_params.txt'), model_params)
return identifier
def train_one(model_folder):
new_model_folder_name = model_folder.replace('_created', '_training')
os_rename(model_folder, new_model_folder_name)
frequencies = glob(os_path_join(new_model_folder_name, 'k_*'))
for frequency in frequencies:
# Load model
print('train.py: training {}'.format(frequency))
model_params_path = os_path_join(frequency, model_params_fname)
# create model
model, model_params = get_which_model_from_params_fname(
model_params_path, return_params=True)
if 'cuda' in model_params:
using_cuda = model_params['cuda'] and torch_cuda_is_available()
else:
using_cuda = torch_cuda_is_available()
if using_cuda is True:
model.cuda()
# save initial weights
if 'save_initial' in model_params and model_params[
'save_initial'] and model_params['save_dir']:
suffix = '_initial'
path = add_suffix_to_path(model_params_fname['save_dir'], suffix) # pylint: disable=E1126
ensure_dir(path)
torch_save(model.state_dict(),
os_path_join(path, MODEL_DATA_FNAME))
save_model_params(os_path_join(path, model_params_fname),
model_params)
# loss
if 'cost_function' in model_params:
loss = model_params['cost_function']
elif 'loss_function' in model_params:
loss = model_params['loss_function']
else:
raise ValueError(
'model_params missing key cost_function or loss_function')
if loss not in ['MSE', 'L1', 'SmoothL1']:
raise TypeError('Error must be MSE, L1, or SmoothL1. You gave ' +
str(loss))
if loss == 'MSE':
from torch.nn import MSELoss
loss = MSELoss()
elif loss == 'L1':
from torch.nn import L1Loss
loss = L1Loss()
elif loss == 'SmoothL1':
from torch.nn import SmoothL1Loss
loss = SmoothL1Loss()
# optimizer
if model_params['optimizer'] == 'Adam':
from torch.optim import Adam
optimizer = Adam(model.parameters(),
lr=model_params['learning_rate'],
weight_decay=model_params['weight_decay'])
elif model_params['optimizer'] == 'SGD':
from torch.optim import SGD
optimizer = SGD(model.parameters(),
lr=model_params['learning_rate'],
momentum=model_params['momentum'],
weight_decay=model_params['weight_decay'])
else:
raise ValueError(
'model_params[\'optimizer\'] must be either Adam or SGD. Got '
+ model_params['optimizer'])
logger = Logger()
# Load training, validation, and test data
# Load primary training data
dat_train = ApertureDataset(
model_params['data_train'],
NUM_SAMPLES_TRAIN,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_train = DataLoader(dat_train,
batch_size=model_params['batch_size'],
shuffle=True,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
# Load secondary training data - used to evaluate training loss after every epoch
dat_train2 = ApertureDataset(
model_params['data_train'],
NUM_SAMPLES_TRAIN_EVAL,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_train_eval = DataLoader(dat_train2,
batch_size=model_params['batch_size'],
shuffle=False,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
# Load validation data - used to evaluate validation loss after every epoch
dat_val = ApertureDataset(
model_params['data_val'],
NUM_SAMPLES_VALID,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_val = DataLoader(dat_val,
batch_size=model_params['batch_size'],
shuffle=False,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
trainer = Trainer(
model=model,
loss=loss,
optimizer=optimizer,
patience=model_params['patience'],
loader_train=loader_train,
loader_train_eval=loader_train_eval,
loader_val=loader_val,
cuda=using_cuda,
logger=logger,
data_noise_gaussian=model_params['data_noise_gaussian'],
save_dir=frequency)
# run training
trainer.train()
os_rename(new_model_folder_name,
new_model_folder_name.replace('_training', '_trained'))