def __init__(self, args, logger, model='top'): # res or top
id_str = str(datetime.datetime.now()).replace(' ',
'_').replace(':', '-')
id_str = '-time_' + id_str.replace('.', '-')
self.model = model
self.model_name = self._parse_args(args)
self.tensorboard_path = os.path.join(args.tb_path,
self.model_name + id_str)
self.logger = logger
self.writer = SummaryWriter(self.tensorboard_path)
if args.pretrained_model_dir == '':
self.save_path = os.path.join(args.save_path,
self.model_name + id_str)
utils.create_save_path(self.save_path, id_str, self.logger)
else:
self.logger.info(
f"Using pretrained path... \nargs.pretrained_model_dir: {args.pretrained_model_dir}"
)
self.save_path = os.path.join(args.save_path,
args.pretrained_model_dir)
self.new_split_type = args.dataset_split_type == 'new'
self.logger.info("** Save path: " + self.save_path)
self.logger.info("** Tensorboard path: " + self.tensorboard_path)
def main():
args = parser.parse_args()
# Use the 24hr datasets!
if args.local_files:
if parameters.LOSS.upper() == "FOCAL":
train_data_path = parameters.LOCAL_FULL_TRAIN
test_data_path = parameters.LOCAL_FULL_TEST
else:
train_data_path = parameters.LOCAL_TRAIN_FILES
test_data_path = parameters.LOCAL_TEST_FILES
else:
if parameters.LOSS.upper() == "FOCAL":
train_data_path = parameters.REMOTE_FULL_TRAIN
test_data_path = parameters.REMOTE_FULL_TEST
else:
train_data_path = parameters.REMOTE_TRAIN_FILES
test_data_path = parameters.REMOTE_TEST_FILES
if parameters.LOSS.upper() != "FOCAL":
train_data_path, _ = create_dataset_path(
train_data_path,
neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.CALL_REPEATS,
shift_windows=parameters.SHIFT_WINDOWS)
test_data_path, _ = create_dataset_path(
test_data_path,
neg_samples=parameters.TEST_NEG_SAMPLES,
call_repeats=1)
else:
if parameters.SHIFT_WINDOWS:
train_data_path += '_OversizeCalls'
train_loader = get_loader_fuzzy(train_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
shift_windows=parameters.SHIFT_WINDOWS,
full_window_predict=True)
test_loader = get_loader_fuzzy(test_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
full_window_predict=True)
# For now model 18 signifies this!!!
save_path = create_save_path(
time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()), args.save_local)
dloaders = {'train': train_loader, 'valid': test_loader}
## Training
model = get_model(parameters.MODEL_ID)
model.to(parameters.device)
print(model)
writer = SummaryWriter(save_path)
writer.add_scalar('batch_size', parameters.BATCH_SIZE)
writer.add_scalar(
'weight_decay',
parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
loss_func, include_boundaries = get_loss()
# Honestly probably do not need to have hyper-parameters per model, but leave it for now.
optimizer = torch.optim.Adam(
model.parameters(),
lr=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr'],
weight_decay=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay_step'],
gamma=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay'])
start_time = time.time()
model_wts = None
model_wts = train(dloaders,
model,
loss_func,
optimizer,
scheduler,
writer,
parameters.NUM_EPOCHS,
include_boundaries=include_boundaries)
if model_wts:
model.load_state_dict(model_wts)
if not os.path.exists(save_path):
os.makedirs(save_path)
save_path = save_path + "/" + "model.pt"
torch.save(model, save_path)
print('Saved best model based on {} to path {}'.format(
parameters.TRAIN_MODEL_SAVE_CRITERIA.upper(), save_path))
else:
print('For some reason I don\'t have a model to save')
print('Training time: {:10f} minutes'.format(
(time.time() - start_time) / 60))
writer.close()
def main():
args = parser.parse_args()
# What do we need to do across all of the settings!
# Get the data loaders!
args = parser.parse_args()
if args.local_files:
train_data_path = parameters.LOCAL_TRAIN_FILES
test_data_path = parameters.LOCAL_TEST_FILES
full_train_path = parameters.LOCAL_FULL_TRAIN
full_test_path = parameters.LOCAL_FULL_TEST
else:
train_data_path = parameters.REMOTE_TRAIN_FILES
test_data_path = parameters.REMOTE_TEST_FILES
full_train_path = parameters.REMOTE_FULL_TRAIN
full_test_path = parameters.REMOTE_FULL_TEST
if parameters.HIERARCHICAL_SHIFT_WINDOWS:
full_train_path += '_OversizeCalls'
model_0_train_data_path, include_boundaries = create_dataset_path(
train_data_path,
neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.CALL_REPEATS,
shift_windows=parameters.SHIFT_WINDOWS)
model_0_test_data_path, _ = create_dataset_path(
test_data_path,
neg_samples=parameters.TEST_NEG_SAMPLES,
call_repeats=1)
# Check if a different dataset is being used for Model_1
model_1_train_data_path = model_0_train_data_path
model_1_test_data_path = model_0_test_data_path
if str(parameters.HIERARCHICAL_REPEATS).lower() != "same":
# SHould prob just have neg samples x1 since doesnt matter!!
model_1_train_data_path, _ = create_dataset_path(
train_data_path,
neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.HIERARCHICAL_REPEATS,
shift_windows=parameters.HIERARCHICAL_SHIFT_WINDOWS)
# Model 0 Loaders
model_0_train_loader = get_loader_fuzzy(
model_0_train_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries,
shift_windows=parameters.SHIFT_WINDOWS,
full_window_predict=True)
model_0_test_loader = get_loader_fuzzy(
model_0_test_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries,
full_window_predict=True)
# Model 1 Loaders
model_1_train_loader = get_loader_fuzzy(
model_1_train_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries,
shift_windows=parameters.HIERARCHICAL_SHIFT_WINDOWS,
full_window_predict=True)
model_1_test_loader = get_loader_fuzzy(
model_1_test_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries,
full_window_predict=True)
if args.models_path is None:
save_path = create_save_path(time.strftime("%Y-%m-%d_%H:%M:%S",
time.localtime()),
args.save_local,
save_prefix='Hierarchical_')
else:
save_path = args.models_path
# Case 1) Do the entire pipeline! Can break now the pipeline into 3 helper functions!
if args.full_pipeline:
# Train and save model_0
#model_0 = train_model_0(dloaders, save_path)
model_0 = train_model_0(model_0_train_loader, model_0_test_loader,
save_path)
# Do the adversarial discovery
adversarial_train_files, adversarial_test_files = adversarial_discovery(
full_train_path, full_test_path, model_0, save_path)
# Train and save model 1
train_model_1(adversarial_train_files, adversarial_test_files,
model_1_train_loader, model_1_test_loader, save_path)
# Just generate new adversarial examples
elif args.adversarial:
# Load model_0
model_0_path = os.path.join(save_path, "Model_0/model.pt")
model_0 = torch.load(model_0_path, map_location=parameters.device)
adversarial_discovery(full_train_path, full_test_path, model_0,
save_path)
# Train just model_1
elif args.model1:
# Read in the adversarial files
train_adversarial_file = "model_0-False_Pos_Train.txt"
if parameters.HIERARCHICAL_SHIFT_WINDOWS:
train_adversarial_file = "model_0-False_Pos_Train_Shift.txt"
adversarial_train_save_path = os.path.join(save_path,
train_adversarial_file)
adversarial_train_files = []
with open(adversarial_train_save_path, 'r') as f:
files = f.readlines()
for file in files:
adversarial_train_files.append(file.strip())
adversarial_test_save_path = os.path.join(
save_path, "model_0-False_Pos_Test.txt")
adversarial_test_files = []
with open(adversarial_test_save_path, 'r') as f:
files = f.readlines()
for file in files:
adversarial_test_files.append(file.strip())
train_model_1(adversarial_train_files, adversarial_test_files,
model_1_train_loader, model_1_test_loader, save_path)
else:
print("Invalid running mode!")
def train_model(run_number):
logger = console_logger(__name__, FLAGS.logger_level)
logger.info("Clearning Keras session.")
K.clear_session()
save_path = create_save_path()
logger.debug(f"New save path: {save_path}")
# __INPUT PIPELINE__ #
logger.info("Loading datasets.")
ds_train, ds_test, num_train_batches, num_test_batches = load_datasets(
FLAGS.load_dir)
logger.debug(
f"num_train_batches: {num_train_batches}, num_test_batches: {num_test_batches}"
)
# __MODEL__ #
logger.info("Initializing kernel.")
kernel_initializer = tf.initializers.TruncatedNormal(
mean=FLAGS.weight_init_mean, stddev=FLAGS.weight_init_stddev)
logger.info("Building model")
model = build_model(kernel_initializer)
# print('Trainable params: {}'.format(model.count_params()))
logger.info(model.summary())
# Load model weights from checkpoint if checkpoint_path is provided
if FLAGS.checkpoint_path:
logger.info("Loading model weights from checkpoint.")
model.load_weights(FLAGS.checkpoint_path)
# save model FLAGS to save_path
logger.info(f"Saving flags (settings) to {save_path}")
save_config(save_path)
# save model architecture image to save_path directory
if FLAGS.save_architecture_image:
logger.info(f"Saving model architecture image to {save_path}")
tf.keras.utils.plot_model(model,
os.path.join(save_path, 'architecture.png'),
show_shapes=FLAGS.show_shapes)
# __LOGGING__ #
logger.info(f"Initializing summyary writers.")
summary_writer_train = tf.summary.create_file_writer(save_path + '/train',
name='sw-train')
summary_writer_test = tf.summary.create_file_writer(save_path + '/test',
name='sw-test')
# __TRAINING__ #
logger.info(f"Initializing variables, optimizer and metrics.")
best_cer = 100.0
best_epoch = -1
optimizer = tf.keras.optimizers.Adam(lr=FLAGS.lr,
epsilon=FLAGS.epsilon,
amsgrad=FLAGS.amsgrad)
train_loss = tf.keras.metrics.Mean(name='train_loss', dtype=tf.float32)
test_loss = tf.keras.metrics.Mean(name='test_loss', dtype=tf.float32)
test_cer = tf.keras.metrics.Mean(name='test_cer', dtype=tf.float32)
test_ccer = tf.keras.metrics.Mean(name="test_ccer", dtype=tf.float32)
for epoch in range(FLAGS.max_epochs):
logger.log(35, f'_______| Run {run_number} | Epoch {epoch} |_______')
# TRAINING DATA
with summary_writer_train.as_default():
logger.info(f"Training model.")
train_fn(model, ds_train, optimizer, train_loss, num_train_batches,
epoch)
# TESTING DATA
with summary_writer_test.as_default():
logger.info(f"Testing model.")
_, test_mean_cer = test_fn(model, ds_test, test_loss, test_cer,
test_ccer, num_test_batches, epoch)
# EARLY STOPPING AND KEEPING THE BEST MODEL
stop_training, best_cer, best_epoch = early_stopping(
model, test_mean_cer, best_cer, epoch, best_epoch, save_path)
logger.log(35, f'| Best CER {best_cer} | Best epoch {best_epoch} |')
if stop_training:
logger.log(35, f'Model stopped early at epoch {epoch}')
break
def main():
# What do we need to do across all of the settings!
# Get the data loaders!
args = parser.parse_args()
if args.local_files:
train_data_path = parameters.LOCAL_TRAIN_FILES
test_data_path = parameters.LOCAL_TEST_FILES
full_train_path = parameters.LOCAL_FULL_TRAIN
full_test_path = parameters.LOCAL_FULL_TEST
else:
if parameters.DATASET.lower() == "noab":
train_data_path = parameters.REMOTE_TRAIN_FILES
test_data_path = parameters.REMOTE_TEST_FILES
full_train_path = parameters.REMOTE_FULL_TRAIN
full_test_path = parameters.REMOTE_FULL_TEST
else:
train_data_path = parameters.REMOTE_BAI_TRAIN_FILES
test_data_path = parameters.REMOTE_BAI_TEST_FILES
full_train_path = parameters.REMOTE_FULL_TRAIN_BAI
full_test_path = parameters.REMOTE_FULL_TEST_BAI
# Get oversized calls if shifting windows or repeating for model 2
# We should try to remove both of these This is an issue too!
if parameters.HIERARCHICAL_SHIFT_WINDOWS: # or parameters.HIERARCHICAL_REPEATS > 1:
full_train_path += '_OversizeCalls'
# For model 2 we need to have oversized calls to generate the randomly located repeats
if parameters.HIERARCHICAL_REPEATS_POS > 1 or parameters.HIERARCHICAL_REPEATS_NEG > 1:
full_train_path += '_OversizeCalls'
model_0_train_data_path, include_boundaries = create_dataset_path(train_data_path, neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.CALL_REPEATS,
shift_windows=parameters.SHIFT_WINDOWS)
model_0_test_data_path, _ = create_dataset_path(test_data_path, neg_samples=parameters.TEST_NEG_SAMPLES,
call_repeats=1)
# Check if a different dataset is being used for Model_1
model_1_train_data_path = model_0_train_data_path
model_1_test_data_path = model_0_test_data_path
# Remove this same thing!
#if str(parameters.HIERARCHICAL_REPEATS).lower() != "same" or parameters.HIERARCHICAL_REPEATS_POS > 1 or parameters.HIERARCHICAL_REPEATS_NEG > 1:
if parameters.HIERARCHICAL_REPEATS_POS > 1 or parameters.HIERARCHICAL_REPEATS_NEG > 1:
# SHould prob just have neg samples x1 since doesnt matter!!
# For now set call repeats to 1, but get shifting windows so we later can do call repeats!
#shift_windows = parameters.HIERARCHICAL_REPEATS > 1 or parameters.HIERARCHICAL_SHIFT_WINDOWS
# For now should make shift windows just be true! Because it does not make a lot of sense to do
# repeats without shifting windows since we can only repeat the pos examples
shift_windows = True
# Set this to 1 because we take care of this later!!!!
call_repeats = 1
model_1_train_data_path, _ = create_dataset_path(train_data_path, neg_samples=parameters.NEG_SAMPLES,
call_repeats=call_repeats,
shift_windows=shift_windows)
# Model 0 Loaders
model_0_train_loader = get_loader_fuzzy(model_0_train_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE,
include_boundaries=include_boundaries, shift_windows=parameters.SHIFT_WINDOWS)
model_0_test_loader = get_loader_fuzzy(model_0_test_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE, include_boundaries=include_boundaries)
# Model 1 Loaders
model_1_train_loader = get_loader_fuzzy(model_1_train_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE,
include_boundaries=include_boundaries, shift_windows=parameters.HIERARCHICAL_SHIFT_WINDOWS)
model_1_test_loader = get_loader_fuzzy(model_1_test_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE, include_boundaries=include_boundaries)
if args.path is None:
save_path = create_save_path(time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()), args.save_local, save_prefix='Hierarchical_')
else:
save_path = args.path
# Case 1) Do the entire pipeline! Can break now the pipeline into 3 helper functions!
if args.full_pipeline:
# Train and save model_0
if args.model_0 == None:
model_0 = train_model_0(model_0_train_loader, model_0_test_loader , save_path, args.pre_train_0)
else: # Load and save model_0
model_0 = torch.load(args.model_0, map_location=parameters.device)
first_model_save_path = os.path.join(save_path, "Model_0")
if not os.path.exists(first_model_save_path):
os.makedirs(first_model_save_path)
model_save_path = os.path.join(first_model_save_path, "model.pt")
torch.save(model_0, model_save_path)
# Do the adversarial discovery
adversarial_train_files, adversarial_test_files = adversarial_discovery(full_train_path, full_test_path,
model_0_train_loader, model_0_test_loader, model_0, save_path)
# Train and save model 1
train_model_1(adversarial_train_files, adversarial_test_files, model_1_train_loader, model_1_test_loader,
save_path, args.pre_train_1)
# Just generate new adversarial examples
elif args.adversarial:
# Load model_0
model_0_path = os.path.join(save_path, "Model_0/model.pt")
model_0 = torch.load(model_0_path, map_location=parameters.device)
adversarial_discovery(full_train_path, full_test_path, model_0_train_loader, model_0_test_loader, model_0, save_path)
# Train just model_1
elif args.model1:
# Read in the adversarial files
train_adversarial_file = "model_0-False_Pos_Train.txt"
#if parameters.HIERARCHICAL_SHIFT_WINDOWS or parameters.HIERARCHICAL_REPEATS > 1:
if parameters.HIERARCHICAL_REPEATS_POS > 1 or parameters.HIERARCHICAL_REPEATS_NEG > 1:
train_adversarial_file = "model_0-False_Pos_Train_Shift.txt"
adversarial_train_save_path = os.path.join(save_path, train_adversarial_file)
adversarial_train_files = []
with open(adversarial_train_save_path, 'r') as f:
files = f.readlines()
for file in files:
adversarial_train_files.append(file.strip())
adversarial_test_save_path = os.path.join(save_path, "model_0-False_Pos_Test.txt")
adversarial_test_files = []
with open(adversarial_test_save_path, 'r') as f:
files = f.readlines()
for file in files:
adversarial_test_files.append(file.strip())
train_model_1(adversarial_train_files, adversarial_test_files, model_1_train_loader,
model_1_test_loader, save_path, args.pre_train_1)
elif args.visualize:
model_0_path = os.path.join(save_path, "Model_0/model.pt")
model_0 = torch.load(model_0_path, map_location=parameters.device)
model_1_name = hierarchical_model_1_path()
model_1_path = os.path.join(save_path, model_1_name+'/model.pt')
model_1 = torch.load(model_1_path, map_location=parameters.device)
# Read in the adversarial files
train_adversarial_file = "model_0-False_Pos_Train.txt"
#if parameters.HIERARCHICAL_SHIFT_WINDOWS or parameters.HIERARCHICAL_REPEATS > 1:
if parameters.HIERARCHICAL_REPEATS_POS > 1 or parameters.HIERARCHICAL_REPEATS_NEG > 1:
train_adversarial_file = "model_0-False_Pos_Train_Shift.txt"
adversarial_train_save_path = os.path.join(save_path, train_adversarial_file)
adversarial_train_files = []
with open(adversarial_train_save_path, 'r') as f:
files = f.readlines()
for file in files:
adversarial_train_files.append(file.strip())
visualize_adversarial(adversarial_train_files, model_1_train_loader, model_0, model_1)
else:
print ("Invalid running mode!")
def main():
args = parser.parse_args()
if args.local_files:
train_data_path = parameters.LOCAL_TRAIN_FILES
test_data_path = parameters.LOCAL_TEST_FILES
full_train_path = parameters.LOCAL_FULL_TRAIN
full_test_path = parameters.LOCAL_FULL_TEST
else:
if parameters.DATASET.lower() == "noab":
train_data_path = parameters.REMOTE_TRAIN_FILES
test_data_path = parameters.REMOTE_TEST_FILES
full_train_path = parameters.REMOTE_FULL_TRAIN
full_test_path = parameters.REMOTE_FULL_TEST
else:
train_data_path = parameters.REMOTE_BAI_TRAIN_FILES
test_data_path = parameters.REMOTE_BAI_TEST_FILES
full_train_path = parameters.REMOTE_FULL_TRAIN_BAI
full_test_path = parameters.REMOTE_FULL_TEST_BAI
train_data_path, include_boundaries = create_dataset_path(train_data_path, neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.CALL_REPEATS,
shift_windows=parameters.SHIFT_WINDOWS)
test_data_path, _ = create_dataset_path(test_data_path, neg_samples=parameters.TEST_NEG_SAMPLES,
call_repeats=1)
train_loader = get_loader_fuzzy(train_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE,
include_boundaries=include_boundaries, shift_windows=parameters.SHIFT_WINDOWS)
test_loader = get_loader_fuzzy(test_data_path, parameters.BATCH_SIZE, random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM, scale=parameters.SCALE, include_boundaries=include_boundaries)
# For now we don't need to save the model
save_path = create_save_path(time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()), args.save_local)
train_dataloaders = {'train':train_loader, 'valid':test_loader}
# Load the full data sets - SET SHUFFLE = False
full_train_loader = get_loader_fuzzy(full_train_path, parameters.BATCH_SIZE, shuffle=False,
norm=parameters.NORM, scale=parameters.SCALE,
include_boundaries=False, shift_windows=False,
is_full_dataset=True)
full_test_loader = get_loader_fuzzy(full_test_path, parameters.BATCH_SIZE, shuffle=False,
norm=parameters.NORM, scale=parameters.SCALE, include_boundaries=False)
full_dataloaders = {'train':full_train_loader, 'valid': full_test_loader}
model = get_model(parameters.MODEL_ID)
model.to(parameters.device)
print(model)
writer = SummaryWriter(save_path)
writer.add_scalar('batch_size', parameters.BATCH_SIZE)
writer.add_scalar('weight_decay', parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
# Want to use focal loss! Next thing to check on!
loss_func, include_boundaries = get_loss()
# Honestly probably do not need to have hyper-parameters per model, but leave it for now.
optimizer = torch.optim.Adam(model.parameters(), lr=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr'],
weight_decay=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay_step'],
gamma=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay'])
start_time = time.time()
curriculum_profiling(model, train_dataloaders, full_dataloaders, loss_func, optimizer, scheduler, writer)
print('Training time: {:10f} minutes'.format((time.time()-start_time)/60))
writer.close()
def main():
args = parser.parse_args()
use_focal = False
if parameters.LOSS.lower() == "focal" or parameters.LOSS.lower(
) == "focal_chunk":
use_focal = True
if args.local_files:
if use_focal:
train_data_path = parameters.LOCAL_FULL_TRAIN
test_data_path = parameters.LOCAL_FULL_TEST
else:
train_data_path = parameters.LOCAL_TRAIN_FILES
test_data_path = parameters.LOCAL_TEST_FILES
else:
if parameters.DATASET.lower() == "noab":
if use_focal:
train_data_path = parameters.REMOTE_FULL_TRAIN
test_data_path = parameters.REMOTE_FULL_TEST
else:
train_data_path = parameters.REMOTE_TRAIN_FILES
test_data_path = parameters.REMOTE_TEST_FILES
else:
if use_focal:
train_data_path = parameters.REMOTE_FULL_TRAIN_BAI
test_data_path = parameters.REMOTE_FULL_TEST_BAI
else:
train_data_path = parameters.REMOTE_BAI_TRAIN_FILES
test_data_path = parameters.REMOTE_BAI_TEST_FILES
if use_focal:
include_boundaries = False
else:
train_data_path, include_boundaries = create_dataset_path(
train_data_path,
neg_samples=parameters.NEG_SAMPLES,
call_repeats=parameters.CALL_REPEATS,
shift_windows=parameters.SHIFT_WINDOWS)
test_data_path, _ = create_dataset_path(
test_data_path,
neg_samples=parameters.TEST_NEG_SAMPLES,
call_repeats=1)
train_loader = get_loader_fuzzy(train_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries,
shift_windows=parameters.SHIFT_WINDOWS)
test_loader = get_loader_fuzzy(test_data_path,
parameters.BATCH_SIZE,
random_seed=parameters.DATA_LOADER_SEED,
norm=parameters.NORM,
scale=parameters.SCALE,
include_boundaries=include_boundaries)
save_path = create_save_path(
time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()), args.save_local)
dloaders = {'train': train_loader, 'valid': test_loader}
## Training
# Load a pre-trained model
if parameters.PRE_TRAIN:
model = torch.load(args.pre_train, map_location=parameters.device)
else:
model = get_model(parameters.MODEL_ID)
model.to(parameters.device)
print(model)
writer = SummaryWriter(save_path)
writer.add_scalar('batch_size', parameters.BATCH_SIZE)
writer.add_scalar(
'weight_decay',
parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
# Want to use focal loss! Next thing to check on!
loss_func, include_boundaries = get_loss()
# Honestly probably do not need to have hyper-parameters per model, but leave it for now.
optimizer = torch.optim.Adam(
model.parameters(),
lr=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr'],
weight_decay=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['l2_reg'])
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay_step'],
gamma=parameters.HYPERPARAMETERS[parameters.MODEL_ID]['lr_decay'])
start_time = time.time()
model_wts = None
model_wts = train(dloaders,
model,
loss_func,
optimizer,
scheduler,
writer,
parameters.NUM_EPOCHS,
include_boundaries=include_boundaries)
if model_wts:
model.load_state_dict(model_wts)
if not os.path.exists(save_path):
os.makedirs(save_path)
save_path = save_path + "/" + "model.pt"
torch.save(model, save_path)
print('Saved best model based on {} to path {}'.format(
parameters.TRAIN_MODEL_SAVE_CRITERIA.upper(), save_path))
else:
print('For some reason I don\'t have a model to save')
print('Training time: {:10f} minutes'.format(
(time.time() - start_time) / 60))
writer.close()