def train_bgd(self):
"""
Optimize the parameters of a model use batch gradient decent.
Loss and trained models are stored in local file.
Returns a tuple of:
- best_model: The model with the lowest validation error.
- train_loss_history: A list containing the value of loss
at each iteration (update) during training.
- val_loss_history: A list containing the value of loss
at each epoch based on trained model.
Others:
- A logging file to store losses.
- Some model files to store trained model at each epoch
"""
# prepare the training and validation datasets
data_info_file = self.data_info_file
data_info = np.array([l.strip() for l in open(data_info_file).readlines()])
# data_info = data_info[0:100]
N = len(data_info)
print (N)
N_val = int(N * self.val_rate)
N_train = N - N_val
perm = np.random.permutation(N)
train_info = data_info[perm[0: N_train]]
val_info = data_info[perm[N_train:]]
print 'Number of training data %d' % N_train
print 'Number of validation data %d' % N_val
# create log file and result directory
log_file, result_dir = self.creat_result_dir_logfile()
print(self.params)
logging.info(self.params) # log the params for training
# get the optimizer
self.optimizer = self.get_optimizer()
lowest_val_loss = float('inf')
best_model = None
self.train_loss_batch_history = []
train_loss_epoch_history = []
val_loss_epoch_history = []
# Add info into to log file
logging.info('Use batch gradient decent!')
logging.info('# of training data: {}'.format(N_train))
logging.info('# of validation data: {}'.format(N_val))
logging.info('# of batch gradient decent')
logging.info('# of iteration: {}'.format(self.num_epochs))
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('start training ...')
for epoch in xrange(1, self.num_epochs + 1):
start_time = time.time()
input_data, label = self.all_data_load(train_info, result_dir)
self.compute_loss_model_update_bgd(input_data, label, self.num_iters)
# each epoch performs an evaluation on subset of training data
# the same as number of validation datasets
mask = np.random.choice(N_train, N_val, replace=False)
train_test_data = input_data[mask]
label_test = label[mask]
train_loss_epoch = self.compute_loss_bgd(train_test_data, label_test)
train_loss_epoch_history.append(train_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('training', train_loss_epoch,
elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
# each epoch perform an evaluation on validation set
start_time = time.time()
input_data, label = self.all_data_load(val_info, result_dir)
val_loss_epoch = self.compute_loss_bgd(input_data, label)
val_loss_epoch_history.append(val_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('validation', val_loss_epoch,
elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
if val_loss_epoch < lowest_val_loss:
lowest_val_loss = val_loss_epoch
best_model = copy.deepcopy(self.model)
# draw loss curve
draw_batch_loss(self.train_loss_batch_history,
result_dir + '/batch_loss.jpg')
draw_loss_from_list(train_loss_epoch_history,
val_loss_epoch_history,
result_dir + '/loss.jpg')
# store the trained model
if epoch == 1 or epoch % self.snapshot == 0:
model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
pickle.dump(self.model, open(model_fn, 'wb'), -1)
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('End training')
return (best_model, self.train_loss_batch_history,
train_loss_epoch_history, val_loss_epoch_history)
def train(self):
"""
Optimize the parameters of a model to minimize the loss.
Loss and trained models are stored in local file.
Returns a tuple of:
- best_model: The model with the lowest validation error.
- train_loss_batch_history: A list all batch loss in all epochs
- train_loss_epoch_history: A list containing the value of loss
at each epoch (update) during training.
- val_loss_epoch_history: A list containing the value of loss
at each epoch based on trained model.
Others:
- A logging file to store losses.
- Some model files to store trained model at each epoch
"""
# prepare the training and validation datasets
data_info_file = self.data_info_file
data_info = np.array([l.strip() for l in open(data_info_file).readlines()])
# data_info = data_info[0:100]
N = len(data_info)
print (N)
N_val = int(N * self.val_rate)
N_train = N - N_val
perm = np.random.permutation(N)
train_info = data_info[perm[0: N_train]]
val_info = data_info[perm[N_train:]]
print 'Number of training data %d' % N_train
print 'Number of validation data %d' % N_val
# create log file and result directory
log_file, result_dir = self.creat_result_dir_logfile()
print(self.params)
logging.info(self.params) # log the params for training
# get the optimizer
self.optimizer = self.get_optimizer()
lowest_val_loss = float('inf')
best_model = None
self.train_loss_batch_history = []
train_loss_epoch_history = []
val_loss_epoch_history = []
# Add info into to log file
logging.info('# of training data: {}'.format(N_train))
logging.info('# of validation data: {}'.format(N_val))
logging.info('# of batch size: {}'.format(self.batch_size))
logging.info('# of epoch: {}'.format(self.num_epochs))
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('start training ...')
for epoch in xrange(1, self.num_epochs + 1):
# use multiprocess to speed up training and save memory
# the model can be trained while the batch data is loading
input_q = Queue() # store data info for loading and transformation
data_q = Queue() # store transformed data
# create process to load training data
data_load_process = Process(target=self.data_load,
args=(input_q, data_q, result_dir))
data_load_process.start()
start_time = time.time()
self.compute_loss_model_update(train_info, input_q, data_q)
input_q.put(None)
data_load_process.join()
# each epoch performs an evaluation on subset of training data
# the same as number of validation datasets
input_q = Queue()
data_q = Queue()
data_load_process = Process(target=self.data_load,
args=(input_q, data_q, result_dir))
data_load_process.start()
train_loss_epoch = self.compute_loss(train_info, N_val, input_q, data_q)
train_loss_epoch_history.append(train_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('training', train_loss_epoch,
elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
input_q.put(None)
data_load_process.join()
# each epoch perform an evaluation on validation set
input_q = Queue()
data_q = Queue()
data_load_process = Process(target=self.data_load,
args=(input_q, data_q, result_dir))
data_load_process.start()
start_time = time.time()
val_loss_epoch = self.compute_loss(val_info, N_val, input_q, data_q)
val_loss_epoch_history.append(val_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('validation',
val_loss_epoch, elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
if val_loss_epoch < lowest_val_loss:
lowest_val_loss = val_loss_epoch
best_model = copy.deepcopy(self.model)
input_q.put(None)
data_load_process.join()
# draw loss curve
draw_batch_loss(self.train_loss_batch_history, result_dir
+ '/batch_loss.jpg')
draw_loss_from_list(train_loss_epoch_history,
val_loss_epoch_history,
result_dir + '/val_loss.jpg')
# store the trained model
if epoch == 1 or epoch % self.snapshot == 0:
model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
pickle.dump(self.model, open(model_fn, 'wb'), -1)
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('End training')
return (best_model, self.train_loss_batch_history,
train_loss_epoch_history, val_loss_epoch_history)
def train(self, X, y):
"""
Optimize the parameters of a model to minimize the loss.
Loss and trained models are stored in local file.
Returns a tuple of:
- best_model: The model with the lowest validation error.
- train_loss_batch_history: A list all batch loss in all epochs
- train_loss_epoch_history: A list containing the value of loss
at each epoch (update) during training.
- val_loss_epoch_history: A list containing the value of loss
at each epoch based on trained model.
Others:
- A logging file to store losses.
- Some model files to store trained model at each epoch
"""
# prepare the training and validation datasets
# chainer only support np.float32
if X.dtype != np.float32:
X = X.astype(np.float32)
if y.dtype != np.float32:
y = y.astype(np.float32)
N = X.shape[0]
N_val = int(N * self.val_rate)
N_train = N - N_val
perm = np.random.permutation(N)
X_train = X[perm[0: N_train]]
y_train = y[perm[0: N_train]]
X_val = X[perm[N_train:]]
y_val = y[perm[N_train:]]
print 'Number of training data %d' % N_train
print 'Number of validation data %d' % N_val
# create log file and result directory
log_file, result_dir = self.creat_result_dir_logfile()
print(self.params)
logging.info(self.params) # log the params for training
# get the optimizer
self.optimizer = self.get_optimizer()
lowest_val_loss = float('inf')
best_model = None
self.train_loss_batch_history = []
train_loss_epoch_history = []
val_loss_epoch_history = []
# Add info into to log file
logging.info('# of training data: {}'.format(N_train))
logging.info('# of validation data: {}'.format(N_val))
logging.info('# of batch size: {}'.format(self.batch_size))
logging.info('# of epoch: {}'.format(self.num_epochs))
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('start training ...')
for epoch in xrange(1, self.num_epochs + 1):
start_time = time.time()
# update model
self.compute_loss_model_update(X_train, y_train)
# each epoch performs an evaluation on subset of training data
# the same as number of validation datasets
mask = np.random.choice(N_train, N_val, replace=False)
train_test_data = X[mask]
label_test = y[mask]
# train_test_data = X_train
# label_test = y_train
train_loss_epoch = self.compute_loss(train_test_data, label_test)
train_loss_epoch_history.append(train_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('training', train_loss_epoch,
elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
# each epoch perform an evaluation on validation set
start_time = time.time()
val_loss_epoch = self.compute_loss(X_val, y_val)
val_loss_epoch_history.append(val_loss_epoch)
elapsed_time = time.time() - start_time
log_msg = self.get_log_msg('validation', val_loss_epoch,
elapsed_time, epoch)
logging.info(log_msg)
print(log_msg)
if val_loss_epoch < lowest_val_loss:
lowest_val_loss = val_loss_epoch
best_model = copy.deepcopy(self.model)
# draw loss curve
draw_batch_loss(self.train_loss_batch_history,
result_dir + '/batch_loss.jpg')
draw_loss_from_list(train_loss_epoch_history,
val_loss_epoch_history,
result_dir + '/loss.jpg')
# store the trained model
if epoch == 1 or epoch % self.snapshot == 0:
model_fn = '%s/epoch_%d.chainermodel' % (result_dir, epoch)
pickle.dump(self.model, open(model_fn, 'wb'), -1)
logging.info(time.strftime('%Y-%m-%d_%H-%M-%S'))
logging.info('End training')
return (best_model, self.train_loss_batch_history,
train_loss_epoch_history, val_loss_epoch_history)
