def _train(self, batch_size, max_epochs):
print_section('Training model')
patience = self.params.initial_patience # look as this many examples regardless
patience_increase = self.params.patience_increase # wait this much longer when a new best is found
improvement_threshold = self.params.improvement_threshold # a relative improvement of this much is considered significant
learning_rate = self.params.learning_rate
learning_adjustment = self.params.learning_adjustment
learning_decrease = self.params.learning_decrease
nr_learning_adjustments = 0
print('---- Initial learning rate {}'.format(learning_rate))
max_factor = self.params.factor_rate
factor_adjustment = self.params.factor_adjustment
factor_decrease = self.params.factor_decrease
factor_minimum = self.params.factor_minimum
print('---- Initial loss mixture ratio {}'.format(max_factor))
curriculum = self.params.curriculum_enable
curriculum_start = self.params.curriculum_start
curriculum_adjustment = self.params.curriculum_adjustment
# go through this many minibatch before checking the network on the validation set
gui_frequency = 500
validation_frequency = min(self.nr_train_batches, patience / 2)
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
self.start_time = timeit.default_timer()
storage = ParamStorage()
nr_chunks = self.data.get_chunk_number()
epoch = 0
done_looping = False
iter = 0
#==== INITIAL PERFORMANCE ====
chunk_batches = self.data.get_elements(0) / batch_size
validation_score = self._get_validation_score(batch_size, epoch, 0)
test_score = self._get_test_score(batch_size)
training_score = self._get_training_score(chunk_batches)
#==== UPDATE GUI ====
if visual_params.gui_enabled:
interface.server.append_job_update(epoch, training_score,
validation_score, test_score,
learning_rate)
try:
while (epoch < max_epochs) and (not done_looping):
epoch = epoch + 1
if (epoch % learning_adjustment == 0):
learning_rate *= learning_decrease
nr_learning_adjustments += 1
#Temp
learning_adjustment = max(10, int(learning_adjustment / 2))
print('---- New learning rate {}'.format(learning_rate))
if (epoch > factor_adjustment):
max_factor = max(max_factor * factor_decrease,
factor_minimum)
print('---- New convex combination {}'.format(max_factor))
if (epoch % 20 == 0):
print('---- Storing temp model')
storage.store_params(self.model.params, id=str(epoch))
if (curriculum and epoch % curriculum_adjustment == 0
and epoch >= curriculum_start):
print(
"---- Mixing examples from next stage with training data"
)
self.data.mix_in_next_stage()
#For current examples chunk in GPU memory
for chunk_index in range(nr_chunks):
self.data.switch_active_training_set(chunk_index)
nr_elements = self.data.get_elements(chunk_index)
chunk_batches = nr_elements / batch_size
#Each chunk contains a certain number of batches.
for minibatch_index in range(chunk_batches):
cost_ij = self.train_model(minibatch_index,
learning_rate, max_factor)
if iter % 1000 == 0:
print(
'---- Training @ iter = {}. Patience = {}. Loss = {}'
.format(iter, patience, cost_ij))
if visual_params.gui_enabled and iter % gui_frequency == 0:
interface.server.get_command_status()
if visual_params.gui_enabled and interface.server.is_testing(
):
self._debug(batch_size, chunk_batches, max_factor)
#if(np.isnan(cost_ij)):
# print('cost IS NAN')
#==== EVAULATE ====
if (iter + 1) % validation_frequency == 0:
#==== CURRENT PERFORMANCE ====
validation_score = self._get_validation_score(
batch_size, epoch, minibatch_index)
test_score = self._get_test_score(batch_size)
train_score = self._get_training_score(
chunk_batches) #No other purpose than charting
#==== UPDATE GUI ====
if visual_params.gui_enabled:
interface.server.append_job_update(
epoch, train_score, validation_score,
test_score, learning_rate)
self.events.append({
"epoch": epoch,
"training_loss": train_score,
"validation_loss": validation_score,
"test_loss": test_score,
"training_rate": learning_rate
})
#==== EARLY STOPPING ====
if validation_score < best_validation_loss:
#improve patience if loss improvement is good enough
if validation_score < best_validation_loss * improvement_threshold:
patience = max(patience,
iter * patience_increase)
print(
"---- New best validation loss. Patience increased to {}"
.format(patience))
# save best validation score and iteration number
best_validation_loss = validation_score
best_iter = iter
if patience <= iter:
done_looping = True
break
if visual_params.gui_enabled and interface.server.stop:
done_looping = True
iter += 1 #Increment interation after each batch has been processed.
except KeyboardInterrupt:
self.set_result(best_iter, iter, best_validation_loss, test_score,
nr_learning_adjustments, epoch)
print(
"Inpterupted by user. Current model params will be saved now.")
except Exception as e:
print "Unexpected error:", sys.exc_info()[0]
raise
self.set_result(best_iter, iter, best_validation_loss, test_score,
nr_learning_adjustments, epoch)
'''
print_section("TOOLS: Measure precision and recall of model")
print("-data: path to dataset | -store: job_gui id to store curve in GUI "\
"| -store_path: store results locally | -model: stored model to use")
#====== Arguments ===============================================
is_dataset_path, dataset_path = get_command(
'-data', default='/home/olav/Pictures/Mass_roads_alpha')
store_gui, job_id = get_command('-store_gui', default='-1')
is_store_path, store_path = get_command('-store_path',
default='./pr_data.json')
is_model, model_path = get_command('-model', default='./results/params.pkl')
#==============================================================
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data['optimization'].batch_size
measurer = PrecisionRecallCurve(dataset_path, data['params'], data['model'],
data['dataset'])
datapoints = measurer.get_curves_datapoints(batch_size)
if store_gui:
send_precision_recall_data(datapoints, None, job_id=job_id)
else:
with open(store_path, 'w') as outfile:
json.dump([{"curve": datapoints, "events": []}], outfile)
plt.suptitle('Precision and recall')
plt.xlabel('Recall')
is_teacher_location, teacher_location = get_command(
'-teacher', default=filename_params.curriculum_teacher)
verify, stage = get_command('-verify', default="0")
stage = "stage" + stage
is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5")
tradeoff = float(tradeoff)
#Dataset path. Config used if not supplied
is_alt_dataset, alt_dataset = get_command('-dataset')
if is_alt_dataset:
dataset_path = alt_dataset
#==============================================================
store = ParamStorage()
teacher = store.load_params(path=teacher_location)
evaluate = util.create_simple_predictor(teacher['model'], teacher['params'])
if not verify:
creator = Creator(pr_path,
dim=(dataset_params.input_dim,
dataset_params.output_dim),
preproccessing=dataset_params.use_preprocessing,
std=dataset_params.dataset_std,
reduce_training=dataset_params.reduce_training,
reduce_testing=dataset_params.reduce_testing,
reduce_validation=dataset_params.reduce_validation)
creator.load_dataset()
data, labels = creator.sample_data(creator.train,
verify, stage = get_command('-verify', default="0")
stage = "stage" + stage
is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5")
tradeoff = float(tradeoff)
#Dataset path. Config used if not supplied
is_alt_dataset, alt_dataset = get_command('-dataset')
if is_alt_dataset:
dataset_path = alt_dataset
#==============================================================
store = ParamStorage()
teacher = store.load_params(path=teacher_location)
evaluate = util.create_simple_predictor(teacher['model'], teacher['params'])
if not verify:
creator = Creator(
pr_path,
dim=(dataset_params.input_dim, dataset_params.output_dim),
preproccessing=dataset_params.use_preprocessing,
std=dataset_params.dataset_std,
reduce_training=dataset_params.reduce_training,
reduce_testing=dataset_params.reduce_testing,
reduce_validation=dataset_params.reduce_validation
)
creator.load_dataset()
"""
print_section("TOOLS: Measure precision and recall of model")
print(
"-data: path to dataset | -store: job_gui id to store curve in GUI "
"| -store_path: store results locally | -model: stored model to use"
)
# ====== Arguments ===============================================
is_dataset_path, dataset_path = get_command("-data", default="/home/olav/Pictures/Mass_roads_alpha")
store_gui, job_id = get_command("-store_gui", default="-1")
is_store_path, store_path = get_command("-store_path", default="./pr_data.json")
is_model, model_path = get_command("-model", default="./results/params.pkl")
# ==============================================================
store = ParamStorage()
data = store.load_params(path=model_path)
batch_size = data["optimization"].batch_size
measurer = PrecisionRecallCurve(dataset_path, data["params"], data["model"], data["dataset"])
datapoints = measurer.get_curves_datapoints(batch_size)
if store_gui:
send_precision_recall_data(datapoints, None, job_id=job_id)
else:
with open(store_path, "w") as outfile:
json.dump([{"curve": datapoints, "events": []}], outfile)
plt.suptitle("Precision and recall")
plt.xlabel("Recall")
def _train(self, batch_size, max_epochs):
print_section('Training model')
patience = self.params.initial_patience # look as this many examples regardless
patience_increase = self.params.patience_increase # wait this much longer when a new best is found
improvement_threshold = self.params.improvement_threshold # a relative improvement of this much is considered significant
learning_rate = self.params.learning_rate
learning_adjustment = self.params.learning_adjustment
learning_decrease = self.params.learning_decrease
nr_learning_adjustments = 0
print('---- Initial learning rate {}'.format(learning_rate))
max_factor = self.params.factor_rate
factor_adjustment = self.params.factor_adjustment
factor_decrease = self.params.factor_decrease
factor_minimum = self.params.factor_minimum
print('---- Initial loss mixture ratio {}'.format(max_factor))
curriculum = self.params.curriculum_enable
curriculum_start = self.params.curriculum_start
curriculum_adjustment = self.params.curriculum_adjustment
# go through this many minibatch before checking the network on the validation set
gui_frequency = 500
validation_frequency = min(self.nr_train_batches, patience / 2)
best_validation_loss = np.inf
best_iter = 0
test_score = 0.
self.start_time = timeit.default_timer()
storage = ParamStorage()
nr_chunks = self.data.get_chunk_number()
epoch = 0
done_looping = False
iter = 0
#==== INITIAL PERFORMANCE ====
chunk_batches = self.data.get_elements( 0 ) / batch_size
validation_score = self._get_validation_score(batch_size, epoch, 0)
test_score = self._get_test_score(batch_size)
training_score = self._get_training_score(chunk_batches)
#==== UPDATE GUI ====
if visual_params.gui_enabled:
interface.server.append_job_update(epoch, training_score, validation_score, test_score, learning_rate)
try:
while (epoch < max_epochs) and (not done_looping):
epoch = epoch + 1
if(epoch % learning_adjustment == 0):
learning_rate *= learning_decrease
nr_learning_adjustments += 1
#Temp
learning_adjustment = max(10, int(learning_adjustment/2))
print('---- New learning rate {}'.format(learning_rate))
if(epoch > factor_adjustment):
max_factor = max(max_factor * factor_decrease, factor_minimum)
print('---- New convex combination {}'.format(max_factor))
if(epoch % 20 == 0):
print('---- Storing temp model')
storage.store_params(self.model.params, id=str(epoch))
if(curriculum and epoch % curriculum_adjustment == 0 and epoch >= curriculum_start):
print("---- Mixing examples from next stage with training data")
self.data.mix_in_next_stage()
#For current examples chunk in GPU memory
for chunk_index in range(nr_chunks):
self.data.switch_active_training_set( chunk_index )
nr_elements = self.data.get_elements( chunk_index )
chunk_batches = nr_elements / batch_size
#Each chunk contains a certain number of batches.
for minibatch_index in range(chunk_batches):
cost_ij = self.train_model(minibatch_index, learning_rate, max_factor)
if iter % 1000 == 0:
print('---- Training @ iter = {}. Patience = {}. Loss = {}'.format(iter, patience, cost_ij))
if visual_params.gui_enabled and iter % gui_frequency == 0:
interface.server.get_command_status()
if visual_params.gui_enabled and interface.server.is_testing():
self._debug(batch_size, chunk_batches, max_factor)
#if(np.isnan(cost_ij)):
# print('cost IS NAN')
#==== EVAULATE ====
if (iter + 1) % validation_frequency == 0:
#==== CURRENT PERFORMANCE ====
validation_score = self._get_validation_score(batch_size, epoch, minibatch_index)
test_score = self._get_test_score(batch_size)
train_score = self._get_training_score(chunk_batches) #No other purpose than charting
#==== UPDATE GUI ====
if visual_params.gui_enabled:
interface.server.append_job_update(
epoch,
train_score,
validation_score,
test_score,
learning_rate)
self.events.append({
"epoch": epoch,
"training_loss": train_score,
"validation_loss": validation_score,
"test_loss": test_score,
"training_rate": learning_rate
})
#==== EARLY STOPPING ====
if validation_score < best_validation_loss:
#improve patience if loss improvement is good enough
if validation_score < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
print("---- New best validation loss. Patience increased to {}".format(patience))
# save best validation score and iteration number
best_validation_loss = validation_score
best_iter = iter
if patience <= iter:
done_looping = True
break
if visual_params.gui_enabled and interface.server.stop:
done_looping = True
iter += 1 #Increment interation after each batch has been processed.
except KeyboardInterrupt:
self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch)
print("Inpterupted by user. Current model params will be saved now.")
except Exception as e:
print "Unexpected error:", sys.exc_info()[0]
raise
self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch)
