def __init__(self,
model,
dataset,
train_cost,
valid_cost,
learning_method,
stop_criteria,
log=None,
verbose=True):
self.model = model
self.dataset = dataset
self.train_cost = train_cost
self.valid_cost = valid_cost
self.learning_method = learning_method
self.stop_criteria = stop_criteria
self.log = log
self.verbose = verbose
if self.log is None:
# use default Log setting
self.log = Log(logger=internal_logger)
elif self.log.save_to_database:
self.log.print_records()
self.log.info('\n')
def train():
# build dataset
batch_size = 64
data = Mnist(batch_size=batch_size, train_valid_test_ratio=[5, 1, 1])
# build model
model = Sequential(input_var=T.matrix(), output_var=T.matrix())
model.add(Linear(prev_dim=28 * 28, this_dim=200))
model.add(RELU())
model.add(Linear(prev_dim=200, this_dim=100))
model.add(RELU())
model.add(Dropout(0.5))
model.add(Linear(prev_dim=100, this_dim=10))
model.add(Softmax())
# build learning method
decay_batch = int(data.train.X.shape[0] * 2 / batch_size)
learning_method = SGD(learning_rate=0.1,
momentum=0.9,
lr_decay_factor=0.9,
decay_batch=decay_batch)
# Build Logger
log = Log(
experiment_name='MLP',
description='This is a tutorial',
save_outputs=True, # log all the outputs from the screen
save_model=True, # save the best model
save_epoch_error=True, # log error at every epoch
save_to_database={
'name': 'Example.sqlite3',
'records': {
'Batch_Size': batch_size,
'Learning_Rate': learning_method.learning_rate,
'Momentum': learning_method.momentum
}
}) # end log
# put everything into the train object
train_object = TrainObject(model=model,
log=log,
dataset=data,
train_cost=mse,
valid_cost=error,
learning_method=learning_method,
stop_criteria={
'max_epoch': 100,
'epoch_look_back': 5,
'percent_decrease': 0.01
})
# finally run the code
train_object.setup()
train_object.run()
ypred = model.fprop(data.get_test().X)
ypred = np.argmax(ypred, axis=1)
y = np.argmax(data.get_test().y, axis=1)
accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
print('test accuracy:', accuracy)
def __init__(self,
train_valid_test_ratio=[8, 1, 1],
batch_size=100,
num_batches=None,
iter_class='SequentialSubsetIterator',
rng=None,
log=None):
'''
DESCRIPTION: Abstract class
PARAMS:
split_mode(sequential | random): sequentially or randomly split the dataset
'''
assert len(train_valid_test_ratio) == 3, 'the size of list is not 3'
self.ratio = train_valid_test_ratio
self.iter_class = iter_class
self.batch_size = batch_size
self.num_batches = num_batches
self.rng = rng
self.log = log
if self.log is None:
# use default Log setting, using the internal logger
self.log = Log(logger=internal_logger)
def __init__(self, model, dataset, train_cost, valid_cost, learning_method, stop_criteria, log=None):
self.model = model
self.dataset = dataset
self.train_cost = train_cost
self.valid_cost = valid_cost
self.learning_method = learning_method
self.stop_criteria = stop_criteria
self.log = log
if self.log is None:
# use default Log setting
self.log = Log(logger=internal_logger)
elif self.log.save_to_database:
self.log.print_records()
self.log.info('\n')
def __init__(self,
train_valid_test_ratio=[8, 1, 1],
log=None,
batch_size=100,
num_batches=None,
iter_class='SequentialSubsetIterator',
rng=None):
assert len(train_valid_test_ratio) == 3, 'the size of list is not 3'
self.ratio = train_valid_test_ratio
self.iter_class = iter_class
self.batch_size = batch_size
self.num_batches = num_batches
self.rng = rng
self.log = log
if self.log is None:
# use default Log setting, using the internal logger
self.log = Log(logger=internal_logger)
def __init__(self, log):
self.log = log
if self.log is None:
# use default Log setting, using the internal logger
self.log = Log(logger=internal_logger)
class TrainObject():
def __init__(self, model, dataset, train_cost, valid_cost, learning_method, stop_criteria, log=None):
self.model = model
self.dataset = dataset
self.train_cost = train_cost
self.valid_cost = valid_cost
self.learning_method = learning_method
self.stop_criteria = stop_criteria
self.log = log
if self.log is None:
# use default Log setting
self.log = Log(logger=internal_logger)
elif self.log.save_to_database:
self.log.print_records()
self.log.info('\n')
def setup(self):
self.log.info( '..begin setting up train object')
#===================[ build params and deltas list ]==================#
params = []
deltas = []
for layer in self.model.layers:
for param in layer.params:
# checked that the param to be updated is shared variable
if is_shared_var(param):
param.name += '_' + layer.__class__.__name__
params += [param]
deltas += [shared_zeros(shape=param.shape.eval())]
#=====================[ training params updates ]=====================#
self.log.info("..update params: " + str(params))
train_y_pred, train_layers_stats = self.model.train_fprop(self.model.input_var)
train_cost = self.train_cost(self.model.output_var, train_y_pred).astype(floatX)
train_updates = []
gparams = T.grad(train_cost, params)
for delta, param, gparam in zip(deltas, params, gparams):
train_updates += self.learning_method.update(delta, gparam)
train_updates += [(param, param+delta)]
#----[ append updates of stats from each layer to train updates ]-----#
self.train_stats_names, train_stats_vars = split_list(train_layers_stats)
train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
self.train_stats_shared = generate_shared_list(train_stats_vars)
train_stats_updates = merge_lists(self.train_stats_shared, train_stats_vars)
train_updates += train_stats_updates
#-------------------------[ train functions ]-------------------------#
self.log.info('..begin compiling functions')
self.training = theano.function(inputs=[self.model.input_var, self.model.output_var],
outputs=train_cost,
updates=train_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..training function compiled')
#=============================[ testing ]=============================#
test_y_pred, test_layers_stats = self.model.test_fprop(self.model.input_var)
#-----[ append updates of stats from each layer to test updates ]-----#
self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
self.test_stats_shared = generate_shared_list(test_stats_vars)
test_stats_updates = merge_lists(self.test_stats_shared, test_stats_vars)
#-------------------------[ test functions ]--------------------------#
test_stopping_error = self.valid_cost(self.model.output_var, test_y_pred).astype(floatX)
test_cost = self.train_cost(self.model.output_var, test_y_pred).astype(floatX)
self.testing = theano.function(inputs=[self.model.input_var, self.model.output_var],
outputs=(test_stopping_error, test_cost),
updates=test_stats_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..testing function compiled')
def run(self):
best_valid_error = float(sys.maxint)
valid_error = float(sys.maxint)
train_cost = float(sys.maxint)
valid_cost = float(sys.maxint)
train_stats_values = []
valid_stats_values = []
epoch = 0
error_dcr = 0
self.best_epoch_last_update = 0
self.best_valid_last_update = float(sys.maxint)
train_stats_names = ['train_' + name for name in self.train_stats_names]
valid_stats_names = ['valid_' + name for name in self.test_stats_names]
job_start = time.time()
while (self.continue_learning(epoch, error_dcr, best_valid_error)):
if epoch > 0:
self.log.info("best_epoch_last_update: %d"%self.best_epoch_last_update)
self.log.info("valid_error_decrease: %f"%error_dcr)
self.log.info("best_valid_last_update: %f"%self.best_valid_last_update)
self.log.info("========[ End of Epoch ]========\n\n")
epoch += 1
start_time = time.time()
num_train_examples = 0
total_train_cost = 0.
train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)
num_valid_examples = 0
total_valid_cost = 0.
total_valid_stopping_cost = 0.
valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)
blk = 0
for block in self.dataset:
block_time = time.time()
blk += 1
train_set = block.get_train()
valid_set = block.get_valid()
#====================[ Training Progress ]====================#
if train_set.dataset_size > 0:
self.log.info('..training '+ self.dataset.__class__.__name__
+ ' block %s/%s'%(blk, self.dataset.nblocks))
progbar = Progbar(target=train_set.dataset_size)
blk_sz = 0
for idx in train_set:
cost = self.training(train_set.X[idx], train_set.y[idx])
total_train_cost += cost * len(idx)
num_train_examples += len(idx)
train_stats_values += len(idx) * get_shared_values(self.train_stats_shared)
blk_sz += len(idx)
progbar.update(blk_sz)
print
#-------[ Update train best cost and error values ]-------#
train_cost = total_train_cost / num_train_examples
train_stats_values /= num_train_examples
#===================[ Validating Progress ]===================#
if valid_set.dataset_size > 0:
self.log.info('..validating ' + self.dataset.__class__.__name__
+ ' block %s/%s'%(blk, self.dataset.nblocks))
progbar = Progbar(target=valid_set.dataset_size)
blk_sz = 0
for idx in valid_set:
stopping_cost, cost = self.testing(valid_set.X[idx], valid_set.y[idx])
total_valid_cost += cost * len(idx)
total_valid_stopping_cost += stopping_cost * len(idx)
num_valid_examples += len(idx)
valid_stats_values += len(idx) * get_shared_values(self.test_stats_shared)
blk_sz += len(idx)
progbar.update(blk_sz)
print
#-------[ Update valid best cost and error values ]-------#
valid_error = total_valid_stopping_cost / num_valid_examples
valid_cost = total_valid_cost / num_valid_examples
valid_stats_values /= num_valid_examples
if valid_error < best_valid_error:
best_valid_error = valid_error
self.log.info('..best validation error so far')
if self.log.save_model:
self.log._save_model(self.model)
self.log.info('..model saved')
if valid_error < self.best_valid_last_update:
error_dcr = self.best_valid_last_update - valid_error
else:
error_dcr = 0
self.log.info('block time: %0.2fs'%(time.time()-block_time))
self.log.info(get_mem_usage())
#==============[ save to database, save epoch error]==============#
if self.log.save_to_database:
self.log._save_to_database(epoch, train_cost, valid_cost, best_valid_error)
self.log.info('..sent to database: %s:%s' % (self.log.save_to_database['name'],
self.log.experiment_name))
if self.log.save_epoch_error:
self.log._save_epoch_error(epoch, valid_error)
self.log.info('..epoch error saved')
end_time = time.time()
#=====================[ log outputs to file ]=====================#
merged_train = merge_lists(train_stats_names, train_stats_values)
merged_valid = merge_lists(valid_stats_names, valid_stats_values)
outputs = [('epoch', epoch),
('runtime(s)', int(end_time-start_time)),
('train_' + self.train_cost.func_name, train_cost),
('valid_' + self.train_cost.func_name, valid_cost),
('valid_' + self.valid_cost.func_name, valid_error),
('best_valid_' + self.valid_cost.func_name, best_valid_error)]
outputs += merged_train + merged_valid
self.log._log_outputs(outputs)
job_end = time.time()
self.log.info('Job Completed on %s'%time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
ttl_time = int(job_end - job_start)
dt = datetime.timedelta(seconds=ttl_time)
self.log.info('Total Time Taken: %s'%str(dt))
self.log.info("========[ End of Job ]========\n\n")
def continue_learning(self, epoch, error_dcr, best_valid_error):
if epoch > self.stop_criteria['max_epoch']:
return False
elif self.stop_criteria['percent_decrease'] is None or \
self.stop_criteria['epoch_look_back'] is None:
return True
elif np.abs(float(error_dcr) / self.best_valid_last_update) \
>= self.stop_criteria['percent_decrease']:
self.best_valid_last_update = best_valid_error
self.best_epoch_last_update = epoch
return True
elif epoch - self.best_epoch_last_update > \
self.stop_criteria['epoch_look_back']:
return False
else:
return True
def train():
batch_size = 256
short_memory = 0.9
learning_rate = 0.005
data = Cifar10(batch_size=batch_size, train_valid_test_ratio=[4, 1, 1])
_, c, h, w = data.train.X.shape
model = Sequential(input_var=T.tensor4(), output_var=T.matrix())
model.add(
Convolution2D(input_channels=c,
filters=8,
kernel_size=(3, 3),
stride=(1, 1),
border_mode='full'))
h, w = full(h, w, kernel=3, stride=1)
model.add(
BatchNormalization(dim=8, layer_type='conv',
short_memory=short_memory))
model.add(RELU())
model.add(
Convolution2D(input_channels=8,
filters=16,
kernel_size=(3, 3),
stride=(1, 1),
border_mode='valid'))
h, w = valid(h, w, kernel=3, stride=1)
model.add(
BatchNormalization(dim=16,
layer_type='conv',
short_memory=short_memory))
model.add(RELU())
model.add(Pooling2D(poolsize=(4, 4), stride=(4, 4), mode='max'))
h, w = valid(h, w, kernel=4, stride=4)
model.add(Flatten())
model.add(Linear(16 * h * w, 512))
model.add(
BatchNormalization(dim=512, layer_type='fc',
short_memory=short_memory))
model.add(RELU())
model.add(Linear(512, 10))
model.add(Softmax())
# learning_method = RMSprop(learning_rate=learning_rate)
learning_method = Adam(learning_rate=learning_rate)
# learning_method = SGD(learning_rate=0.001)
# Build Logger
log = Log(
experiment_name='cifar10_cnn_tutorial',
description='This is a tutorial',
save_outputs=True, # log all the outputs from the screen
save_model=True, # save the best model
save_epoch_error=True, # log error at every epoch
save_to_database={
'name': 'hyperparam.sqlite3',
'records': {
'Batch_Size': batch_size,
'Learning_Rate': learning_method.learning_rate
}
}) # end log
# put everything into the train object
train_object = TrainObject(model=model,
log=log,
dataset=data,
train_cost=entropy,
valid_cost=error,
learning_method=learning_method,
stop_criteria={
'max_epoch': 100,
'epoch_look_back': 10,
'percent_decrease': 0.01
})
# finally run the code
train_object.setup()
train_object.run()
# test the model on test set
ypred = model.fprop(data.get_test().X)
ypred = np.argmax(ypred, axis=1)
y = np.argmax(data.get_test().y, axis=1)
accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
print 'test accuracy:', accuracy
def train():
max_features=20000
maxseqlen = 100 # cut texts after this number of words (among top max_features most common words)
batch_size = 16
word_vec_len = 256
iter_class = 'SequentialRecurrentIterator'
seq_len = 10
data = IMDB(pad_zero=True, maxlen=100, nb_words=max_features, batch_size=batch_size,
train_valid_test_ratio=[8,2,0], iter_class=iter_class, seq_len=seq_len)
print('Build model...')
model = Sequential(input_var=T.matrix(), output_var=T.matrix())
model.add(Embedding(max_features, word_vec_len))
# MLP layers
model.add(Transform((word_vec_len,))) # transform from 3d dimensional input to 2d input for mlp
model.add(Linear(word_vec_len, 100))
model.add(RELU())
model.add(BatchNormalization(dim=100, layer_type='fc'))
model.add(Linear(100,100))
model.add(RELU())
model.add(BatchNormalization(dim=100, layer_type='fc'))
model.add(Linear(100, word_vec_len))
model.add(RELU())
model.add(Transform((maxseqlen, word_vec_len))) # transform back from 2d to 3d for recurrent input
# Stacked up BiLSTM layers
model.add(BiLSTM(word_vec_len, 50, output_mode='concat', return_sequences=True))
model.add(BiLSTM(100, 24, output_mode='sum', return_sequences=True))
model.add(LSTM(24, 24, return_sequences=True))
# MLP layers
model.add(Reshape((24 * maxseqlen,)))
model.add(BatchNormalization(dim=24 * maxseqlen, layer_type='fc'))
model.add(Linear(24 * maxseqlen, 50))
model.add(RELU())
model.add(Dropout(0.2))
model.add(Linear(50, 1))
model.add(Sigmoid())
# build learning method
decay_batch = int(data.train.X.shape[0] * 5 / batch_size)
learning_method = SGD(learning_rate=0.1, momentum=0.9,
lr_decay_factor=1.0, decay_batch=decay_batch)
# Build Logger
log = Log(experiment_name = 'MLP',
description = 'This is a tutorial',
save_outputs = True, # log all the outputs from the screen
save_model = True, # save the best model
save_epoch_error = True, # log error at every epoch
save_to_database = {'name': 'Example.sqlite3',
'records': {'Batch_Size': batch_size,
'Learning_Rate': learning_method.learning_rate,
'Momentum': learning_method.momentum}}
) # end log
# put everything into the train object
train_object = TrainObject(model = model,
log = log,
dataset = data,
train_cost = mse,
valid_cost = error,
learning_method = learning_method,
stop_criteria = {'max_epoch' : 100,
'epoch_look_back' : 5,
'percent_decrease' : 0.01}
)
# finally run the code
train_object.setup()
train_object.run()
def train(args):
# build dataset
xpath = os.environ['MOZI_DATA_PATH'] + '/X_{}_augment_{}.npy'.format(
'_'.join([str(d) for d in _IMG_INPUT_DIM_]), str(img_augment))
ypath = os.environ['MOZI_DATA_PATH'] + '/y_{}_augment_{}.npy'.format(
'_'.join([str(d) for d in _IMG_INPUT_DIM_]), str(img_augment))
if not os.path.exists(xpath) or not os.path.exists(ypath):
X, y = make_Xy(args, img_augment)
with open(xpath, 'wb') as fout:
np.save(fout, X)
print '..saved to', xpath
with open(ypath, 'wb') as fout:
np.save(fout, y)
print '..saved to', ypath
else:
with open(xpath) as xin, open(ypath) as yin:
X = np.load(xin)
y = np.load(yin)
print '..data loaded'
if img_preprocess:
X = img_preprocess.apply(X)
# import pdb; pdb.set_trace()
idxs = np.arange(len(X))
np.random.shuffle(idxs)
data = MultiInputsData(X=X[idxs][:10000],
y=y[idxs][:10000],
train_valid_test_ratio=train_valid_test_ratio,
batch_size=batch_size)
if load_model:
print '..loading model', load_model
model_path = os.environ[
'MOZI_SAVE_PATH'] + '/' + load_model + '/model.pkl'
with open(model_path) as fin:
model = cPickle.load(fin)
else:
# c, h, w = _IMG_INPUT_DIM_
# build the master model
model = Sequential(input_var=T.tensor4(),
output_var=T.tensor4(),
verbose=verbose)
ks = 11
model.add(
Convolution2D(input_channels=3,
filters=16,
kernel_size=(ks, ks),
stride=(1, 1),
border_mode='full'))
model.add(Crop(border=(ks / 2, ks / 2)))
model.add(
BatchNormalization(dim=16,
layer_type='conv',
short_memory=short_memory))
model.add(RELU())
model.add(
Pooling2D(poolsize=(3, 3),
stride=(1, 1),
padding=(1, 1),
mode='max'))
# model.add(RELU())
# h, w = full(h, w, 5, 1)
ks = 9
model.add(
Convolution2D(input_channels=16,
filters=32,
kernel_size=(ks, ks),
stride=(1, 1),
border_mode='full'))
model.add(Crop(border=(ks / 2, ks / 2)))
model.add(
BatchNormalization(dim=32,
layer_type='conv',
short_memory=short_memory))
model.add(RELU())
model.add(
Pooling2D(poolsize=(3, 3),
stride=(1, 1),
padding=(1, 1),
mode='max'))
ks = 5
model.add(
Convolution2D(input_channels=32,
filters=1,
kernel_size=(ks, ks),
stride=(1, 1),
border_mode='full'))
# model.add(BatchNormalization(dim=1, layer_type='conv', short_memory=short_memory))
model.add(Crop(border=(ks / 2, ks / 2)))
model.add(Sigmoid())
# build learning method
# learning_method = SGD(learning_rate=lr, momentum=momentum,
# lr_decay_factor=lr_decay_factor, decay_batch=decay_batch)
learning_method = Adam(learning_rate=lr)
# learning_method = RMSprop(learning_rate=lr)
# Build Logger
log = Log(
experiment_name=experiment_name,
description=desc,
save_outputs=True, # log all the outputs from the screen
save_model=save_model, # save the best model
save_epoch_error=True, # log error at every epoch
save_to_database={
'name': 'skin_segment.sqlite3',
'records': {
'learning_rate': lr,
'valid_cost_func': valid_cost,
'train_cost_func': train_cost
}
}) # end log
os.system('cp {} {}'.format(__file__, log.exp_dir))
dname = os.path.dirname(os.path.realpath(__file__))
# put everything into the train object
train_object = TrainObject(model=model,
log=log,
dataset=data,
train_cost=train_cost,
valid_cost=valid_cost,
learning_method=learning_method,
stop_criteria={
'max_epoch': 100,
'epoch_look_back': 5,
'percent_decrease': 0.01
})
# finally run the code
train_object.setup()
train_object.run()
class TrainObject():
def __init__(self,
model,
dataset,
train_cost,
valid_cost,
learning_method,
stop_criteria,
log=None):
self.model = model
self.dataset = dataset
self.train_cost = train_cost
self.valid_cost = valid_cost
self.learning_method = learning_method
self.stop_criteria = stop_criteria
self.log = log
if self.log is None:
# use default Log setting
self.log = Log(logger=internal_logger)
elif self.log.save_to_database:
self.log.print_records()
self.log.info('\n')
def setup(self):
self.log.info('..begin setting up train object')
#===================[ build params and deltas list ]==================#
params = []
deltas = []
for layer in self.model.layers:
for param in layer.params:
# checked that the param to be updated is shared variable
if is_shared_var(param):
param.name += '_' + layer.__class__.__name__
params += [param]
deltas += [shared_zeros(shape=param.shape.eval())]
#=====================[ training params updates ]=====================#
self.log.info("..update params: " + str(params))
train_y_pred, train_layers_stats = self.model.train_fprop(
self.model.input_var)
train_cost = self.train_cost(self.model.output_var,
train_y_pred).astype(floatX)
train_updates = []
gparams = T.grad(train_cost, params)
for delta, param, gparam in zip(deltas, params, gparams):
train_updates += self.learning_method.update(delta, gparam)
train_updates += [(param, param + delta)]
#----[ append updates of stats from each layer to train updates ]-----#
self.train_stats_names, train_stats_vars = split_list(
train_layers_stats)
train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
self.train_stats_shared = generate_shared_list(train_stats_vars)
train_stats_updates = merge_lists(self.train_stats_shared,
train_stats_vars)
train_updates += train_stats_updates
#-------------------------[ train functions ]-------------------------#
self.log.info('..begin compiling functions')
self.training = theano.function(
inputs=[self.model.input_var, self.model.output_var],
outputs=train_cost,
updates=train_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..training function compiled')
#======================[ testing params updates ]=====================#
test_y_pred, test_layers_stats = self.model.test_fprop(
self.model.input_var)
#-----[ append updates of stats from each layer to test updates ]-----#
self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
self.test_stats_shared = generate_shared_list(test_stats_vars)
test_stats_updates = merge_lists(self.test_stats_shared,
test_stats_vars)
#-------------------------[ test functions ]--------------------------#
test_stopping_error = self.valid_cost(self.model.output_var,
test_y_pred).astype(floatX)
test_cost = self.train_cost(self.model.output_var,
test_y_pred).astype(floatX)
self.testing = theano.function(
inputs=[self.model.input_var, self.model.output_var],
outputs=(test_stopping_error, test_cost),
updates=test_stats_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..testing function compiled')
def run(self):
best_valid_error = float(sys.maxint)
valid_error = float(sys.maxint)
train_cost = float(sys.maxint)
valid_cost = float(sys.maxint)
train_stats_values = []
valid_stats_values = []
epoch = 0
error_dcr = 0
self.best_epoch_last_update = 0
self.best_valid_last_update = float(sys.maxint)
train_stats_names = [
'train_' + name for name in self.train_stats_names
]
valid_stats_names = ['valid_' + name for name in self.test_stats_names]
job_start = time.time()
while (self.continue_learning(epoch, error_dcr, best_valid_error)):
if epoch > 0:
self.log.info("best_epoch_last_update: %d" %
self.best_epoch_last_update)
self.log.info("valid_error_decrease: %f" % error_dcr)
self.log.info("best_valid_last_update: %f" %
self.best_valid_last_update)
self.log.info("========[ End of Epoch ]========\n\n")
epoch += 1
start_time = time.time()
num_train_examples = 0
total_train_cost = 0.
train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)
num_valid_examples = 0
total_valid_cost = 0.
total_valid_stopping_cost = 0.
valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)
blk = 0
for block in self.dataset:
block_time = time.time()
blk += 1
train_set = block.get_train()
valid_set = block.get_valid()
#====================[ Training Progress ]====================#
if train_set.dataset_size > 0:
self.log.info('..training ' +
self.dataset.__class__.__name__ +
' block %s/%s' % (blk, self.dataset.nblocks))
progbar = Progbar(target=train_set.dataset_size)
for idx in train_set:
cost = self.training(train_set.X[idx],
train_set.y[idx])
total_train_cost += cost * len(idx)
num_train_examples += len(idx)
train_stats_values += len(idx) * get_shared_values(
self.train_stats_shared)
progbar.update(num_train_examples)
print
#-------[ Update train best cost and error values ]-------#
train_cost = total_train_cost / num_train_examples
train_stats_values /= num_train_examples
#===================[ Validating Progress ]===================#
if valid_set.dataset_size > 0:
self.log.info('..validating ' +
self.dataset.__class__.__name__ +
' block %s/%s' % (blk, self.dataset.nblocks))
progbar = Progbar(target=valid_set.dataset_size)
for idx in valid_set:
stopping_cost, cost = self.testing(
valid_set.X[idx], valid_set.y[idx])
total_valid_cost += cost * len(idx)
total_valid_stopping_cost += stopping_cost * len(idx)
num_valid_examples += len(idx)
valid_stats_values += len(idx) * get_shared_values(
self.test_stats_shared)
progbar.update(num_valid_examples)
print
#-------[ Update valid best cost and error values ]-------#
valid_error = total_valid_stopping_cost / num_valid_examples
valid_cost = total_valid_cost / num_valid_examples
valid_stats_values /= num_valid_examples
if valid_error < best_valid_error:
best_valid_error = valid_error
self.log.info('..best validation error so far')
if self.log.save_model:
self.log._save_model(self.model)
self.log.info('..model saved')
if valid_error < self.best_valid_last_update:
error_dcr = self.best_valid_last_update - valid_error
else:
error_dcr = 0
self.log.info('block time: %0.2fs' %
(time.time() - block_time))
self.log.info(get_mem_usage())
#==============[ save to database, save epoch error]==============#
if self.log.save_to_database:
self.log._save_to_database(epoch, train_cost, valid_cost,
best_valid_error)
self.log.info('..sent to database: %s:%s' %
(self.log.save_to_database['name'],
self.log.experiment_name))
if self.log.save_epoch_error:
self.log._save_epoch_error(epoch, valid_error)
self.log.info('..epoch error saved')
end_time = time.time()
#=====================[ log outputs to file ]=====================#
merged_train = merge_lists(train_stats_names, train_stats_values)
merged_valid = merge_lists(valid_stats_names, valid_stats_values)
outputs = [('epoch', epoch),
('runtime(s)', int(end_time - start_time)),
('train_' + self.train_cost.func_name, train_cost),
('valid_' + self.train_cost.func_name, valid_cost),
('valid_' + self.valid_cost.func_name, valid_error),
('best_valid_' + self.valid_cost.func_name,
best_valid_error)]
outputs += merged_train + merged_valid
self.log._log_outputs(outputs)
job_end = time.time()
self.log.info(
'Job Completed on %s' %
time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
ttl_time = int(job_end - job_start)
dt = datetime.timedelta(seconds=ttl_time)
self.log.info('Total Time Taken: %s' % str(dt))
self.log.info("========[ End of Job ]========\n\n")
def continue_learning(self, epoch, error_dcr, best_valid_error):
if epoch > self.stop_criteria['max_epoch']:
return False
elif self.stop_criteria['percent_decrease'] is None or \
self.stop_criteria['epoch_look_back'] is None:
return True
elif np.abs(float(error_dcr) / self.best_valid_last_update) \
>= self.stop_criteria['percent_decrease']:
self.best_valid_last_update = best_valid_error
self.best_epoch_last_update = epoch
return True
elif epoch - self.best_epoch_last_update > \
self.stop_criteria['epoch_look_back']:
return False
else:
return True
def train():
data = Cifar10(batch_size=32, train_valid_test_ratio=[4, 1, 1])
model = Sequential(input_var=T.tensor4(), output_var=T.matrix())
model.add(
Convolution2D(input_channels=3,
filters=8,
kernel_size=(3, 3),
stride=(1, 1),
border_mode='full'))
model.add(RELU())
model.add(
Convolution2D(input_channels=8,
filters=16,
kernel_size=(3, 3),
stride=(1, 1)))
model.add(RELU())
model.add(Pooling2D(poolsize=(4, 4), stride=(4, 4), mode='max'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Linear(16 * 8 * 8, 512))
model.add(RELU())
model.add(Dropout(0.5))
model.add(Linear(512, 10))
model.add(Softmax())
# build learning method
learning_method = SGD(learning_rate=0.01,
momentum=0.9,
lr_decay_factor=0.9,
decay_batch=5000)
# Build Logger
log = Log(
experiment_name='cifar10_cnn',
description='This is a tutorial',
save_outputs=True, # log all the outputs from the screen
save_model=True, # save the best model
save_epoch_error=True, # log error at every epoch
save_to_database={
'name': 'hyperparam.sqlite3',
'records': {
'Batch_Size': data.batch_size,
'Learning_Rate': learning_method.learning_rate,
'Momentum': learning_method.momentum
}
}) # end log
# put everything into the train object
train_object = TrainObject(model=model,
log=log,
dataset=data,
train_cost=entropy,
valid_cost=error,
learning_method=learning_method,
stop_criteria={
'max_epoch': 30,
'epoch_look_back': 5,
'percent_decrease': 0.01
})
# finally run the code
train_object.setup()
train_object.run()
# test the model on test set
ypred = model.fprop(data.get_test().X)
ypred = np.argmax(ypred, axis=1)
y = np.argmax(data.get_test().y, axis=1)
accuracy = np.equal(ypred, y).astype('f4').sum() / len(y)
print 'test accuracy:', accuracy