def train(self, dataset, model=None):
"""Trains the passed model on the given dataset. If no model is passed, `generate_default_model` is used."""
print "[%s] Starting training..." % self.model_name
start = time.time()
# The training will be run on the CPU. If a GPU is available it should be used instead.
backend = gen_backend(backend='cpu',
batch_size=self.batch_size,
rng_seed=self.random_seed,
stochastic_round=False)
cost = GeneralizedCost(
name='cost',
costfunc=CrossEntropyMulti())
optimizer = GradientDescentMomentum(
learning_rate=self.lrate,
momentum_coef=0.9)
# set up the model and experiment
if not model:
model = self.generate_default_model(dataset.num_labels)
args = NeonCallbackParameters()
args.output_file = os.path.join(self.root_path, self.Callback_Store_Filename)
args.evaluation_freq = 1
args.progress_bar = False
args.epochs = self.max_epochs
args.save_path = os.path.join(self.root_path, self.Intermediate_Model_Filename)
args.serialize = 1
args.history = 100
args.model_file = None
callbacks = Callbacks(model, dataset.train(), args, eval_set=dataset.test())
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(self.model_path)
# Uncomment line below to run on GPU using cudanet backend
# backend = gen_backend(rng_seed=0, gpu='cudanet')
model.fit(
dataset.train(),
optimizer=optimizer,
num_epochs=self.max_epochs,
cost=cost,
callbacks=callbacks)
print("[%s] Misclassification error = %.1f%%"
% (self.model_name, model.eval(dataset.test(), metric=Misclassification()) * 100))
print "[%s] Finished training!" % self.model_name
end = time.time()
print "[%s] Duration in seconds", end - start
return model
def train_mlp():
"""
Train data and save scaling and network weights and biases to file
to be used by forward prop phase on test data
"""
parser = NeonArgparser(__doc__)
args = parser.parse_args()
logger = logging.getLogger()
logger.setLevel(args.log_thresh)
# hyperparameters
num_epochs = args.epochs
#preprocessor
std_scale = preprocessing.StandardScaler(with_mean=True,with_std=True)
#std_scale = feature_scaler(type='Standardizer',with_mean=True,with_std=True)
#number of non one-hot encoded features, including ground truth
num_feat = 4
# load up the mnist data set
# split into train and tests sets
#load data from csv-files and rescale
#training
traindf = pd.DataFrame.from_csv('data/train.csv')
ncols = traindf.shape[1]
#tmpmat=std_scale.fit_transform(traindf.as_matrix())
#print std_scale.scale_
#print std_scale.mean_
tmpmat = traindf.as_matrix()
#print tmpmat[:,1:num_feat]
tmpmat[:,:num_feat] = std_scale.fit_transform(tmpmat[:,:num_feat])
X_train = tmpmat[:,1:]
y_train = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#validation
validdf = pd.DataFrame.from_csv('data/validate.csv')
ncols = validdf.shape[1]
tmpmat = validdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_valid = tmpmat[:,1:]
y_valid = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#test
testdf = pd.DataFrame.from_csv('data/test.csv')
ncols = testdf.shape[1]
tmpmat = testdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_test = tmpmat[:,1:]
y_test = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
# setup a training set iterator
train_set = CustomDataIterator(X_train, lshape=(X_train.shape[1]), y_c=y_train)
# setup a validation data set iterator
valid_set = CustomDataIterator(X_valid, lshape=(X_valid.shape[1]), y_c=y_valid)
# setup a validation data set iterator
test_set = CustomDataIterator(X_test, lshape=(X_test.shape[1]), y_c=y_test)
# setup weight initialization function
init_norm = Xavier()
# setup model layers
layers = [Affine(nout=X_train.shape[1], init=init_norm, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=X_train.shape[1]/2, init=init_norm, activation=Rectlin()),
Linear(nout=1, init=init_norm)]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=SmoothL1Loss())
# setup optimizer
#schedule
#schedule = ExpSchedule(decay=0.3)
#optimizer = GradientDescentMomentum(0.0001, momentum_coef=0.9, stochastic_round=args.rounding, schedule=schedule)
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1.e-8)
# initialize model object
mlp = Model(layers=layers)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
#evaluate model
print('Evaluation Error = %.4f'%(mlp.eval(valid_set, metric=SmoothL1Metric())))
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# Saving the model
print 'Saving model parameters!'
mlp.save_params("model/homeapp_model.prm")
# Reloading saved model
# This should go in run.py
mlp=Model("model/homeapp_model.prm")
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# save the preprocessor vectors:
np.savez("model/homeapp_preproc", mean=std_scale.mean_, std=std_scale.scale_)
return 1
# define stopping function
# it takes as input a tuple (State,val[t])
# which describes the cumulative validation state (generated by this function)
# and the validation error at time t
# and returns as output a tuple (State', Bool),
# which represents the new state and whether to stop
# Stop if validation error ever increases from epoch to epoch
def stop_func(s, v):
if s is None:
return (v, False)
return (min(v, s), v > s)
# fit and validate
optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
]
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
lunaModel = Model(layers=layers)
modelFileName = 'LUNA16_CADIMI_subset{}.prm'.format(SUBSET)
# If model file exists, then load the it and start from there.
# if (os.path.isfile(modelFileName)):
# lunaModel = Model(modelFileName)
# Nesterov accelerated gradient descent with a learning rate of 0.01, a decay of 10^-3 and a momentum of 0.9
#opt = GradientDescentMomentum(0.01, 0.9, wdecay=0.001, nesterov=True)
opt = Adadelta(decay=0.95, epsilon=1e-6)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args)
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(modelFileName)
lunaModel.fit(train_set,
optimizer=opt,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# Create datasets
X_spec, y_spec, spec_out = filter_dataset(X_train, y_train, cluster)
X_spec_test, y_spec_test, spec_out = filter_dataset(
X_test, y_test, cluster)
spec_out = nout
spec_set = DataIterator(
X_spec, y_spec, nclass=spec_out, lshape=(3, 32, 32))
spec_test = DataIterator(
X_spec_test, y_spec_test, nclass=spec_out, lshape=(3, 32, 32))
# Train the specialist
specialist, opt, cost = spec_net(nout=spec_out, archive_path=gene_path)
callbacks = Callbacks(specialist, spec_set, args, eval_set=spec_test)
callbacks.add_early_stop_callback(early_stop)
callbacks.add_save_best_state_callback(path)
specialist.fit(spec_set, optimizer=opt,
num_epochs=specialist.epoch_index + num_epochs, cost=cost, callbacks=callbacks)
# Print results
print 'Specialist Train misclassification error: ', specialist.eval(spec_set, metric=Misclassification())
print 'Specialist Test misclassification error: ', specialist.eval(spec_test, metric=Misclassification())
print 'Generalist Train misclassification error: ', generalist.eval(spec_set, metric=Misclassification())
print 'Generalist Test misclassification error: ', generalist.eval(spec_test, metric=Misclassification())
# specialists.append(specialist)
save_obj(specialist.serialize(), path)
except:
path = confusion_matrix_name + '_' + clustering_name + '_' + str(num_clusters) + 'clusters/'
print 'Failed for ', path
failed.append(path)
def stopFunc(s, v):
# Stop if validation error ever increases from epoch to epoch
if s is None:
return (v, False)
return (min(v, s), v > s)
# fit and validate
optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# configure callbacks
if args.validation_freq is None:
args.validation_freq = 1
callbacks = Callbacks(mlp,
train_set,
output_file=args.output_file,
valid_set=valid_set,
valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
callbacks.add_early_stop_callback(stopFunc)
callbacks.add_save_best_state_callback(
os.path.join(args.data_dir, "early_stop-best_state.pkl"))
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
validation=False,
remove_history=False,
minimal_set=False,
next_N=3)
valid = HDF5Iterator(filenames,
ndata=(16 * 2014),
validation=True,
remove_history=False,
minimal_set=False,
next_N=1)
out1, out2, out3 = model.layers.get_terminal()
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True)),
GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True)),
GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))])
schedule = ExpSchedule(decay=(1.0 / 50)) # halve the learning rate every 50 epochs
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding,
gradient_clip_value=1,
gradient_clip_norm=5,
wdecay=0.0001,
schedule=schedule)
callbacks = Callbacks(model, eval_set=valid, metric=TopKMisclassification(5), **args.callback_args)
callbacks.add_save_best_state_callback(os.path.join(args.workspace_dir, "best_state_h5resnet.pkl"))
model.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
model.save_params(os.path.join(args.workspace_dir, "final_state_h5resnet.pkl"))
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# setup optimizer
optimizer = GradientDescentMomentum(0.1,
momentum_coef=0.9,
stochastic_round=args.rounding)
# initialize model object
mlp = Model(layers=layers)
path = "./best_state_" + str(i) + ".prm"
# configure callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(path)
# print "epocs", args.epochs
# run fit
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=15,
cost=cost,
callbacks=callbacks)
result_float = (mlp.eval(valid_set, metric=Misclassification())) * 100
result_string = '%.1f%%' % result_float
print(result_string[:-1])
# print('Misclassification error = %.1f%%' % r)
def train(self,
content,
targets,
test_content=None,
test_targets=None,
has_features=False,
serialize=0,
save_path=None,
learning_rate=0.001,
epochs=5):
"""
:param content: numerical content returned from gen_training_set
:param targets: numerical targets returned from gen_training_set
:param test_content: separate test set to be used for evaluation
:param test_targets: ""
:param test_features: ""
:param features: features arte a list of float lists that can also be in string form, but will be converted
to arrays of floats that must be of the same length as the features specified when creating the classifier.
:param receiver_address: deprecated in favor of features, but left in for testing
:param serialize:
:param save_path:
:param model_file:
:param holdout_pct:
:param learning_rate:
:param epochs:
:return:
"""
multicost = self.exclusive_classes is not None and self.overlapping_classes is not None
if multicost:
metric = MultiMetric(Misclassification(), 0)
elif self.overlapping_classes is None:
metric = Misclassification()
else:
metric = AverageLogLoss()
print('Training neural networks on {} samples for {} epochs.'.format(
len(targets[0]), epochs))
if not test_content is None and not test_targets is None:
valid = BatchIterator(test_content,
targets=test_targets,
steps=[1] if has_features else [1, 1])
else:
valid = None
train = BatchIterator(content,
targets=targets,
steps=[1] if has_features else [1, 1])
callbacks = Callbacks(self.neuralnet,
train_set=train,
multicost=multicost,
metric=metric,
eval_freq=None if valid is None else 1,
eval_set=valid)
#save_path=save_path, serialize=serialize)
if serialize:
callbacks.add_save_best_state_callback(save_path)
if not valid is None:
if self.exclusive_classes is not None:
print('Starting misclassification error = {:.03}%'.format(
self.neuralnet.eval(valid, metric)[0] * 100))
callbacks.add_callback(MisclassificationTest(valid, metric))
else:
print('Starting average logloss = {:.04}'.format(
self.neuralnet.eval(valid, metric)[0]))
callbacks.add_callback(LogLossTest(valid, metric))
if hasattr(self.optimizer, 'learning_rate'):
self.optimizer.learning_rate = learning_rate
print('learning rate = {}'.format(learning_rate))
self.fit(train, self.optimizer, epochs, callbacks)
#print(layer.name + ", " + params['config']['name'])
layer.load_weights(params, load_states=True)
if args.model_file:
import os
assert os.path.exists(args.model_file), '%s not found' % args.model_file
lunaModel.load_params(args.model_file)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args)
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(
'LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm')
if args.deconv:
callbacks.add_deconv_callback(train_set, valid_set)
lunaModel.fit(train_set,
optimizer=opt,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
lunaModel.save_params('LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm')
neon_logger.display(
'Calculating metrics on the test set. This could take a while...')
neon_logger.display('Misclassification error (test) = {:.2f}%'.format(
# configure callbacks
if not args.neon_progress:
args.callback_args['progress_bar'] = False
callbacks = Callbacks(model,
eval_set=test,
metric=metric,
**args.callback_args)
if not args.neon_progress:
callbacks.add_callback(EMEpochCallback(
args.callback_args['eval_freq'], train.nmacrobatches),
insert_pos=None)
# xxx - thought of making this an option but not clear that it slows anything down?
#callbacks.add_hist_callback() # xxx - not clear what information this conveys
if args.save_best_path:
callbacks.add_save_best_state_callback(args.save_best_path)
model.fit(train,
optimizer=opt,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
print('Model training complete for %d epochs!' % (args.epochs, ))
#test.stop(); train.stop()
elif args.write_output:
# write_output mode, must have model loaded
if args.data_config:
test = EMDataIterator(args.data_config,
write_output=args.write_output,
# configure cost and test metrics
cost = GeneralizedCost(costfunc=(CrossEntropyBinary() \
if train.parser.independent_labels else CrossEntropyMulti()))
metric = EMMetric(oshape=test.parser.oshape, use_softmax=not train.parser.independent_labels) if test else None
# configure callbacks
if not args.neon_progress:
args.callback_args['progress_bar'] = False
callbacks = Callbacks(model, eval_set=test, metric=metric, **args.callback_args)
if not args.neon_progress:
callbacks.add_callback(EMEpochCallback(args.callback_args['eval_freq'],train.nmacrobatches),insert_pos=None)
# xxx - thought of making this an option but not clear that it slows anything down?
#callbacks.add_hist_callback() # xxx - not clear what information this conveys
if args.save_best_path:
callbacks.add_save_best_state_callback(args.save_best_path)
model.fit(train, optimizer=opt, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print('Model training complete for %d epochs!' % (args.epochs,))
#test.stop(); train.stop()
elif args.write_output:
# write_output mode, must have model loaded
if args.data_config:
test = EMDataIterator(args.data_config, write_output=args.write_output,
chunk_skip_list=args.chunk_skip_list, dim_ordering=args.dim_ordering,
batch_range=args.test_range, name='test', isTest=True, concatenate_batches=False,
NBUF=args.nbebuf, image_in_size=args.image_in_size)
if hasattr(model,'batch_meta'):
test.parser.batch_meta['prior_train_count'] = model.batch_meta['prior_train_count']
