def train_mlp():
"""
Train data and save scaling and network weights and biases to file
to be used by forward prop phase on test data
"""
parser = NeonArgparser(__doc__)
args = parser.parse_args()
logger = logging.getLogger()
logger.setLevel(args.log_thresh)
# hyperparameters
num_epochs = args.epochs
#preprocessor
std_scale = preprocessing.StandardScaler(with_mean=True,with_std=True)
#std_scale = feature_scaler(type='Standardizer',with_mean=True,with_std=True)
#number of non one-hot encoded features, including ground truth
num_feat = 4
# load up the mnist data set
# split into train and tests sets
#load data from csv-files and rescale
#training
traindf = pd.DataFrame.from_csv('data/train.csv')
ncols = traindf.shape[1]
#tmpmat=std_scale.fit_transform(traindf.as_matrix())
#print std_scale.scale_
#print std_scale.mean_
tmpmat = traindf.as_matrix()
#print tmpmat[:,1:num_feat]
tmpmat[:,:num_feat] = std_scale.fit_transform(tmpmat[:,:num_feat])
X_train = tmpmat[:,1:]
y_train = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#validation
validdf = pd.DataFrame.from_csv('data/validate.csv')
ncols = validdf.shape[1]
tmpmat = validdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_valid = tmpmat[:,1:]
y_valid = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#test
testdf = pd.DataFrame.from_csv('data/test.csv')
ncols = testdf.shape[1]
tmpmat = testdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_test = tmpmat[:,1:]
y_test = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
# setup a training set iterator
train_set = CustomDataIterator(X_train, lshape=(X_train.shape[1]), y_c=y_train)
# setup a validation data set iterator
valid_set = CustomDataIterator(X_valid, lshape=(X_valid.shape[1]), y_c=y_valid)
# setup a validation data set iterator
test_set = CustomDataIterator(X_test, lshape=(X_test.shape[1]), y_c=y_test)
# setup weight initialization function
init_norm = Xavier()
# setup model layers
layers = [Affine(nout=X_train.shape[1], init=init_norm, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=X_train.shape[1]/2, init=init_norm, activation=Rectlin()),
Linear(nout=1, init=init_norm)]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=SmoothL1Loss())
# setup optimizer
#schedule
#schedule = ExpSchedule(decay=0.3)
#optimizer = GradientDescentMomentum(0.0001, momentum_coef=0.9, stochastic_round=args.rounding, schedule=schedule)
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1.e-8)
# initialize model object
mlp = Model(layers=layers)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
#evaluate model
print('Evaluation Error = %.4f'%(mlp.eval(valid_set, metric=SmoothL1Metric())))
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# Saving the model
print 'Saving model parameters!'
mlp.save_params("model/homeapp_model.prm")
# Reloading saved model
# This should go in run.py
mlp=Model("model/homeapp_model.prm")
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# save the preprocessor vectors:
np.savez("model/homeapp_preproc", mean=std_scale.mean_, std=std_scale.scale_)
return 1
# define stopping function
# it takes as input a tuple (State,val[t])
# which describes the cumulative validation state (generated by this function)
# and the validation error at time t
# and returns as output a tuple (State', Bool),
# which represents the new state and whether to stop
# Stop if validation error ever increases from epoch to epoch
def stop_func(s, v):
if s is None:
return (v, False)
return (min(v, s), v > s)
# fit and validate
optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
def stopFunc(s, v):
# Stop if validation error ever increases from epoch to epoch
if s is None:
return (v, False)
return (min(v, s), v > s)
# fit and validate
optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# configure callbacks
if args.validation_freq is None:
args.validation_freq = 1
callbacks = Callbacks(mlp,
train_set,
output_file=args.output_file,
valid_set=valid_set,
valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
callbacks.add_early_stop_callback(stopFunc)
callbacks.add_save_best_state_callback(
os.path.join(args.data_dir, "early_stop-best_state.pkl"))
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
path = EXPERIMENT_DIR + confusion_matrix_name + '_' + clustering_name + '_' + str(num_clusters) + 'clusters/' + 'specialist' + '_' + str(i) + '.prm'
# Create datasets
X_spec, y_spec, spec_out = filter_dataset(X_train, y_train, cluster)
X_spec_test, y_spec_test, spec_out = filter_dataset(
X_test, y_test, cluster)
spec_out = nout
spec_set = DataIterator(
X_spec, y_spec, nclass=spec_out, lshape=(3, 32, 32))
spec_test = DataIterator(
X_spec_test, y_spec_test, nclass=spec_out, lshape=(3, 32, 32))
# Train the specialist
specialist, opt, cost = spec_net(nout=spec_out, archive_path=gene_path)
callbacks = Callbacks(specialist, spec_set, args, eval_set=spec_test)
callbacks.add_early_stop_callback(early_stop)
callbacks.add_save_best_state_callback(path)
specialist.fit(spec_set, optimizer=opt,
num_epochs=specialist.epoch_index + num_epochs, cost=cost, callbacks=callbacks)
# Print results
print 'Specialist Train misclassification error: ', specialist.eval(spec_set, metric=Misclassification())
print 'Specialist Test misclassification error: ', specialist.eval(spec_test, metric=Misclassification())
print 'Generalist Train misclassification error: ', generalist.eval(spec_set, metric=Misclassification())
print 'Generalist Test misclassification error: ', generalist.eval(spec_test, metric=Misclassification())
# specialists.append(specialist)
save_obj(specialist.serialize(), path)
except:
path = confusion_matrix_name + '_' + clustering_name + '_' + str(num_clusters) + 'clusters/'
print 'Failed for ', path
failed.append(path)
from neon.models import Model
mlp = Model(layers=layers)
# set up a cost function
from neon.layers import GeneralizedCost
from neon.transforms import CrossEntropyBinary, SumSquared, LogLoss
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# optimizer
from neon.optimizers import RMSProp, GradientDescentMomentum
optimizer = RMSProp()
# progress bar
from neon.callbacks.callbacks import Callbacks, MetricCallback, EarlyStopCallback, SerializeModelCallback, DummyClass, CollectWeightsCallback
# callbacks = Callbacks(model = mlp , eval_set = train, eval_freq = 1, output_file = "model_stats/some_data_1")
callbacks = Callbacks(mlp, eval_set=TRAIN, eval_freq=1)
#callbacks.add_callback(SerializeModelCallback('right_here.data',1,10))
#callbacks.add_callback(CollectWeightsCallback('weight_history'))
callbacks.add_early_stop_callback(slow_change)
# callbacks.add_callback(MetricCallback(train,metric = LogLoss()))
# callbacks.add_callback(MetricCallback(eval_set=train, metric=LogLoss(), epoch_freq=1))
# callbacks.add_callback(LossCallback(eval_set=test, epoch_freq=1))
# fit the model
mlp.fit(TRAIN,
optimizer=optimizer,
num_epochs=100000000,
cost=cost,
callbacks=callbacks) # what is this args.epochs business?
# test the model on test set
results = mlp.get_outputs(TRAIN)
# evaluate the model on test_set using the log loss metric
train_error = mlp.eval(TRAIN, metric=LogLoss())
average_train_cost += train_error
#test_error = mlp.eval(test, metric=LogLoss())
