def train_eval(
train_set,
valid_set,
args,
hidden_size = 100,
clip_gradients = True,
gradient_limit = 5):
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
layers = [
LSTM(hidden_size, init, Logistic(), Tanh()),
LSTM(hidden_size, init, Logistic(), Tanh()),
Affine(2, init, bias=init, activation=Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
model = Model(layers=layers)
optimizer = RMSProp(clip_gradients=clip_gradients, gradient_limit=gradient_limit, stochastic_round=args.rounding)
# configure callbacks
callbacks = Callbacks(model, train_set, progress_bar=args.progress_bar)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
pred = model.get_outputs(valid_set)
pred_neg_rate = model.eval(valid_set, metric=Misclassification())
return (pred[:,1], pred_neg_rate)
def run(args, train, test):
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = [Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=True),
Affine(nout=10, init=init_uni, activation=Softmax())]
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
callbacks = Callbacks(mlp, train, eval_set=test, **args.callback_args)
mlp.fit(train, optimizer=opt_gdm, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
err = mlp.eval(test, metric=Misclassification())*100
print('Misclassification error = %.2f%%' % err)
return err
class MostCommonWordSense:
def __init__(self, rounding, callback_args, epochs):
# setup weight initialization function
self.init = Gaussian(loc=0.0, scale=0.01)
# setup optimizer
self.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9,
stochastic_round=rounding)
# setup cost function as CrossEntropy
self.cost = GeneralizedCost(costfunc=SumSquared())
self.epochs = epochs
self.model = None
self.callback_args = callback_args
def build(self):
# setup model layers
layers = [Affine(nout=100, init=self.init, bias=self.init, activation=Rectlin()),
Affine(nout=2, init=self.init, bias=self.init, activation=Softmax())]
# initialize model object
self.model = Model(layers=layers)
def fit(self, valid_set, train_set):
# configure callbacks
callbacks = Callbacks(self.model, eval_set=valid_set, **self.callback_args)
self.model.fit(train_set, optimizer=self.optimizer, num_epochs=self.epochs,
cost=self.cost, callbacks=callbacks)
def save(self, save_path):
self.model.save_params(save_path)
def load(self, model_path):
self.model = Model(model_path)
def eval(self, valid_set):
eval_rate = self.model.eval(valid_set, metric=Misclassification())
return eval_rate
def get_outputs(self, valid_set):
return self.model.get_outputs(valid_set)
def run(args, train, test):
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = [
Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=True),
Affine(nout=10, init=init_uni, activation=Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
callbacks = Callbacks(mlp, eval_set=test, **args.callback_args)
mlp.fit(train,
optimizer=opt_gdm,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
err = mlp.eval(test, metric=Misclassification()) * 100
print('Misclassification error = %.2f%%' % err)
return err
# define layers
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=init_emb,
pad_idx=0, update=embedding_update),
LSTM(hidden_size, init_glorot, activation=Tanh(), gate_activation=Logistic(),
reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(nclass, init_glorot, bias=init_glorot, activation=Softmax())
]
# set the cost, metrics, optimizer
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "\nTrain Accuracy -", 100 * model.eval(train_set, metric=metric)
print "Test Accuracy -", 100 * model.eval(valid_set, metric=metric)
test = DataIterator(X_test, y_test, nclass=nclass)
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
# set up the model layers
layers = []
layers.append(Affine(nout=200, init=init_uni, activation=Rectlin()))
layers.append(
Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)))
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp,
train,
output_file=args.output_file,
valid_set=test,
valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
mlp.fit(train,
optimizer=opt_gdm,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
print mlp.eval(test, metric=Misclassification())
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------
from neon.util.argparser import NeonArgparser
from neon.util.persist import load_obj
from neon.transforms import Misclassification
from neon.models import Model
from neon.data import ImageLoader
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
args = parser.parse_args()
# setup data provider
test_set = ImageLoader(set_name='validation',
repo_dir=args.data_dir,
inner_size=32,
scale_range=40,
do_transforms=False)
model = Model(load_obj(args.model_file))
print 'Accuracy: %.1f %% (Top-1)' % (
1.0 - model.eval(test_set, metric=Misclassification()) * 100)
Affine(nout=16, linear_name="b1_l1", **normrelu),
Affine(nout=10, linear_name="b1_l2", **normsigm)]
p3 = [b2,
Affine(nout=16, linear_name="b2_l1", **normrelu),
Affine(nout=10, linear_name="b2_l2", **normsigm)]
# setup cost function as CrossEntropy
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCost(costfunc=CrossEntropyBinary()),
GeneralizedCost(costfunc=CrossEntropyBinary())],
weights=[1, 0., 0.])
# setup optimizer
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding)
# initialize model object
alphas = [1, 0.25, 0.25]
mlp = Model(layers=Tree([p1, p2, p3], alphas=alphas))
# setup standard fit callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
logging.getLogger('neon').info("Misclassification error = %.1f%%",
(mlp.eval(valid_set, metric=Misclassification())*100))
print('Misclassification error = %.1f%%' % (mlp.eval(valid_set, metric=Misclassification())*100))
valmetric = TopKMisclassification(k=5)
# dummy optimizer for benchmarking
# training implementation coming soon
opt_gdm = GradientDescentMomentum(0.0, 0.0)
opt_biases = GradientDescentMomentum(0.0, 0.0)
opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases})
# setup cost function as CrossEntropy
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCost(costfunc=CrossEntropyMulti())],
weights=[1, 0., 0.]) # We only want to consider the CE of the main path
assert os.path.exists(args.model_file), 'script requires the trained weights file'
model.load_params(args.model_file)
model.initialize(test, cost)
print 'running speed benchmark...'
model.benchmark(test, cost, opt)
print '\nCalculating performance on validation set...'
test.reset()
mets = model.eval(test, metric=valmetric)
print 'Validation set metrics:'
print 'LogLoss: %.2f, Accuracy: %.1f %% (Top-1), %.1f %% (Top-5)' % (mets[0],
(1.0-mets[1])*100,
(1.0-mets[2])*100)
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=init_emb),
LSTM(hidden_size, init_glorot, activation=Tanh(),
gate_activation=Logistic(), reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, init_glorot, bias=init_glorot, activation=Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01, clip_gradients=clip_gradients)
# configure callbacks
callbacks = Callbacks(model, train_set, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "Test Accuracy - ", 100 * model.eval(valid_set, metric=metric)
print "Train Accuracy - ", 100 * model.eval(train_set, metric=metric)
common_params = dict(sampling_freq=22050, clip_duration=16000, frame_duration=16)
train_params = AudioParams(**common_params)
valid_params = AudioParams(**common_params)
common = dict(target_size=1, nclasses=10, repo_dir=args.data_dir)
train = DataLoader(set_name='music-train', media_params=train_params,
index_file=train_idx, shuffle=True, **common)
valid = DataLoader(set_name='music-valid', media_params=valid_params,
index_file=valid_idx, shuffle=False, **common)
init = Gaussian(scale=0.01)
layers = [Conv((2, 2, 4), init=init, activation=Rectlin(),
strides=dict(str_h=2, str_w=4)),
Pooling(2, strides=2),
Conv((3, 3, 4), init=init, batch_norm=True, activation=Rectlin(),
strides=dict(str_h=1, str_w=2)),
DeepBiRNN(128, init=GlorotUniform(), batch_norm=True, activation=Rectlin(),
reset_cells=True, depth=3),
RecurrentMean(),
Affine(nout=common['nclasses'], init=init, activation=Softmax())]
model = Model(layers=layers)
opt = Adagrad(learning_rate=0.01, gradient_clip_value=15)
metric = Misclassification()
callbacks = Callbacks(model, eval_set=valid, metric=metric, **args.callback_args)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
print('Misclassification error = %.1f%%' % (model.eval(valid, metric=metric)*100))
display(model, ['Convolution_0'], 'inputs')
display(model, ['Convolution_0', 'Convolution_1', 'Pooling_0'], 'outputs')
rlayer = DeepBiRNN(hidden_size, g_uni, activation=Tanh(),
depth=1, reset_cells=reset_cells, batch_norm=False)
elif args.rlayer_type == 'bibnrnn':
rlayer = DeepBiRNN(hidden_size, g_uni, activation=Tanh(),
depth=1, reset_cells=reset_cells, batch_norm=True)
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=uni),
rlayer,
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, g_uni, bias=g_uni, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = Adagrad(learning_rate=0.01,
gradient_clip_value=gradient_clip_value)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set, optimizer=optimizer,
num_epochs=args.epochs, cost=cost, callbacks=callbacks)
# eval model
neon_logger.display("Train Accuracy - {}".format(100 * model.eval(train_set, metric=Accuracy())))
neon_logger.display("Test Accuracy - {}".format(100 * model.eval(valid_set, metric=Accuracy())))
init=init_uni,
bias=init_cst,
padding=0,
activation=Rectlin()))
layers.append(Pooling(fshape=2, strides=2))
layers.append(Affine(nout=2, init=init_uni, activation=Softmax()))
from neon.models import Model
model = Model(layers)
from neon.layers import GeneralizedCost
from neon.transforms import CrossEntropyBinary
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
from neon.optimizers import GradientDescentMomentum
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9)
from neon.callbacks.callbacks import Callbacks
callbacks = Callbacks(model, train_set)
model.fit(dataset=train_set,
cost=cost,
optimizer=optimizer,
num_epochs=10,
callbacks=callbacks)
from neon.transforms import Misclassification
error_pct = 100 * model.eval(test_set, metric=Misclassification())
accuracy_fp = 100 - error_pct
print 'Model Accuracy : %.1f%%' % accuracy_fp
rlayer = DeepBiLSTM(hidden_size, g_uni, activation=Tanh(), depth=1,
gate_activation=Logistic(), reset_cells=True)
elif args.rlayer_type == 'rnn':
rlayer = Recurrent(hidden_size, g_uni, activation=Tanh(), reset_cells=True)
elif args.rlayer_type == 'birnn':
rlayer = DeepBiRNN(hidden_size, g_uni, activation=Tanh(), depth=1, reset_cells=True)
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=uni),
rlayer,
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, g_uni, bias=g_uni, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = Adagrad(learning_rate=0.01, gradient_clip_value=gradient_clip_value)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
# eval model
print "Train Accuracy - ", 100 * model.eval(train_set, metric=Accuracy())
print "Test Accuracy - ", 100 * model.eval(valid_set, metric=Accuracy())
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
elif args.datatype in [np.float16]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = [Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=True),
Pooling((2, 2)),
Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=True),
Affine(nout=10, init=init_uni, activation=Softmax())]
if args.datatype in [np.float32, np.float64]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
elif args.datatype in [np.float16]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti(scale=cost_scale))
model = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(model, eval_set=test, **args.callback_args)
# callbacks = Callbacks.load_callbacks(callbacks.get_description(), model, data=[train, test])
model.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
error_rate = model.eval(test, metric=Misclassification())
neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100))
init_glorot,
activation=Tanh(),
gate_activation=Logistic(),
reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(nclass, init_glorot, bias=init_glorot, activation=Softmax())
]
# set the cost, metrics, optimizer
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
neon_logger.display("Train Accuracy - {}".format(
100 * model.eval(train_set, metric=metric)))
neon_logger.display("Test Accuracy - {}".format(
100 * model.eval(valid_set, metric=metric)))
lunaModel = Model(layers=vgg_layers)
if args.model_file:
import os
assert os.path.exists(args.model_file), '%s not found' % args.model_file
lunaModel.load_params(args.model_file)
# configure callbacks
#callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args)
callbacks = Callbacks(lunaModel, eval_set=valid_set, metric=Misclassification(), **args.callback_args)
if args.deconv:
callbacks.add_deconv_callback(train_set, valid_set)
lunaModel.fit(train_set, optimizer=opt, num_epochs=num_epochs,
cost=cost, callbacks=callbacks)
lunaModel.save_params('LUNA16_VGG_model.prm')
neon_logger.display('Finished training. Calculating error on the validation set...')
neon_logger.display('Misclassification error (validation) = {:.2f}%'.format(lunaModel.eval(valid_set, metric=Misclassification())[0] * 100))
neon_logger.display('Precision/recall (validation) = {}'.format(lunaModel.eval(valid_set, metric=PrecisionRecall(num_classes=2))))
neon_logger.display('Calculating metrics on the test set. This could take a while...')
neon_logger.display('Misclassification error (test) = {:.2f}%'.format(lunaModel.eval(test_set, metric=Misclassification())[0] * 100))
neon_logger.display('Precision/recall (test) = {}'.format(lunaModel.eval(test_set, metric=PrecisionRecall(num_classes=2))))
momentum_coef=0.9,
wdecay=0.0005)
opt_b = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
opt = MultiOptimizer({'default': opt_w, 'Bias': opt_b}, name='multiopt')
# configure callbacks
callbacks = Callbacks(model,
eval_set=valid_set,
metric=Misclassification(),
**args.callback_args)
callbacks.add_callback(
TrainByStageCallback(model, valid_set, Misclassification(),
max_patience=5))
num_prune = [5, 10, 25, 10]
callbacks.add_callback(
FuzzyPruneCallback(num_states=100, num_prune=num_prune, model=model))
print('Original Accuracy = %.2f%%' %
(100. - model.eval(valid_set, metric=Misclassification()) * 100))
logger.info('Training ...')
model.fit(train_set,
optimizer=opt,
num_epochs=250,
cost=cost,
callbacks=callbacks)
print('Accuracy = %.2f%%' %
(100. - model.eval(valid_set, metric=Misclassification()) * 100))
model.save_params('./models/mnist/mnistfp.pkl')
from neon.optimizers import GradientDescentMomentum, RMSProp
optimizer = GradientDescentMomentum(learning_rate=0.005,
momentum_coef=0.9)
# Set up callbacks. By default sets up a progress bar
from neon.callbacks.callbacks import Callbacks
callbacks = Callbacks(model, train_set)
model.fit(dataset=train_set, cost=cost, optimizer=optimizer, num_epochs=num_epochs, callbacks=callbacks)
model.save_params("cifar10_model.prm")
# Evaluate performance
from neon.transforms import Misclassification
error_pct = 100 * model.eval(test_set, metric=Misclassification())
print 'Misclassification error = %.1f%%' % error_pct
# Sanity check 1
# an image of a frog from wikipedia
# img_source = "https://upload.wikimedia.org/wikipedia/commons/thumb/5/55/Atelopus_zeteki1.jpg/440px-Atelopus_zeteki1.jpg"
# import urllib
# urllib.urlretrieve(img_source, filename="image.jpg")
from PIL import Image
import numpy as np
# To view images, install Imagemagic
def show_sample(x):
image = x.reshape(3, 32, 32)
# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()
dataset = CIFAR10(path=args.data_dir,
normalize=True,
contrast_normalize=False,
whiten=False)
train = dataset.train_iter
test = dataset.valid_iter
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
# set up the model layers
layers = [Affine(nout=200, init=init_uni, activation=Rectlin()),
Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True))]
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, eval_set=test, **args.callback_args)
mlp.fit(train, optimizer=opt_gdm, num_epochs=args.epochs,
cost=cost, callbacks=callbacks)
neon_logger.display('Misclassification error = %.1f%%' %
(mlp.eval(test, metric=Misclassification()) * 100))
args = parser.parse_args()
train_idx, val_idx = create_index_files(args.data_dir)
common_params = dict(sampling_freq=22050, clip_duration=31000, frame_duration=20)
train_params = AudioParams(random_scale_percent=5, **common_params)
val_params = AudioParams(**common_params)
common = dict(target_size=1, nclasses=10, repo_dir=args.data_dir)
train = DataLoader(set_name='genres-train', media_params=train_params,
index_file=train_idx, shuffle=True, **common)
val = DataLoader(set_name='genres-val', media_params=val_params,
index_file=val_idx, shuffle=False, **common)
init = Gaussian(scale=0.01)
layers = [Conv((5, 5, 64), init=init, activation=Rectlin(),
strides=dict(str_h=2, str_w=4)),
Pooling(2, strides=2),
Conv((5, 5, 64), init=init, batch_norm=True, activation=Rectlin(),
strides=dict(str_h=1, str_w=2)),
BiRNN(256, init=init, activation=Rectlin(), reset_cells=True),
RecurrentMean(),
Affine(128, init=init, batch_norm=True, activation=Rectlin()),
Affine(nout=common['nclasses'], init=init, activation=Softmax())]
model = Model(layers=layers)
opt = Adadelta()
metric = Misclassification()
callbacks = Callbacks(model, eval_set=val, metric=metric, **args.callback_args)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
print('Misclassification error = %.1f%%' % (model.eval(val, metric=metric)*100))
def main():
# Get command-line parameters
parser = get_p1b2_parser()
args = parser.parse_args()
#print('Args:', args)
# Get parameters from configuration file
fileParameters = p1b2.read_config_file(args.config_file)
#print ('Params:', fileParameters)
# Correct for arguments set by default by neon parser
# (i.e. instead of taking the neon parser default value fall back to the config file,
# if effectively the command-line was used, then use the command-line value)
# This applies to conflictive parameters: batch_size, epochs and rng_seed
if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv):
args.batch_size = fileParameters['batch_size']
if not any("--epochs" in ag or "-e" in ag for ag in sys.argv):
args.epochs = fileParameters['epochs']
if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv):
args.rng_seed = fileParameters['rng_seed']
# Consolidate parameter set. Command-line parameters overwrite file configuration
gParameters = p1_common.args_overwrite_config(args, fileParameters)
print('Params:', gParameters)
# Determine verbosity level
loggingLevel = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=loggingLevel, format='')
# Construct extension to save model
ext = p1b2.extension_from_parameters(gParameters, '.neon')
# Get default parameters for initialization and optimizer functions
kerasDefaults = p1_common.keras_default_config()
seed = gParameters['rng_seed']
# Load dataset
#(X_train, y_train), (X_test, y_test) = p1b2.load_data(gParameters, seed)
(X_train, y_train), (X_val,
y_val), (X_test,
y_test) = p1b2.load_data(gParameters, seed)
print("Shape X_train: ", X_train.shape)
print("Shape X_val: ", X_val.shape)
print("Shape X_test: ", X_test.shape)
print("Shape y_train: ", y_train.shape)
print("Shape y_val: ", y_val.shape)
print("Shape y_test: ", y_test.shape)
print("Range X_train --> Min: ", np.min(X_train), ", max: ",
np.max(X_train))
print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))
print("Range y_train --> Min: ", np.min(y_train), ", max: ",
np.max(y_train))
print("Range y_val --> Min: ", np.min(y_val), ", max: ", np.max(y_val))
print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test))
input_dim = X_train.shape[1]
num_classes = int(np.max(y_train)) + 1
output_dim = num_classes # The backend will represent the classes using one-hot representation (but requires an integer class as input !)
# Re-generate the backend after consolidating parsing and file config
gen_backend(backend=args.backend,
rng_seed=seed,
device_id=args.device_id,
batch_size=gParameters['batch_size'],
datatype=gParameters['data_type'],
max_devices=args.max_devices,
compat_mode=args.compat_mode)
train = ArrayIterator(X=X_train, y=y_train, nclass=num_classes)
val = ArrayIterator(X=X_val, y=y_val, nclass=num_classes)
test = ArrayIterator(X=X_test, y=y_test, nclass=num_classes)
# Initialize weights and learning rule
initializer_weights = p1_common_neon.build_initializer(
gParameters['initialization'], kerasDefaults, seed)
initializer_bias = p1_common_neon.build_initializer(
'constant', kerasDefaults, 0.)
activation = p1_common_neon.get_function(gParameters['activation'])()
# Define MLP architecture
layers = []
reshape = None
for layer in gParameters['dense']:
if layer:
layers.append(
Affine(nout=layer,
init=initializer_weights,
bias=initializer_bias,
activation=activation))
if gParameters['dropout']:
layers.append(Dropout(keep=(1 - gParameters['dropout'])))
layers.append(
Affine(nout=output_dim,
init=initializer_weights,
bias=initializer_bias,
activation=activation))
# Build MLP model
mlp = Model(layers=layers)
# Define cost and optimizer
cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])())
optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
callbacks = Callbacks(mlp, eval_set=val, metric=Accuracy(), eval_freq=1)
# Seed random generator for training
np.random.seed(seed)
mlp.fit(train,
optimizer=optimizer,
num_epochs=gParameters['epochs'],
cost=cost,
callbacks=callbacks)
# model save
#save_fname = "model_mlp_W_" + ext
#mlp.save_params(save_fname)
# Evalute model on test set
print('Model evaluation by neon: ', mlp.eval(test, metric=Accuracy()))
y_pred = mlp.get_outputs(test)
#print ("Shape y_pred: ", y_pred.shape)
scores = p1b2.evaluate_accuracy(p1_common.convert_to_class(y_pred), y_test)
print('Evaluation on test data:', scores)
# setup optimizer
opt_w = GradientDescentMomentum(0.001 * learning_rate_scale, 0.9, wdecay=0.0005)
opt_b = GradientDescentMomentum(0.002 * learning_rate_scale, 0.9)
optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b})
# setup model
model = Model(layers=Tree([frcn_layers, bb_layers]))
# if training a new model, seed the Alexnet conv layers with pre-trained weights
# otherwise, just load the model file
if args.model_file is None:
load_imagenet_weights(model, args.data_dir)
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCostMask(costfunc=SmoothL1Loss())],
weights=[1, 1])
callbacks = Callbacks(model, **args.callback_args)
model.fit(train_set, optimizer=optimizer,
num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print 'running eval on the training set...'
metric_train = model.eval(train_set, metric=ObjectDetection())
print 'Train: label accuracy - {}%, object detection SmoothL1Loss - {}'.format(
metric_train[0]*100,
metric_train[1])
gate_activation=Logistic(), reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, init_glorot, bias=init_glorot, activation=Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01, clip_gradients=clip_gradients)
# configure callbacks
callbacks = Callbacks(model, train_set, output_file=args.output_file,
valid_set=test_set, valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "Test Accuracy - ", 100 * model.eval(test_set, metric=metric)
print "Train Accuracy - ", 100 * model.eval(train_set, metric=metric)
def train_mlp():
"""
Train data and save scaling and network weights and biases to file
to be used by forward prop phase on test data
"""
parser = NeonArgparser(__doc__)
args = parser.parse_args()
logger = logging.getLogger()
logger.setLevel(args.log_thresh)
# hyperparameters
num_epochs = args.epochs
#preprocessor
std_scale = preprocessing.StandardScaler(with_mean=True,with_std=True)
#std_scale = feature_scaler(type='Standardizer',with_mean=True,with_std=True)
#number of non one-hot encoded features, including ground truth
num_feat = 4
# load up the mnist data set
# split into train and tests sets
#load data from csv-files and rescale
#training
traindf = pd.DataFrame.from_csv('data/train.csv')
ncols = traindf.shape[1]
#tmpmat=std_scale.fit_transform(traindf.as_matrix())
#print std_scale.scale_
#print std_scale.mean_
tmpmat = traindf.as_matrix()
#print tmpmat[:,1:num_feat]
tmpmat[:,:num_feat] = std_scale.fit_transform(tmpmat[:,:num_feat])
X_train = tmpmat[:,1:]
y_train = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#validation
validdf = pd.DataFrame.from_csv('data/validate.csv')
ncols = validdf.shape[1]
tmpmat = validdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_valid = tmpmat[:,1:]
y_valid = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
#test
testdf = pd.DataFrame.from_csv('data/test.csv')
ncols = testdf.shape[1]
tmpmat = testdf.as_matrix()
tmpmat[:,:num_feat] = std_scale.transform(tmpmat[:,:num_feat])
X_test = tmpmat[:,1:]
y_test = np.reshape(tmpmat[:,0],(tmpmat[:,0].shape[0],1))
# setup a training set iterator
train_set = CustomDataIterator(X_train, lshape=(X_train.shape[1]), y_c=y_train)
# setup a validation data set iterator
valid_set = CustomDataIterator(X_valid, lshape=(X_valid.shape[1]), y_c=y_valid)
# setup a validation data set iterator
test_set = CustomDataIterator(X_test, lshape=(X_test.shape[1]), y_c=y_test)
# setup weight initialization function
init_norm = Xavier()
# setup model layers
layers = [Affine(nout=X_train.shape[1], init=init_norm, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=X_train.shape[1]/2, init=init_norm, activation=Rectlin()),
Linear(nout=1, init=init_norm)]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=SmoothL1Loss())
# setup optimizer
#schedule
#schedule = ExpSchedule(decay=0.3)
#optimizer = GradientDescentMomentum(0.0001, momentum_coef=0.9, stochastic_round=args.rounding, schedule=schedule)
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1.e-8)
# initialize model object
mlp = Model(layers=layers)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
#evaluate model
print('Evaluation Error = %.4f'%(mlp.eval(valid_set, metric=SmoothL1Metric())))
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# Saving the model
print 'Saving model parameters!'
mlp.save_params("model/homeapp_model.prm")
# Reloading saved model
# This should go in run.py
mlp=Model("model/homeapp_model.prm")
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# save the preprocessor vectors:
np.savez("model/homeapp_preproc", mean=std_scale.mean_, std=std_scale.scale_)
return 1
def ShiftAdaMax_with_Scale(LR=1):
return ShiftAdaMax(learning_rate=LR_start * LR,
schedule=ShiftSchedule(2, shift_size=1))
optimizer = MultiOptimizer({
'default': ShiftAdaMax_with_Scale(),
'BinaryLinear_0': ShiftAdaMax_with_Scale(57.038),
'BinaryLinear_1': ShiftAdaMax_with_Scale(73.9008),
'BinaryLinear_2': ShiftAdaMax_with_Scale(73.9008),
'BinaryLinear_3': ShiftAdaMax_with_Scale(52.3195)
})
# initialize model object
bnn = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(bnn, eval_set=val_set, **args.callback_args)
# run fit
bnn.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
print('Misclassification error = %.1f%%' %
(bnn.eval(val_set, metric=Misclassification()) * 100))
bnn.save_params("bin_model/final_model.prm")
# define layers
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=init_emb,
pad_idx=0, update=embedding_update),
LSTM(hidden_size, init_glorot, activation=Tanh(), gate_activation=Logistic(),
reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(nclass, init_glorot, bias=init_glorot, activation=Softmax())
]
# set the cost, metrics, optimizer
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "\nTrain Accuracy -", 100 * model.eval(train_set, metric=metric)
print "Test Accuracy -", 100 * model.eval(valid_set, metric=metric)
reset_cells=True, name=name+"Enc"))
decoder.append(GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic(),
reset_cells=True, name=name+"Dec"))
decoder_connections.append(ii)
decoder.append(Affine(train_set.nout, init, bias=init, activation=Softmax(), name="AffOut"))
layers = Seq2Seq([encoder, decoder],
decoder_connections=decoder_connections,
name="Seq2Seq")
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
model = Model(layers=layers)
optimizer = RMSProp(gradient_clip_value=gradient_clip_value, stochastic_round=args.rounding)
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost, callbacks=callbacks)
# Misclassification rate on validation set
error_rate = model.eval(valid_set, metric=Misclassification(steps=time_steps))
neon_logger.display('Misclassification error = %.2f%%' % (error_rate * 100))
# Print some example predictions.
prediction, groundtruth = get_predictions(model, valid_set, time_steps)
# convert them into text and display
display_text(valid_set.index_to_token, groundtruth, prediction)
relu = Rectlin()
layers = []
layers.append(Dropout(keep=.8))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu, pad=1, strides=2))
layers.append(Dropout(keep=.5))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1, strides=2))
layers.append(Dropout(keep=.5))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((1, 1, 192), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((1, 1, 16), init=init_uni, activation=relu))
layers.append(Pooling(6, op="avg"))
layers.append(Activation(Softmax()))
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, train_set, output_file=args.output_file, valid_set=valid_set,
valid_freq=args.validation_freq, progress_bar=args.progress_bar)
mlp.fit(train_set, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print mlp.eval(valid_set, metric=Misclassification())
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# setup optimizer
optimizer = GradientDescentMomentum(0.1,
momentum_coef=0.9,
stochastic_round=args.rounding)
# initialize model object
mlp = Model(layers=layers)
path = "./best_state_" + str(i) + ".prm"
# configure callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(path)
# print "epocs", args.epochs
# run fit
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=15,
cost=cost,
callbacks=callbacks)
result_float = (mlp.eval(valid_set, metric=Misclassification())) * 100
result_string = '%.1f%%' % result_float
print(result_string[:-1])
# print('Misclassification error = %.1f%%' % r)
rlayer = Recurrent(hidden_size, g_uni, activation=Tanh(), reset_cells=True)
elif args.rlayer_type == 'birnn':
rlayer = DeepBiRNN(hidden_size, g_uni, activation=Tanh(),
depth=1, reset_cells=True)
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=uni),
rlayer,
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, g_uni, bias=g_uni, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = Adagrad(learning_rate=0.01,
gradient_clip_value=gradient_clip_value)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set, optimizer=optimizer,
num_epochs=args.epochs, cost=cost, callbacks=callbacks)
# eval model
neon_logger.display("Train Accuracy - {}".format(100 * model.eval(train_set, metric=Accuracy())))
neon_logger.display("Test Accuracy - {}".format(100 * model.eval(valid_set, metric=Accuracy())))
import os from neon.util.argparser import NeonArgparser from neon.util.persist import load_obj from neon.transforms import Misclassification, CrossEntropyMulti from neon.optimizers import GradientDescentMomentum from neon.layers import GeneralizedCost from neon.models import Model from neon.data import DataLoader, ImageParams # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) args = parser.parse_args() # setup data provider test_dir = os.path.join(args.data_dir, 'val') shape = dict(channel_count=3, height=32, width=32) test_params = ImageParams(center=True, flip=False, **shape) common = dict(target_size=1, nclasses=10) test_set = DataLoader(set_name='val', repo_dir=test_dir, media_params=test_params, **common) model = Model(load_obj(args.model_file)) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) opt = GradientDescentMomentum(0.1, 0.9, wdecay=0.0001) model.initialize(test_set, cost=cost) acc = 1.0 - model.eval(test_set, metric=Misclassification())[0] print 'Accuracy: %.1f %% (Top-1)' % (acc*100.0) model.benchmark(test_set, cost=cost, optimizer=opt)
bias=init,
activation=Softmax(),
name="AffOut"))
layers = Seq2Seq([encoder, decoder],
decoder_connections=decoder_connections,
name="Seq2Seq")
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
model = Model(layers=layers)
optimizer = RMSProp(gradient_clip_value=gradient_clip_value,
stochastic_round=args.rounding)
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
# Misclassification rate on validation set
error_rate = model.eval(valid_set, metric=Misclassification(steps=time_steps))
neon_logger.display('Misclassification error = %.2f%%' % (error_rate * 100))
# Print some example predictions.
prediction, groundtruth = get_predictions(model, valid_set, time_steps)
# convert them into text and display
display_text(valid_set.index_to_token, groundtruth, prediction)
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
train = DataIterator(X_train, y_train, nclass=nclass)
test = DataIterator(X_test, y_test, nclass=nclass)
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
# set up the model layers
layers = []
layers.append(Affine(nout=200, init=init_uni, activation=Rectlin()))
layers.append(Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)))
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, train, output_file=args.output_file,
valid_set=test, valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print mlp.eval(test, metric=Misclassification())
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=init_emb),
LSTM(hidden_size, init_glorot, activation=Tanh(),
gate_activation=Logistic(), reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, init_glorot, bias=init_glorot, activation=Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01, clip_gradients=clip_gradients)
# configure callbacks
callbacks = Callbacks(model, train_set, args, valid_set=valid_set)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "Test Accuracy - ", 100 * model.eval(valid_set, metric=metric)
print "Train Accuracy - ", 100 * model.eval(train_set, metric=metric)
# Model construction
story_path = [LookupTable(**lookup_params), rlayer_obj(**rlayer_params)]
query_path = [LookupTable(**lookup_params), rlayer_obj(**rlayer_params)]
layers = [
MergeMultistream(layers=[story_path, query_path], merge="stack"),
Affine(babi.vocab_size, init=GlorotUniform(), activation=Softmax())
]
model = Model(layers=layers)
# setup callbacks
callbacks = Callbacks(model,
train_set,
eval_set=valid_set,
**args.callback_args)
# train model
model.fit(train_set,
optimizer=Adam(),
num_epochs=args.epochs,
cost=GeneralizedCost(costfunc=CrossEntropyMulti()),
callbacks=callbacks)
# output accuracies
print('Train Accuracy = %.1f%%' %
(model.eval(train_set, metric=Accuracy()) * 100))
print('Test Accuracy = %.1f%%' %
(model.eval(valid_set, metric=Accuracy()) * 100))
mlp = Model(layers=layers)
logger.info("model construction complete...")
"""
model training and classification accurate rate
"""
#model training and results
callbacks = Callbacks(mlp, eval_set=valid,metric=Misclassification())
#add lost and metric call backs facilitate more diagnostic
#callbacks.add_callback(MetricCallback(mlp,eval_set=train,metric=Misclassification(),epoch_freq=args.evaluation_freq))
#callbacks.add_callback(MetricCallback(mlp,eval_set=valid,metric=Misclassification(),epoch_freq=args.evaluation_freq))
#run the model
mlp.fit(train, optimizer=opt_gdm, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
#final classification accuracy
t_mis_rate=mlp.eval(train, metric=Misclassification())*100
v_mis_rate=mlp.eval(valid, metric=Misclassification())*100
#test_mis_rate=mlp.eval(test, metric=Misclassification())*100
print ('Train Misclassification error = %.1f%%' %t_mis_rate)
print ('Valid Miscladdifcaiton error = %.1f%%' %v_mis_rate)
#print ('Test Miscladdifcaiton error = %.1f%%' %test_mis_rate)
# set up 3x3 conv stacks with different feature map sizes
for nofm in [64, 128, 256, 512, 512]:
layers.append(Conv((3, 3, nofm), **conv_params))
layers.append(Conv((3, 3, nofm), **conv_params))
if nofm > 128:
layers.append(Conv((3, 3, nofm), **conv_params))
if args.vgg_version == 'E':
layers.append(Conv((3, 3, nofm), **conv_params))
layers.append(Pooling(2, strides=2))
layers.append(Affine(nout=4096, init=initfc, bias=Constant(0), activation=relu))
layers.append(Dropout(keep=0.5))
layers.append(Affine(nout=4096, init=initfc, bias=Constant(0), activation=relu))
layers.append(Dropout(keep=0.5))
layers.append(Affine(nout=1000, init=initfc, bias=Constant(0), activation=Softmax()))
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model = Model(layers=layers)
# configure callbacks
top5 = TopKMisclassification(k=5)
callbacks = Callbacks(model, eval_set=test, metric=top5, **args.callback_args)
model.load_params(args.model_file)
mets=model.eval(test, metric=TopKMisclassification(k=5))
print 'Validation set metrics:'
print 'LogLoss: %.2f, Accuracy: %.1f %% (Top-1), %.1f %% (Top-5)' % (mets[0],
(1.0-mets[1])*100,
(1.0-mets[2])*100)
stochastic_round=args.rounding)
bn = True
layers = [
Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=bn),
Pooling((2, 2)),
Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=bn),
Pooling((2, 2)),
Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=bn),
Affine(nout=10, init=init_uni, activation=Softmax())
]
if args.datatype in [np.float32, np.float64]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
elif args.datatype in [np.float16]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti(scale=cost_scale))
model = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(model, eval_set=test, **args.callback_args)
model.fit(train,
optimizer=opt_gdm,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
error_rate = model.eval(test, metric=Misclassification())
neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100))
from neon.optimizers import GradientDescentMomentum
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9)
#callbacks show the progress of calculations
from neon.callbacks.callbacks import Callbacks
callbacks = Callbacks(mlp, eval_set=test_set, **args.callback_args)
#train the model
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=10,
cost=cost,
callbacks=callbacks)
#The variable results is a numpy array with
#shape (num_test_examples, num_outputs) = (10000,10) with the model probabilities for each label.
results = mlp.get_outputs(test_set)
print "labels: %s, %s, %s: " % (y_test[2], y_test[5], y_test[100])
#ind1, val1 = max(results[2].tolist())
#ind2, val2 = max(results[5].tolist())
#ind3, val3 = max(results[100].tolist())
print "results: %s, %s, %s:" % (results[2].tolist(), results[5].tolist(),
results[100].tolist())
from neon.transforms import Misclassification
# evaluate the model on test_set using the misclassification metric
error = mlp.eval(test_set, metric=Misclassification()) * 100
print('Misclassification error = %.1f%%' % error)
# scale LR by 0.1 every 20 epochs (this assumes batch_size = 256)
weight_sched = Schedule(20, 0.1)
opt_gdm = GradientDescentMomentum(0.01,
0.9,
wdecay=0.0005,
schedule=weight_sched)
opt_biases = GradientDescentMomentum(0.02, 0.9, schedule=weight_sched)
opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases})
# configure callbacks
valmetric = TopKMisclassification(k=5)
callbacks = Callbacks(model,
eval_set=test,
metric=valmetric,
**args.callback_args)
if args.model_file is not None:
model.load_params(args.model_file)
if not args.test_only:
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.fit(train,
optimizer=opt,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
mets = model.eval(test, metric=valmetric)
print 'Validation set metrics:'
print 'LogLoss: %.2f, Accuracy: %.1f %% (Top-1), %.1f %% (Top-5)' % (
mets[0], (1.0 - mets[1]) * 100, (1.0 - mets[2]) * 100)
layers = [
Conv((4, 4, 32), init=init_norm, activation=Rectlin()),
Pooling((2, 2)),
Conv((3, 3, 16), init=init_norm, activation=Rectlin()),
Pooling((2, 2)),
Dropout(keep=0.8),
Affine(nout=128, init=init_norm, activation=Rectlin()),
Affine(nout=64, init=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Softmax())
]
mlp = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
optimizer = GradientDescentMomentum(learning_rate=0.05,
momentum_coef=0.9,
wdecay=1e-5)
callbacks = Callbacks(mlp, eval_set=train_set, **args.callback_args)
start = time.time()
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=10,
cost=cost,
callbacks=callbacks)
end = time.time()
print('Train time:', end - start, 'sec')
error_rate = mlp.eval(test_set, metric=Misclassification())
print('Error rate:', 100 * error_rate[0], '%')
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=uni),
rlayer,
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, g_uni, bias=g_uni, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = Adagrad(learning_rate=0.01,
gradient_clip_value=gradient_clip_value)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
# eval model
neon_logger.display("Train Accuracy - {}".format(
100 * model.eval(train_set, metric=Accuracy())))
neon_logger.display("Test Accuracy - {}".format(
100 * model.eval(valid_set, metric=Accuracy())))
strides=dict(str_h=2, str_w=4)),
Pooling(2, strides=2),
Conv((5, 5, 64),
init=init,
batch_norm=True,
activation=Rectlin(),
strides=dict(str_h=1, str_w=2)),
DeepBiRNN(128,
init=GlorotUniform(),
batch_norm=True,
activation=Rectlin(),
reset_cells=True,
depth=3),
RecurrentMean(),
Affine(nout=common['nclasses'], init=init, activation=Softmax())
]
model = Model(layers=layers)
opt = Adagrad(learning_rate=0.01)
metric = Misclassification()
callbacks = Callbacks(model, eval_set=val, metric=metric, **args.callback_args)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
model.fit(train,
optimizer=opt,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
print('Misclassification error = %.1f%%' %
(model.eval(val, metric=metric) * 100))
#!/usr/bin/env python
# ----------------------------------------------------------------------------
# Copyright 2016 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------
from neon.util.argparser import NeonArgparser
from neon.util.persist import load_obj
from neon.transforms import Misclassification
from neon.models import Model
from neon.data import ImageLoader
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
args = parser.parse_args()
# setup data provider
test_set = ImageLoader(set_name='validation', repo_dir=args.data_dir,
inner_size=32, scale_range=40, do_transforms=False)
model = Model(load_obj(args.model_file), test_set)
print 'Accuracy: %.1f %% (Top-1)' % (1.0-model.eval(test_set, metric=Misclassification())*100)
Conv((1, 1, 16), **conv),
Pooling(8, op="avg"),
Activation(Softmax())
]
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp)
def do_nothing(_):
pass
callbacks.callbacks = []
callbacks.on_train_begin = do_nothing
callbacks.on_epoch_end = do_nothing
mlp.fit(train_set,
optimizer=opt_gdm,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
opt_metric = 1.0 - mlp.eval(valid_set, metric=Misclassification())
print('Metric = {}'.format(opt_metric))
conn.experiments(experiment.id).observations().create(
suggestion=suggestion.id,
value=float(opt_metric[0]),
)
relu = Rectlin()
layers = []
layers.append(Dropout(keep=.8))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 96), init=init_uni, batch_norm=True, activation=relu, pad=1, strides=2))
layers.append(Dropout(keep=.5))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu, pad=1, strides=2))
layers.append(Dropout(keep=.5))
layers.append(Conv((3, 3, 192), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((1, 1, 192), init=init_uni, batch_norm=True, activation=relu))
layers.append(Conv((1, 1, 16), init=init_uni, activation=relu))
layers.append(Pooling(6, op="avg"))
layers.append(Activation(Softmax()))
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, train_set, output_file=args.output_file, valid_set=valid_set,
valid_freq=args.validation_freq, progress_bar=args.progress_bar)
mlp.fit(train_set, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print('Misclassification error = %.1f%%' % (mlp.eval(valid_set, metric=Misclassification())*100))
# initialization
init_glorot = GlorotUniform()
# define layers
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=init_emb, pad_idx=0, update=embedding_update),
LSTM(hidden_size, init_glorot, activation=Tanh(), gate_activation=Logistic(), reset_cells=True),
RecurrentSum(),
Dropout(keep=0.5),
Affine(nclass, init_glorot, bias=init_glorot, activation=Softmax()),
]
# set the cost, metrics, optimizer
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
metric = Accuracy()
model = Model(layers=layers)
optimizer = Adagrad(learning_rate=0.01)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set, optimizer=optimizer, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
# eval model
neon_logger.display("Train Accuracy - {}".format(100 * model.eval(train_set, metric=metric)))
neon_logger.display("Test Accuracy - {}".format(100 * model.eval(valid_set, metric=metric)))
# setup model layers
layers = [
Affine(nout=100, init=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# setup optimizer
optimizer = GradientDescentMomentum(0.1,
momentum_coef=0.9,
stochastic_round=args.rounding)
# initialize model object
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)
# run fit
mlp.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
print('Misclassification error = %.1f%%' %
(mlp.eval(valid_set, metric=Misclassification()) * 100))
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
elif args.datatype in [np.float16]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01/cost_scale,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = []
layers.append(Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=True))
layers.append(Pooling((2, 2)))
layers.append(Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=True))
layers.append(Pooling((2, 2)))
layers.append(Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=True))
layers.append(Affine(nout=10, init=init_uni, activation=Softmax()))
if args.datatype in [np.float32, np.float64]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
elif args.datatype in [np.float16]:
cost = GeneralizedCost(costfunc=CrossEntropyMulti(scale=cost_scale))
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, train, output_file=args.output_file,
valid_set=test, valid_freq=args.validation_freq,
progress_bar=args.progress_bar)
mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print 'Misclassification error = %.1f%%' % (mlp.eval(test, metric=Misclassification())*100)
reset_cells=True,
batch_norm=True)
layers = [
LookupTable(vocab_size=vocab_size, embedding_dim=embedding_dim, init=uni),
rlayer,
RecurrentSum(),
Dropout(keep=0.5),
Affine(2, g_uni, bias=g_uni, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = Adagrad(learning_rate=0.01,
gradient_clip_value=gradient_clip_value)
# configure callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=optimizer,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
# eval model
print "Train Accuracy - ", 100 * model.eval(train_set, metric=Accuracy())
print "Test Accuracy - ", 100 * model.eval(valid_set, metric=Accuracy())
class NpSemanticSegClassifier:
"""
NP Semantic Segmentation classifier model (based on Neon framework).
Args:
num_epochs(int): number of epochs to train the model
**callback_args (dict): callback args keyword arguments to init a Callback for the model
cost: the model's cost function. Default is 'neon.transforms.CrossEntropyBinary' cost
optimizer (:obj:`neon.optimizers`): the model's optimizer. Default is
'neon.optimizers.GradientDescentMomentum(0.07, momentum_coef=0.9)'
"""
def __init__(self, num_epochs, callback_args,
optimizer=GradientDescentMomentum(0.07, momentum_coef=0.9)):
"""
Args:
num_epochs(int): number of epochs to train the model
**callback_args (dict): callback args keyword arguments to init Callback for the model
cost: the model's cost function. Default is 'neon.transforms.CrossEntropyBinary' cost
optimizer (:obj:`neon.optimizers`): the model's optimizer. Default is
`neon.optimizers.GradientDescentMomentum(0.07, momentum_coef=0.9)`
"""
self.model = None
self.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
self.optimizer = optimizer
self.epochs = num_epochs
self.callback_args = callback_args
def build(self):
"""
Build the model's layers
"""
first_layer_dens = 64
second_layer_dens = 64
output_layer_dens = 2
# setup weight initialization function
init_norm = Gaussian(scale=0.01)
# setup model layers
layers = [Affine(nout=first_layer_dens, init=init_norm,
activation=Rectlin()),
Affine(nout=second_layer_dens, init=init_norm,
activation=Rectlin()),
Affine(nout=output_layer_dens, init=init_norm,
activation=Logistic(shortcut=True))]
# initialize model object
self.model = Model(layers=layers)
def fit(self, test_set, train_set):
"""
Train and fit the model on the datasets
Args:
test_set (:obj:`neon.data.ArrayIterators`): The test set
train_set (:obj:`neon.data.ArrayIterators`): The train set
args: callback_args and epochs from ArgParser input
"""
# configure callbacks
callbacks = Callbacks(self.model, eval_set=test_set, **self.callback_args)
self.model.fit(train_set, optimizer=self.optimizer, num_epochs=self.epochs, cost=self.cost,
callbacks=callbacks)
def save(self, model_path):
"""
Save the model's prm file in model_path location
Args:
model_path(str): local path for saving the model
"""
self.model.save_params(model_path)
def load(self, model_path):
"""
Load pre-trained model's .prm file to NpSemanticSegClassifier object
Args:
model_path(str): local path for loading the model
"""
self.model = Model(model_path)
def eval(self, test_set):
"""
Evaluate the model's test_set on error_rate, test_accuracy_rate and precision_recall_rate
Args:
test_set (ArrayIterator): The test set
Returns:
tuple(int): error_rate, test_accuracy_rate and precision_recall_rate
"""
error_rate = self.model.eval(test_set, metric=Misclassification())
test_accuracy_rate = self.model.eval(test_set, metric=Accuracy())
precision_recall_rate = self.model.eval(test_set, metric=PrecisionRecall(2))
return error_rate, test_accuracy_rate, precision_recall_rate
def get_outputs(self, test_set):
"""
Classify the dataset on the model
Args:
test_set (:obj:`neon.data.ArrayIterators`): The test set
Returns:
list(float): model's predictions
"""
return self.model.get_outputs(test_set)
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
train = ArrayIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32))
test = ArrayIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32))
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
# set up the model layers
layers = [
Affine(nout=200, init=init_uni, activation=Rectlin()),
Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True))
]
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, eval_set=test, **args.callback_args)
mlp.fit(train,
optimizer=opt_gdm,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
print('Misclassification error = %.1f%%' %
(mlp.eval(test, metric=Misclassification()) * 100))
optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b})
# setup model
model = Model(layers=Tree([frcn_layers, bb_layers]))
# if training a new model, seed the Alexnet conv layers with pre-trained weights
# otherwise, just load the model file
if args.model_file is None:
load_imagenet_weights(model, args.data_dir)
cost = Multicost(costs=[
GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCostMask(costfunc=SmoothL1Loss())
],
weights=[1, 1])
callbacks = Callbacks(model, **args.callback_args)
model.fit(train_set,
optimizer=optimizer,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
print 'running eval on the training set...'
metric_train = model.eval(train_set, metric=ObjectDetection())
print 'Train: label accuracy - {}%, object deteciton SmoothL1Loss - {}'.format(
metric_train[0] * 100, metric_train[1])
# use the iterator that generates 1-hot output. other HDF5Iterator (sub) classes are
# available for different data layouts
train_set = HDF5IteratorOneHot('mnist_train.h5')
valid_set = HDF5IteratorOneHot('mnist_test.h5')
# setup weight initialization function
init_norm = Gaussian(loc=0.0, scale=0.01)
# setup model layers
layers = [Affine(nout=100, init=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
# setup optimizer
optimizer = GradientDescentMomentum(
0.1, momentum_coef=0.9, stochastic_round=args.rounding)
# initialize model object
mlp = Model(layers=layers)
# configure callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
# run fit
mlp.fit(train_set, optimizer=optimizer,
num_epochs=args.epochs, cost=cost, callbacks=callbacks)
error_rate = mlp.eval(valid_set, metric=Misclassification())
neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100))
Affine(nout=16, name="b2_l1", **normrelu),
Affine(nout=10, name="b2_l2", **normsigm)]
# setup cost function as CrossEntropy
cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCost(costfunc=CrossEntropyBinary()),
GeneralizedCost(costfunc=CrossEntropyBinary())],
weights=[1, 0., 0.])
# setup optimizer
optimizer = GradientDescentMomentum(
0.1, momentum_coef=0.9, stochastic_round=args.rounding)
# initialize model object
alphas = [1, 0.25, 0.25]
mlp = Model(layers=SingleOutputTree([p1, p2, p3], alphas=alphas))
# setup standard fit callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, multicost=True, **args.callback_args)
# run fit
mlp.fit(train_set, optimizer=optimizer,
num_epochs=args.epochs, cost=cost, callbacks=callbacks)
# TODO: introduce Multicost metric support. The line below currently fails
# since the Misclassification metric expects a single Tensor not a list of
# Tensors
neon_logger.display('Misclassification error = %.1f%%' %
(mlp.eval(valid_set, metric=Misclassification()) * 100))
p3 = [b2, Affine(nout=16, linear_name="b2_l1", **normrelu), Affine(nout=10, linear_name="b2_l2", **normsigm)]
# setup cost function as CrossEntropy
cost = Multicost(
costs=[
GeneralizedCost(costfunc=CrossEntropyMulti()),
GeneralizedCost(costfunc=CrossEntropyBinary()),
GeneralizedCost(costfunc=CrossEntropyBinary()),
],
weights=[1, 0.0, 0.0],
)
# setup optimizer
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding)
# initialize model object
alphas = [1, 0.25, 0.25]
mlp = Model(layers=Tree([p1, p2, p3], alphas=alphas))
# setup standard fit callbacks
callbacks = Callbacks(mlp, train_set, eval_set=valid_set, **args.callback_args)
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
logging.getLogger("neon").info(
"Misclassification error = %.1f%%", (mlp.eval(valid_set, metric=Misclassification()) * 100)
)
print("Misclassification error = %.1f%%" % (mlp.eval(valid_set, metric=Misclassification()) * 100))
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(mlp, eval_set=valid_set, **args.callback_args)
#callbacks.add_early_stop_callback(stop_func)
#callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
callbacks.add_early_stop_callback(stop_func)
callbacks.add_save_best_state_callback(os.path.join(args.data_dir, "early_stop-best_state.pkl"))
# run fit
mlp.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
#evaluate model
print('Evaluation Error = %.4f'%(mlp.eval(valid_set, metric=SmoothL1Metric())))
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# Saving the model
print 'Saving model parameters!'
mlp.save_params("jobwait_model.prm")
# Reloading saved model
# This should go in run.py
mlp=Model("jobwait_model.prm")
print('Test set error = %.4f'%(mlp.eval(test_set, metric=SmoothL1Metric())))
# save the preprocessor vectors:
np.savez("jobwait_preproc", mean=std_scale.mean_, std=std_scale.scale_)
