def __init__(self, model, ngpu, options,
data_options=None, time_options=None):
self.model = model
#self.model.set_batch_size(data_options['batch_size'])
self.ngpu = ngpu
self.gpu_mode = True if ngpu >= 1 else False
self.time_options = time_options
self.data_options = data_options
if self.gpu_mode:
try:
self.be = gen_backend(backend='nervanagpu',
batch_size=data_options['batch_size'])
print("Backgrand: nervanagpu")
except:
self.be = gen_backend(backend='gpu',
batch_size=data_options['batch_size'])
print("Backgrand: gpu")
else:
self.be = gen_backend(backend='mkl',
batch_size=data_options['batch_size'])
self.loss = L.GeneralizedCost(costfunc=TF.CrossEntropyMulti())
B = self.data_options['batch_size']
self.model.bsz(B)
C, W, H = self.data_options['image_shape']
self.model.initialize(((C, H, W), B), self.loss)
def test_cpu_randomstate():
# run 1
be = gen_backend(backend='cpu', rng_seed=100)
a = be.empty((3, 3))
be.make_binary_mask(a, keepthresh=be.rng.rand())
x0 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
x1 = a.get()
# run 2, using reset
be.rng_reset()
be.make_binary_mask(a, keepthresh=be.rng.rand())
y0 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
y1 = a.get()
del(be)
# run 3, using a new backend
be = gen_backend(backend='cpu', rng_seed=100)
a = be.empty((3, 3))
be.make_binary_mask(a, keepthresh=be.rng.rand())
z0 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
z1 = a.get()
# check equality
assert tensors_allclose([x0, x1], [y0, y1], rtol=0., atol=0.)
assert tensors_allclose([x0, x1], [z0, z1], rtol=0., atol=0.)
del(be)
def test_gpu_randomstate():
# run 1
be = gen_backend(backend='gpu', rng_seed=100)
a = be.empty((3, 3))
a[:] = be.rand() # gpu rand
x0 = a.get()
x1 = be.rng.rand(3, 3) # host rand
a[:] = be.rand() # gpu rand
x2 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
x3 = a.get()
assert len(be.context_rand_state_map) == 1 and len(be.context_rand_state_alive) == 1
for ctx in be.context_rand_state_alive:
assert be.context_rand_state_alive[ctx] is True
# run 2, using reset
be.rng_reset()
for ctx in be.context_rand_state_alive:
assert be.context_rand_state_alive[ctx] is False
a[:] = be.rand()
y0 = a.get()
y1 = be.rng.rand(3, 3)
a[:] = be.rand()
y2 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
y3 = a.get()
assert len(be.context_rand_state_map) == 1 and len(be.context_rand_state_alive) == 1
for ctx in be.context_rand_state_alive:
assert be.context_rand_state_alive[ctx] is True
del(be)
# run 3, using a new backend
be = gen_backend(backend='gpu', rng_seed=100)
a = be.empty((3, 3))
a[:] = be.rand() # gpu rand
z0 = a.get()
z1 = be.rng.rand(3, 3) # host rand
a[:] = be.rand() # gpu rand
z2 = a.get()
be.make_binary_mask(a, keepthresh=be.rng.rand())
z3 = a.get()
# check equality
assert_tensors_allclose([x0, x1, x2, x3], [y0, y1, y2, y3], rtol=0., atol=0.)
assert_tensors_allclose([x0, x1, x2, x3], [z0, z1, z2, z3], rtol=0., atol=0.)
del(be)
def __init__(self, num_actions, args):
# remember parameters
self.num_actions = num_actions
self.batch_size = args.batch_size
self.discount_rate = args.discount_rate
self.history_length = args.history_length
self.screen_dim = (args.screen_height, args.screen_width)
self.clip_error = args.clip_error
self.min_reward = args.min_reward
self.max_reward = args.max_reward
self.batch_norm = args.batch_norm
# create Neon backend
self.be = gen_backend(backend = args.backend,
batch_size = args.batch_size,
rng_seed = args.random_seed,
device_id = args.device_id,
datatype = np.dtype(args.datatype).type,
stochastic_round = args.stochastic_round)
# prepare tensors once and reuse them
self.input_shape = (self.history_length,) + self.screen_dim + (self.batch_size,)
self.input = self.be.empty(self.input_shape)
self.input.lshape = self.input_shape # HACK: needed for convolutional networks
self.targets = self.be.empty((self.num_actions, self.batch_size))
# create model
layers = self._createLayers(num_actions)
self.model = Model(layers = layers)
self.cost = GeneralizedCost(costfunc = SumSquared())
# Bug fix
for l in self.model.layers.layers:
l.parallelism = 'Disabled'
self.model.initialize(self.input_shape[:-1], self.cost)
if args.optimizer == 'rmsprop':
self.optimizer = RMSProp(learning_rate = args.learning_rate,
decay_rate = args.decay_rate,
stochastic_round = args.stochastic_round)
elif args.optimizer == 'adam':
self.optimizer = Adam(learning_rate = args.learning_rate,
stochastic_round = args.stochastic_round)
elif args.optimizer == 'adadelta':
self.optimizer = Adadelta(decay = args.decay_rate,
stochastic_round = args.stochastic_round)
else:
assert false, "Unknown optimizer"
# create target model
self.train_iterations = 0
if args.target_steps:
self.target_model = Model(layers = self._createLayers(num_actions))
# Bug fix
for l in self.target_model.layers.layers:
l.parallelism = 'Disabled'
self.target_model.initialize(self.input_shape[:-1])
self.save_weights_prefix = args.save_weights_prefix
else:
self.target_model = self.model
self.callback = None
def backend(request):
"""
Fixture to setup the backend before running a
test. Also registers the teardown function to clean
up the backend after a test is done. This had module
scope, so this will be run once for each test in a
given test file (module).
This fixture is parameterized to run both the cpu and
gpu backends for every test
"""
be = gen_backend(backend=request.param, rng_seed=0)
NervanaObject.be = be
be.bsz = 128 # hardwires here to 128
# add a cleanup call - will run after all
# test in module are done
def cleanup():
be = request.getfuncargvalue("backend")
del be
request.addfinalizer(cleanup)
# tests using this fixture can
# access the backend object from
# backend or use the NervanaObject.be global
return be
def backend_default(request):
'''
Fixture to setup the backend before running a
test. Also registers the teardown function to clean
up the backend after a test is done. This had module
scope, so this will be run once for each test in a
given test file (module).
This fixture is parameterized to run both the cpu and
gpu backends for every test
'''
be = gen_backend(backend=request.param,
default_dtype=np.float32,
batch_size=128,
rng_seed=0)
# add a cleanup call - will run after all
# test in module are done
def cleanup():
be = request.getfuncargvalue('backend_default')
del be
request.addfinalizer(cleanup)
# tests using this fixture can
# access the backend object from
# backend or use the NervanaObject.be global
return be
def sanity_check(conf_file, result, **be_args):
experiment = deserialize(os.path.join(dir, conf_file))
backend = gen_backend(model=experiment.model, **be_args)
experiment.initialize(backend)
res = experiment.run()
print(float(res['test']['MisclassRate_TOP_1']))
assert float(res['test']['MisclassRate_TOP_1']) == result
def __init__(self, env, args, rng, name = "DQNNeon"):
""" Initializes a network based on the Neon framework.
Args:
env (AtariEnv): The envirnoment in which the agent actuates.
args (argparse.Namespace): All settings either with a default value or set via command line arguments.
rng (mtrand.RandomState): initialized Mersenne Twister pseudo-random number generator.
name (str): The name of the network object.
Note:
This function should always call the base class first to initialize
the common values for the networks.
"""
_logger.info("Initializing new object of type " + str(type(self).__name__))
super(DQNNeon, self).__init__(env, args, rng, name)
self.input_shape = (self.sequence_length,) + self.frame_dims + (self.batch_size,)
self.dummy_batch = np.zeros((self.batch_size, self.sequence_length) + self.frame_dims, dtype=np.uint8)
self.batch_norm = args.batch_norm
self.be = gen_backend(
backend = args.backend,
batch_size = args.batch_size,
rng_seed = args.random_seed,
device_id = args.device_id,
datatype = np.dtype(args.datatype).type,
stochastic_round = args.stochastic_round)
# prepare tensors once and reuse them
self.input = self.be.empty(self.input_shape)
self.input.lshape = self.input_shape # HACK: needed for convolutional networks
self.targets = self.be.empty((self.output_shape, self.batch_size))
# create model
layers = self._create_layer()
self.model = Model(layers = layers)
self.cost_func = GeneralizedCost(costfunc = SumSquared())
# Bug fix
for l in self.model.layers.layers:
l.parallelism = 'Disabled'
self.model.initialize(self.input_shape[:-1], self.cost_func)
self._set_optimizer()
if not self.args.load_weights == None:
self.load_weights(self.args.load_weights)
# create target model
if self.target_update_frequency:
layers = self._create_layer()
self.target_model = Model(layers)
# Bug fix
for l in self.target_model.layers.layers:
l.parallelism = 'Disabled'
self.target_model.initialize(self.input_shape[:-1])
else:
self.target_model = self.model
self.callback = None
_logger.debug("%s" % self)
def run():
model = create_model(nin=784)
backend = gen_backend(rng_seed=0)
dataset = MNIST(repo_path='~/data/')
experiment = FitPredictErrorExperiment(model=model,
backend=backend,
dataset=dataset)
experiment.run()
def test_loader_exception_iter():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
manifest = random_manifest(10, 2)
config = generic_config(manifest.name)
dl = DataLoader(config, gen_backend(backend='cpu'))
assert len(list(iter(dl))) == 4
def serialize_check(conf_file, result, tol, res_string, **be_args):
experiment = deserialize(conf_file)
backend = gen_backend(model=experiment.model, **be_args)
experiment.initialize(backend)
res = experiment.run()
print float(res[res_string]['MisclassPercentage_TOP_1']), result,
assert abs(
float(res[res_string]['MisclassPercentage_TOP_1']) - result) < tol
def speed_check(conf_file, num_epochs, **be_args):
experiment = deserialize(os.path.join(dir, conf_file))
experiment.model.num_epochs = num_epochs
backend = gen_backend(model=experiment.model, **be_args)
experiment.initialize(backend)
start = time.time()
experiment.run()
return (time.time() - start)
def __init__(self, batch_size, its, layer_def, W, I, gradO):
gen_backend(backend='gpu', batch_size=batch_size,
datatype=np.float32, device_id=0)
assert layer_def['iH'] == layer_def['iW']
assert layer_def['kH'] == layer_def['kW']
assert layer_def['dH'] == layer_def['dW']
assert layer_def['padH'] == layer_def['padW']
input_filters = layer_def['Ci']
output_filters = layer_def['Co']
image_size = layer_def['iW']
filter_size = layer_def['kH']
padding = layer_def['padH']
stride = layer_def['dH']
self.I = I
self.W = W
self.gradO = gradO
I_cuda = gpuarray.to_gpu(I)
gradO_cuda = gpuarray.to_gpu(gradO)
W_cuda = gpuarray.to_gpu(W)
conv = Convolution((filter_size, filter_size, output_filters), strides=stride, padding=padding, init=init)
conv.configure((input_filters, image_size, image_size))
conv.allocate()
# conv.allocate_deltas()
conv.W = W_cuda
self.conv = conv
deltas = np.zeros(I.shape, dtype=np.float32)
deltas_cuda = gpuarray.to_gpu(deltas)
conv.deltas = deltas_cuda
# self.O = O
# self.gradW = gradW
# self.gradI = gradI
self.I_cuda = I_cuda
self.O_cuda = conv.outputs
self.gradO_cuda = gradO_cuda
self.gradW_cuda = conv.dW
self.gradI_cuda = conv.deltas
def compare_helper(op, inA, inB, dtype):
numpy_result = math_helper(np, op, inA, inB, dtype=np.float32)
if np.dtype(dtype).kind == 'i' or np.dtype(dtype).kind == 'u':
numpy_result = np.around(numpy_result)
numpy_result = numpy_result.clip(np.iinfo(dtype).min, np.iinfo(dtype).max)
numpy_result = numpy_result.astype(dtype)
if dtype in (np.float32, np.float16):
gpu = gen_backend(backend='gpu', default_dtype=dtype)
nervanaGPU_result = math_helper(gpu, op, inA, inB, dtype=dtype)
nervanaGPU_result = nervanaGPU_result.get()
np.allclose(numpy_result, nervanaGPU_result, rtol=0, atol=1e-5)
cpu = gen_backend(backend='cpu', default_dtype=dtype)
nervanaCPU_result = math_helper(cpu, op, inA, inB, dtype=dtype)
nervanaCPU_result = nervanaCPU_result.get()
np.allclose(numpy_result, nervanaCPU_result, rtol=0, atol=1e-5)
def get_backend(request, datatype=np.float32):
be = gen_backend(backend=request.param,
datatype=datatype,
device_id=request.config.getoption("--device_id"),
batch_size=128,
rng_seed=0)
if request.param == 'gpu':
be.enable_winograd = 2 if be.enable_winograd else be.enable_winograd
return be
def test_loader_exception_next():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
manifest = random_manifest(10, 2)
config = generic_config(manifest.name)
dl = DataLoader(config, gen_backend(backend='cpu'))
dl.next()
with pytest.raises(LoaderRuntimeError):
dl.next()
def test_loader_missing_config_field():
manifest = random_manifest(10)
config = generic_config(manifest.name)
del config['image']
with pytest.raises(Exception) as ex:
dl = DataLoader(config, gen_backend(backend='cpu'))
assert 'image' in str(ex)
def test_loader_invalid_config_type():
manifest = random_manifest(10)
config = generic_config(manifest.name)
config['type'] = 'invalid type name'
with pytest.raises(Exception) as ex:
dl = DataLoader(config, gen_backend(backend='cpu'))
assert 'invalid type name' in str(ex)
def gen_model(backend_type):
# setup backend
gen_backend(
backend=backend_type, batch_size=batch_size, rng_seed=2, device_id=args.device_id, default_dtype=args.datatype
)
init_uni = Uniform(low=-0.1, high=0.1)
# Set up the model layers
layers = []
layers.append(Conv((5, 5, 16), init=init_uni, bias=Constant(0), activation=Rectlin()))
layers.append(Pooling(2))
layers.append(Conv((5, 5, 32), init=init_uni, activation=Rectlin()))
layers.append(Pooling(2))
layers.append(Affine(nout=500, init=init_uni, activation=Rectlin()))
layers.append(Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)))
mlp = Model(layers=layers)
return mlp
def run():
model = create_model(nin=11)
backend = gen_backend(rng_seed=0)
dataset = Axa()
experiment = FitExperiment(model=model,
backend=backend,
dataset=dataset)
experiment.run()
outputs, targets = model.predict_fullset(dataset, 'test')
print outputs.asnumpyarray()
def train(self, dataset, model=None):
"""Trains the passed model on the given dataset. If no model is passed, `generate_default_model` is used."""
print "[%s] Starting training..." % self.model_name
start = time.time()
# The training will be run on the CPU. If a GPU is available it should be used instead.
backend = gen_backend(backend='cpu',
batch_size=self.batch_size,
rng_seed=self.random_seed,
stochastic_round=False)
cost = GeneralizedCost(
name='cost',
costfunc=CrossEntropyMulti())
optimizer = GradientDescentMomentum(
learning_rate=self.lrate,
momentum_coef=0.9)
# set up the model and experiment
if not model:
model = self.generate_default_model(dataset.num_labels)
args = NeonCallbackParameters()
args.output_file = os.path.join(self.root_path, self.Callback_Store_Filename)
args.evaluation_freq = 1
args.progress_bar = False
args.epochs = self.max_epochs
args.save_path = os.path.join(self.root_path, self.Intermediate_Model_Filename)
args.serialize = 1
args.history = 100
args.model_file = None
callbacks = Callbacks(model, dataset.train(), args, eval_set=dataset.test())
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(self.model_path)
# Uncomment line below to run on GPU using cudanet backend
# backend = gen_backend(rng_seed=0, gpu='cudanet')
model.fit(
dataset.train(),
optimizer=optimizer,
num_epochs=self.max_epochs,
cost=cost,
callbacks=callbacks)
print("[%s] Misclassification error = %.1f%%"
% (self.model_name, model.eval(dataset.test(), metric=Misclassification()) * 100))
print "[%s] Finished training!" % self.model_name
end = time.time()
print "[%s] Duration in seconds", end - start
return model
def test_model_training():
"""
Test model end2end training
"""
data_path = path.join(get_data_real_path(), 'np_semantic_segmentation_prepared_data.csv')
model_path = path.join(get_data_real_path(), 'np_semantic_segmentation.prm')
num_epochs = 200
be = gen_backend(batch_size=64)
# load data sets from file
data_set = NpSemanticSegData(data_path, train_to_test_ratio=0.8)
# train the mlp classifier
train_mlp_classifier(data_set, model_path, num_epochs, {})
assert path.isfile(path.join(get_data_real_path(), 'np_semantic_segmentation.prm')) is True
def __init__(self, state_size, num_steers, num_speeds, args):
# remember parameters
self.state_size = state_size
self.num_steers = num_steers
self.num_speeds = num_speeds
self.num_actions = num_steers + num_speeds
self.num_layers = args.hidden_layers
self.hidden_nodes = args.hidden_nodes
self.batch_size = args.batch_size
self.discount_rate = args.discount_rate
self.clip_error = args.clip_error
# create Neon backend
self.be = gen_backend(backend = args.backend,
batch_size = args.batch_size,
rng_seed = args.random_seed,
device_id = args.device_id,
datatype = np.dtype(args.datatype).type,
stochastic_round = args.stochastic_round)
# prepare tensors once and reuse them
self.input_shape = (self.state_size, self.batch_size)
self.input = self.be.empty(self.input_shape)
self.targets = self.be.empty((self.num_actions, self.batch_size))
# create model
self.model = Model(layers = self._createLayers())
self.cost = GeneralizedCost(costfunc = SumSquared())
self.model.initialize(self.input_shape[:-1], self.cost)
if args.optimizer == 'rmsprop':
self.optimizer = RMSProp(learning_rate = args.learning_rate,
decay_rate = args.decay_rate,
stochastic_round = args.stochastic_round)
elif args.optimizer == 'adam':
self.optimizer = Adam(learning_rate = args.learning_rate,
stochastic_round = args.stochastic_round)
elif args.optimizer == 'adadelta':
self.optimizer = Adadelta(decay = args.decay_rate,
stochastic_round = args.stochastic_round)
else:
assert false, "Unknown optimizer"
# create target model
self.target_steps = args.target_steps
self.train_iterations = 0
if self.target_steps:
self.target_model = Model(layers = self._createLayers())
self.target_model.initialize(self.input_shape[:-1])
self.save_weights_prefix = args.save_weights_prefix
else:
self.target_model = self.model
def run(subj):
"""
Train and perform inference on data from a single subject.
"""
try:
backend = gen_backend(rng_seed=0, gpu='nervanagpu')
except:
backend = gen_backend(rng_seed=0)
ds = GalData(subj=subj)
sumpreds = None
winlist = [1024] if validate else [768, 1024, 1280, 1536]
for winsize in winlist:
ds.setwin(winsize=winsize, subsample=16)
network = ConvNet(backend, ds, subj)
labs, preds, inds = network.fit().predict()
if sumpreds is None:
sumpreds = preds
else:
sumpreds += preds
if validate:
aucs = [auc(labs[:, i], sumpreds[:, i]) for i in range(ds.nclasses)]
print('Subject %d AUC %.4f' % (subj, np.mean(aucs)))
return labs, sumpreds, inds
def test_vs_numpy(custom_args):
test_idx, f, flag, dim, dtype, backend_type = custom_args
# backend
be = gen_backend(backend_type, default_dtype=dtype)
# tensors
tensors = gen_backend_tensors(
[np, be], 5, [dim] * 5, [flag] * 5, dtype=dtype)
# compare function value and gradient
numpy_func_val = call_func(f, np, tensors[0])
backend_func_val = call_func(f, be, tensors[1])
assert_tensors_allclose(
numpy_func_val, backend_func_val, rtol=1e-2, atol=1e-2)
def call_neon(params):
"""
runs the system call to neon and reads the result to give back to sm
"""
timestring = str(int(time.time()))
experiment_dir = os.path.realpath(os.environ['HYPEROPT_PATH'])
# Generate the yaml file
hyper_file = os.path.join(experiment_dir, 'hyperyaml.yaml')
yaml_file = os.path.join(experiment_dir, 'yamels',
'temp' + timestring + '.yaml')
try:
os.mkdir(os.path.join(experiment_dir, 'yamels'))
except OSError:
"Directory exists"
write_params(hyper_file, yaml_file, params)
# Initialize the neon experiment
logging.basicConfig(level=20)
experiment = deserialize(yaml_file)
backend = gen_backend(model=experiment.model) # , gpu='nervanagpu'
experiment.initialize(backend)
# ensure TOP1 error is calculated
if not hasattr(experiment, 'metrics'):
experiment.metrics = {'validation': [MisclassPercentage(error_rank=1)],
'test': [MisclassPercentage(error_rank=1)]}
for item in ['validation', 'test']:
if item not in experiment.metrics:
experiment.metrics[item] = [MisclassPercentage(error_rank=1)]
metriclist = [str(x) for x in experiment.metrics[item]]
if 'MisclassPercentage_TOP_1' not in metriclist:
experiment.metrics[item].append(MisclassPercentage(error_rank=1))
result = experiment.run()
# check if validation set is available
if experiment.dataset.has_set('validation'):
hyperopt_set = 'validation'
elif experiment.dataset.has_set('test'):
hyperopt_set = 'test'
print("Warning: No validation set found, performing hyperparameter "
"optimization on test set.")
else:
raise AttributeError("No error found.")
return result[hyperopt_set]['MisclassPercentage_TOP_1']
def backend_tests(request):
'''
Fixture that returns cpu and gpu backends for 16 and 32 bit
'''
be = gen_backend(backend=request.param[0],
datatype=request.param[1],
batch_size=128,
rng_seed=0)
# add a cleanup call - will run after all tests in module are done
def cleanup():
be = request.getfuncargvalue('backend_tests')
del be
request.addfinalizer(cleanup)
# tests using this fixture can access the backend object from
# backend or use the NervanaObject.be global
return be
def test_sr():
"""
Performs stochastic rounding with 1 bit mantissa for an addition operation
and checks that the resulting array is rounded correctly
"""
gpu = gen_backend(backend='gpu', stochastic_round=False)
n = 10
A = gpu.ones((n, n), dtype=np.float16)
B = gpu.ones((n, n), dtype=np.float16)
gpu.multiply(B, 0.1, out=B)
C = gpu.ones((n, n), dtype=np.float16)
C.rounding = 1
C[:] = A + B
C_host = C.get()
# make sure everything is either 1. (rounded down 1 bit) or 1.5 (rounded up 1 bit)
print C_host
assert sum([C_host.flatten()[i] in [1., 1.5] for i in range(n**2)]) == n**2
assert sum([C_host.flatten()[i] in [1.5] for i in range(n**2)]) > .1 * n**2
assert sum([C_host.flatten()[i] in [1.] for i in range(n**2)]) > .7 * n**2
def test_gradients(custom_args):
test_idx, f, flag, dim, dtype, backend_type = custom_args
be = gen_backend(backend_type, default_dtype=dtype)
# tensors
tensors = gen_backend_tensors(
[np, be], 5, [dim] * 5, [flag] * 5, dtype=dtype)
# compare function value and gradient
numpy_func_val = call_func(f, np, tensors[0])
backend_func_val = call_func(f, be, tensors[1])
numerical_gradient = get_numerical_gradient(f, tensors[0])
autodiff_gradient = get_audiff_gradient(f, be, tensors[1])
# TODO: stricter test to fix numerical issues
assert_tensors_allclose(
numpy_func_val, backend_func_val, rtol=1e-2, atol=1e-2)
assert_tensors_allclose(
numerical_gradient, autodiff_gradient, rtol=1e-02, atol=1e-3)
def __init__(self, num_actions, args):
# remember parameters
self.num_actions = num_actions
self.batch_size = args.batch_size
self.discount_rate = args.discount_rate
self.history_length = args.history_length
self.screen_dim = (args.screen_height, args.screen_width)
self.clip_error = args.clip_error
# create Neon backend
self.be = gen_backend(backend = args.backend,
batch_size = args.batch_size,
rng_seed = args.random_seed,
device_id = args.device_id,
default_dtype = np.dtype(args.datatype).type,
stochastic_round = args.stochastic_round)
# prepare tensors once and reuse them
self.input_shape = (self.history_length,) + self.screen_dim + (self.batch_size,)
self.tensor = self.be.empty(self.input_shape)
self.tensor.lshape = self.input_shape # needed for convolutional networks
self.targets = self.be.empty((self.num_actions, self.batch_size))
# create model
layers = self.createLayers(num_actions)
self.model = Model(layers = layers)
self.cost = GeneralizedCost(costfunc = SumSquared())
self.model.initialize(self.tensor.shape[:-1], self.cost)
self.optimizer = RMSProp(learning_rate = args.learning_rate,
decay_rate = args.rmsprop_decay_rate,
stochastic_round = args.stochastic_round)
# create target model
self.target_steps = args.target_steps
self.train_iterations = 0
if self.target_steps:
self.target_model = Model(layers = self.createLayers(num_actions))
self.target_model.initialize(self.tensor.shape[:-1])
self.save_weights_path = args.save_weights_path
else:
self.target_model = self.model
self.callback = None
def serialize_check_alexnet(conf_file, result, **be_args):
experiment = deserialize(os.path.join(dir, conf_file))
backend = gen_backend(model=experiment.model, **be_args)
experiment.initialize(backend)
res = experiment.run()
print float(res['validation']['MisclassPercentage_TOP_1']), result
dataset.test_set)
neon_logger.display('Misclassification error = %.1f%%' %
(error_rate * 100))
neon_logger.display('Test accuracy rate = %.1f%%' %
(test_accuracy_rate * 100))
neon_logger.display('precision rate = %s!!' %
(str(precision_recall_rate[0])))
neon_logger.display('recall rate = %s!!' % (str(precision_recall_rate[1])))
if __name__ == "__main__":
# parse the command line arguments
parser = NeonArgparser()
parser.set_defaults(epochs=200)
parser.add_argument(
'--data',
type=validate_existing_filepath,
help='Path to the CSV file where the prepared dataset is saved')
parser.add_argument('--model_path',
type=validate_parent_exists,
help='Path to save the model')
args = parser.parse_args()
data_path = absolute_path(args.data)
model_path = absolute_path(args.model_path)
# generate backend
be = gen_backend(batch_size=64)
# load data sets from file
data_set = NpSemanticSegData(data_path, train_to_test_ratio=0.8)
# train the mlp classifier
train_mlp_classifier(data_set, model_path, args.epochs, args.callback_args)
ds = GalData(subj=subj, winsize=winsize, subsample=16)
network = ConvNet(backend, ds, subj)
labs, preds, inds = network.fit().predict()
if sumpreds is None:
sumpreds = preds
else:
sumpreds += preds
if validate:
aucs = [auc(labs[:, i], sumpreds[:, i]) for i in range(ds.nclasses)]
print(('Subject %d AUC %.4f' % (subj, np.mean(aucs))))
return labs, sumpreds, inds
if __name__ == '__main__':
print(('\'validate\' is %s' % validate))
backend = gen_backend(rng_seed=0)
# Launch a separate process for each subject.
pool = Pool()
nsubjects = 12
results = pool.map(run, list(range(1, nsubjects + 1)))
pool.close()
labs = np.vstack([tup[0] for tup in results])
preds = np.vstack([tup[1] for tup in results])
if validate:
# Compute AUC metric.
nclasses = labs.shape[1]
aucs = [auc(labs[:, i], preds[:, i]) for i in range(nclasses)]
print(('Mean AUC %.4f' % np.mean(aucs)))
else:
# Generate submission file.
columns = [
err = dtypeu(np.random.random((nout, nin * batch_size)))
layer.bprop(layer.be.array(err)).get()
dw = layer.dW.get()
unqidx, count = np.unique(inp, return_counts=True)
dw_exp = np.zeros((1, nout))
for wrd_id, cnt in zip(unqidx, count):
dw_exp[:] = 0
cnt_exp = 0
for i, w_id in enumerate(inp):
if w_id == wrd_id:
dw_exp[:] = dw_exp[:] + err[:, i]
cnt_exp += 1
assert allclose_with_out(dw[wrd_id, :], dw_exp, atol=0, rtol=1e-4)
assert allclose_with_out(dw_exp, dw[wrd_id, :], atol=0, rtol=1e-4)
assert cnt == cnt_exp
return
if __name__ == '__main__':
fargs = [1, 128, 1, 1]
be = gen_backend(backend='cpu',
datatype=np.float32,
batch_size=128,
rng_seed=0)
test_lookuptable_zeros_error(be, fargs)
args = parser.parse_args(gen_be=False)
if args.model_file is None:
parser.print_usage()
exit('You need to specify model file to evaluate.')
if args.ssd_config:
args.ssd_config = {k: v for k, v in [ss.split(':') for ss in args.ssd_config]}
config = json.load(open(args.ssd_config['val']), object_pairs_hook=OrderedDict)
if args.batch_size == 0:
args.batch_size = config["batch_size"]
# setup backend
be = gen_backend(backend=args.backend, batch_size=args.batch_size,
device_id=args.device_id, compat_mode='caffe', rng_seed=1,
deterministic_update=True, deterministic=True)
be.enable_winograd = 0
config["manifest_root"] = args.manifest_root
config["manifest_filename"] = args.manifest['val']
config["batch_size"] = be.bsz
val_set = build_dataloader(config, args.manifest_root, args.batch_size)
model = Model(layers=SSD(ssd_config=config['ssd_config'], dataset=val_set))
model.initialize(dataset=val_set)
model.load_params(args.model_file)
if args.num_images > 0:
assert args.image_dir is not None, "Specify ---image_dir for path to save sampled images."
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
assert type(p) == type(p_e)
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
elif isinstance(p, np.ndarray):
assert np.allclose(p, p_e)
else:
assert p == p_e
os.remove(tmp_save)
if __name__ == '__main__':
be = gen_backend(backend='gpu', batch_size=50)
test_model_get_outputs_rnn(be, '~/nervana/data')
cpu_model_name = subp.check_output(cat_cmd, shell=True)
print('CPU model name = {}'.format(cpu_model_name))
else:
cpu_model_name = "unknown"
if cpu_model_name == 'CPU E5-2699\n':
platform = "BDW"
elif cpu_model_name == 'CPU 7250\n':
platform = "KNL"
# temporary identification for KNM model name
elif cpu_model_name == 'CPU 0000\n':
platform = "KNM"
else:
platform = "unknown"
print('Test platform = {}'.format(platform))
assert allclose_with_out(w1, w2, atol=1e-1, rtol=1e-3)
if __name__ == '__main__':
be_cpu = gen_backend(backend='cpu', rng_seed=0, batch_size=128)
fargs_tests = [3, 2, 1]
test_dilated_conv(be_cpu)
print('OK')
be_mkl = gen_backend(backend='mkl', rng_seed=0, batch_size=128)
fargs_tests = [3, 2, 1]
test_dilated_conv(be_mkl)
print('OK')
parser = NeonArgparser(__doc__)
parser.add_argument('--serialize', nargs='?', type=int,
default=0, const=1, metavar='N',
help='serialize model every N epochs')
parser.add_argument('--model_file', help='load model from pkl file')
args = parser.parse_args()
# hyperparameters
batch_size = 128
num_epochs = args.epochs
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype,
stochastic_round=False)
# load up the mnist data set
# split into train and tests sets
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
# setup a training set iterator
train_set = DataIterator(X_train, y_train, nclass=nclass)
# setup a validation data set iterator
valid_set = DataIterator(X_test, y_test, nclass=nclass)
# setup weight initialization function
init_norm = Gaussian(loc=0.0, scale=0.01)
outputs_fprop_be = outputs_dev.get().reshape(-1, rois_per_batch)
assert allclose_with_out(outputs_fprop_ref_in,
outputs_fprop_be,
atol=1e-6,
rtol=0)
start_time = timeit()
be.roipooling_bprop(input_error_dev, rois_dev, output_error_dev,
argmax_dev, rois_per_batch, img_fm_c, img_fm_h,
img_fm_w, roi_size, roi_size, spatial_scale)
neon_logger.display(
"NervanaGPU roipooling bprop (sec): {}".format(timeit() - start_time))
outputs_backend = output_error_dev.get()
assert allclose_with_out(outputs_fprop_ref_in,
outputs_backend,
atol=1e-6,
rtol=0)
if __name__ == '__main__':
bsz = 2
be = gen_backend(backend='gpu', batch_size=bsz, compat_mode='caffe')
# compare using random data
fargs = (2, 2, 62, 62, 6, bsz)
test_roipooling_fprop_random(be, fargs)
test_roipooling_bprop_random(be, fargs)
testFileName = 'manifest_subset9_augmented.csv'
#testFileName = 'manifest_subset9_ALL.csv'
# hyperparameters
num_epochs = args.epochs
# Next line gets rid of the deterministic warning
args.deterministic = None
if (args.rng_seed is None):
args.rng_seed = 16
print('Batch size = {}'.format(args.batch_size))
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
window_size = 64
# Set up the testset to load via aeon
image_config = dict(height=window_size, width=window_size, channels=3)
label_config = dict(binary=False)
config = dict(type="image,label",
image=image_config,
label=label_config,
manifest_filename=testFileName,
minibatch_size=args.batch_size,
subset_fraction=1,
cache_directory='')
test_set = DataLoader(config, be)
test_set = TypeCast(test_set, index=0, dtype=np.float32) # cast image to float
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
import numpy as np
import pytest
from neon import NervanaObject
from neon.backends import gen_backend
from neon.benchmark import Benchmark
from neon.data import ArrayIterator
from neon.initializers import Gaussian
from neon.layers import Conv, Affine, Pooling
from neon.models import Model
gen_backend(batch_size=16)
test_layers = [
Conv((2, 2, 4), init=Gaussian(scale=0.01), padding=3, strides=4),
Pooling(3, strides=2),
Affine(nout=10, init=Gaussian(scale=0.01))
]
NervanaObject.be.enable_winograd = 4
x = np.random.uniform(-1, 1, (16, 3 * 2 * 2))
y = np.random.randint(0, 9, (16, 10))
test_dataset = ArrayIterator(x, y, nclass=10, lshape=(3, 2, 2))
def test_empty_dataset():
model = Model(test_layers)
b = Benchmark(model=model)
with pytest.raises(ValueError):
self.send_response(active_idx)
elif active_idx == self.SERVER_RESET:
self._i1klib.reset(self.worker)
else:
print("Server Exiting")
break
if __name__ == "__main__":
from timeit import default_timer
from neon.backends import gen_backend
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
args = parser.parse_args()
be = gen_backend(backend='gpu', rng_seed=100)
NervanaObject.be.bsz = 128
master = ImgMaster(repo_dir=args.data_dir,
set_name='train',
inner_size=224,
subset_pct=10)
master.init_batch_provider()
t0 = default_timer()
total_time = 0
for epoch in range(3):
for x, t in master:
print "****", epoch, master.start, master.idx, master.ndata
print t.get().argmax(axis=0)[:17]
master.send_request(master.SERVER_KILL)
display_iter = 100
eval_iter = 100
store_img_iter = 100
save_iter = 1000
lr_init = 0.0002
batch_size = 100
zdim = 100
n_classes = 10
dropout = 0.2
im_size = [64, 64]
dname, gname = 'd_', 'g_'
tf.set_random_seed(1234)
# DataLoader
be = gen_backend(backend='cpu', batch_size=batch_size, datatype=np.float32)
root_files = './dataset/wikiart'
manifestfile = os.path.join(root_files, 'genre-train-index.csv')
testmanifest = os.path.join(root_files, 'genre-val-index.csv')
train = train_loader(manifestfile, root_files, be, h=im_size[0], w=im_size[1])
test = validation_loader(testmanifest,
root_files,
be,
h=im_size[0],
w=im_size[1])
OneHot = OneHot(be, n_classes)
# Graph input
is_train = tf.placeholder(tf.bool)
keep_prob = tf.placeholder(tf.float32)
x_n = tf.placeholder(tf.float32, [batch_size, 3, im_size[0], im_size[1]])
if __name__ == "__main__":
# parse the command line arguments
parser = NeonArgparser()
parser.add_argument('--data_set_file',
default='data/data_set.pkl',
type=validate_existing_filepath,
help='train and validation sets path')
parser.add_argument('--model_prm',
default='data/mcs_model.prm',
type=validate_parent_exists,
help='trained model full path')
args = parser.parse_args()
# generate backend, it is optional to change to backend='mkl'
be = gen_backend(backend='cpu', batch_size=10)
# read training and validation data file
with open(args.data_set_file, 'rb') as fp:
data_in = pickle.load(fp)
X_train = data_in['X_train']
X_valid = data_in['X_valid']
Y_train = data_in['y_train']
Y_valid = data_in['y_valid']
logger.info('training set size: %s', str(len(Y_train)))
logger.info('validation set size: %s', str(len(Y_valid)))
results = most_common_word_train(x_train=X_train,
y_train=Y_train,
from neon.transforms import Rectlin, CrossEntropyBinary, Misclassification, Softmax, Logistic
from neon.util.argparser import NeonArgparser
from neon import logger as neon_logger
from neon.data import ArrayIterator
from neon.backends import gen_backend
from load_aiff import load_data
parser = NeonArgparser(__doc__) # Use gpu
args = parser.parse_args(gen_be=False)
args.epochs = 13
args.batch_size = 200
gen_backend(backend=args.backend, # Modify backend
rng_seed=args.rng_seed,
device_id=args.device_id,
batch_size=args.batch_size,
datatype=args.datatype,
max_devices=args.max_devices,
compat_mode=args.compat_mode)
train_set_x, train_set_y, valid_set_x, valid_set_y = load_data("data/")
train_set = ArrayIterator(train_set_x, train_set_y, nclass=2, lshape=(1, 65, 122))
valid_set = ArrayIterator(valid_set_x, valid_set_y, nclass=2, lshape=(1, 65, 122))
# init_uni = Gaussian(loc=0.0, scale=0.01)
init_uni = Uniform(low=-0.1, high=0.1)
# setup model layers
hidden_size = 128
layers1 = [DeepBiLSTM(hidden_size, init_uni, activation=Rectlin(), gate_activation=Logistic()),
DeepBiLSTM(hidden_size, init_uni, activation=Rectlin(), gate_activation=Logistic()),
DeepBiLSTM(hidden_size, init_uni, activation=Rectlin(), gate_activation=Logistic()),
from neon.data import DataIterator, load_cifar10
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser
# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()
# hyperparameters
batch_size = 128
num_epochs = args.epochs
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
# really 10 classes, pad to nearest power of 2 to match conv output
train_set = DataIterator(X_train, y_train, nclass=16, lshape=(3, 32, 32))
valid_set = DataIterator(X_test, y_test, nclass=16, lshape=(3, 32, 32))
init_uni = GlorotUniform()
opt_gdm = GradientDescentMomentum(learning_rate=0.5,
schedule=Schedule(
step_config=[200, 250, 300], change=0.1),
momentum_coef=0.9,
wdecay=.0001)
model.initialize(train_shape, cost)
load(weights, model, K, fsz, dilation)
print(model)
model.optimizer = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9)
outputs = fprop(model, inp)
weights = bprop(model, outputs)
model.optimizer.optimize(model.layers_to_optimize, epoch=0)
return outputs.get(), weights.get()
def test_dilated_conv(backend_default, fargs_tests):
fsz = fargs_tests[0]
dil = fargs_tests[1]
stride = fargs_tests[2]
be = backend_default
o1, w1 = run(be, False, fsz, stride, 1, dil)
o2, w2 = run(be, True, fsz, stride, 1, dil)
# Verify that the results of faked dilation match those of actual dilation.
assert np.allclose(o1, o2, atol=0, rtol=3e-3)
assert np.allclose(w1, w2, atol=0, rtol=1e-3)
if __name__ == '__main__':
be = gen_backend(backend='cpu', rng_seed=0, batch_size=128)
fargs_tests = [3, 2, 1]
test_dilated_conv(be)
print('OK')
def test_branch_model_fork():
from neon.layers import BranchNode, Tree
NervanaObject.be = gen_backend("gpu", batch_size=64)
be = NervanaObject.be
bnode = BranchNode()
i1 = inception([(32, ), (32, 32), ('max', 16)])
top1 = top_branch()
top2 = top_branch()
p1 = Sequential(main_branch() + [bnode, i1] + top1)
p2 = [bnode] + top2
alpha2 = 0.3
neon_layer = Tree([p1, p2], alphas=[1.0, alpha2])
inshape = (3, 224, 224)
insize = np.prod(inshape)
inpa = np.random.random((insize, batch_size))
neon_layer.configure(inshape)
inp = neon_layer.be.array(inpa)
neon_layer.allocate()
print neon_layer.nested_str()
neon_layer.layers[0].layers[0].prev_layer = True
neon_layer.allocate_deltas()
neon_layer.layers[0].layers[0].set_deltas([be.iobuf(inshape)])
neon_out_dev = neon_layer.fprop(inp)
neon_out = [d.get() for d in neon_out_dev]
# Now make the reference pathways:
main_trunk2 = Sequential(main_branch())
main_trunk2.configure(inshape)
main2 = main_trunk2.layers
main2[0].prev_layer = True
main2[0].set_deltas([be.iobuf(inshape)])
branch2 = Sequential(top_branch())
lbranch2 = branch2.layers
(b1, b2, b3) = inception_bare(i1, [(32, ), (32, 32), ('max', 16)])
for bb in (b1, b2, b3, lbranch2):
oshape = inshape
for ll in main2 + bb:
oshape = ll.configure(oshape)
main1_trunk = neon_layer.layers[0].layers[:8]
for ll, lo in zip(main2, main1_trunk):
if ll.has_params:
ll.set_params(lo.W.get())
ll.allocate()
ll.set_deltas([be.iobuf(ll.in_shape)])
for ll, lo in zip(lbranch2, neon_layer.layers[1].layers[1:]):
if ll.has_params:
ll.set_params(lo.W.get())
for bb in (b1, b2, b3, lbranch2):
for ll in bb:
ll.allocate()
ll.set_deltas([be.iobuf(ll.in_shape)])
# Create the combined output buffer
merge_output = be.empty_like(neon_layer.layers[0].layers[9].outputs)
x = inp
for ll in main2:
x = ll.fprop(x)
main2_out = x
start = 0
for bb in (b1, b2, b3):
xb = main2_out
for ll in bb:
xb = ll.fprop(xb)
end = start + xb.shape[0]
merge_output[start:end] = xb
start = end
x = merge_output
top_trunk = Sequential(top1).layers
for ll in top_trunk:
x = ll.fprop(x)
neon_out_ref = x.get()
difference = neon_out_ref - neon_out[0]
assert np.max(np.abs(difference)) < 1e-7
print np.max(np.abs(difference))
# Now do second branch
neon_out_ref2 = branch2.fprop(main2_out).get()
difference = neon_out_ref2 - neon_out[1]
assert np.max(np.abs(difference)) < 1e-7
print np.max(np.abs(difference))
print "Beginning Back prop"
erra = [np.random.random(d.shape) for d in neon_out]
err = [be.array(d) for d in erra]
neon_layer.layers[0].layers[0].deltas = be.iobuf(inshape)
neon_layer.bprop(err)
bottom_neon_deltas = neon_layer.layers[0].layers[1].deltas.get()
middle_neon_deltas = neon_layer.layers[1].layers[1].deltas.get()
err0 = err[0]
for ll in reversed(top_trunk):
err0 = ll.bprop(err0)
err1 = err[1]
for ll in reversed(lbranch2):
err1 = ll.bprop(err1)
for bb, errb in zip((b1, b2, b3),
neon_layer.layers[0].layers[-5].error_views):
for ll in reversed(bb):
errb = ll.bprop(errb)
# Now sum up the deltas at the root of the branch layer and compare
ref_deltas = be.zeros_like(b1[0].deltas)
ref_deltas[:] = b1[0].deltas + b2[0].deltas + b3[
0].deltas + alpha2 * lbranch2[0].deltas
neon_ref_deltas = ref_deltas.get()
difference = middle_neon_deltas - neon_ref_deltas
print np.max(np.abs(difference))
assert np.max(np.abs(difference)) < 1e-8
x = ref_deltas
main2[0].deltas = be.iobuf(inshape)
for ll in reversed(main2):
x = ll.bprop(x)
bottom_neon_ref_deltas = main2[1].deltas.get()
difference = bottom_neon_deltas - bottom_neon_ref_deltas
print np.max(np.abs(difference))
assert np.max(np.abs(difference)) < 1e-8
from neon.callbacks import Callbacks
from neon.data import ArrayIterator
from neon.initializers import Uniform
from neon.layers import Conv, Pooling, Affine, GeneralizedCost, SkipNode
from neon.layers import Linear
from neon.models import Model
from neon.optimizers import GradientDescentMomentum
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Tanh, Logistic
import numpy as np
from neon.backends import gen_backend
import bot_params as params
import replay_memory as mem
be = gen_backend(backend='cpu', batch_size=params.batch_size)
init_uni = Uniform(low=-0.1, high=0.1)
bn = True
layers = [
Conv((4, 4, 32), init=init_uni, activation=Rectlin(), batch_norm=bn),
Conv((8, 8, 32), init=init_uni, activation=Rectlin(), batch_norm=bn),
Affine(nout=1000, init=init_uni, activation=Rectlin(), batch_norm=bn),
Affine(nout=1000, init=init_uni, activation=Rectlin(), batch_norm=bn),
Linear(nout=160 * 120, init=init_uni)
]
model = Model(layers=layers)
def test_branch_model():
NervanaObject.be = gen_backend("gpu", batch_size=64)
be = NervanaObject.be
main1 = main_branch()
i1 = inception([(32, ), (32, 32), ('max', 16)])
top = top_branch()
neon_layer = Sequential(main1 + i1 + top)
inshape = (3, 224, 224)
insize = np.prod(inshape)
inpa = np.random.random((insize, batch_size))
neon_layer.configure(inshape)
inp = neon_layer.be.array(inpa)
neon_layer.allocate()
print neon_layer.nested_str()
neon_layer.layers[0].prev_layer = True
neon_layer.allocate_deltas()
neon_layer.layers[0].set_deltas([be.iobuf(inshape)])
neon_out = neon_layer.fprop(inp).get()
# Now make the reference pathways:
main_trunk2 = Sequential(main_branch())
main_trunk2.configure(inshape)
main2 = main_trunk2.layers
main2[0].prev_layer = True
main2[0].set_deltas([be.iobuf(inshape)])
(b1, b2, b3) = inception_bare(i1, [(32, ), (32, 32), ('max', 16)])
for bb in (b1, b2, b3):
oshape = inshape
for ll in main2 + bb:
oshape = ll.configure(oshape)
main1_trunk = neon_layer.layers[:8]
for ll, lo in zip(main2, main1_trunk):
if ll.has_params:
ll.set_params(lo.W.get())
ll.allocate()
ll.set_deltas([be.iobuf(ll.in_shape)])
for bb in (b1, b2, b3):
for ll in bb:
ll.allocate()
ll.set_deltas([be.iobuf(ll.in_shape)])
# Create the combined output buffer
merge_output = be.empty_like(neon_layer.layers[8].outputs)
x = inp
for ll in main2:
x = ll.fprop(x)
start = 0
for bb in (b1, b2, b3):
xb = x
for ll in bb:
xb = ll.fprop(xb)
end = start + xb.shape[0]
merge_output[start:end] = xb
start = end
x = merge_output
top_trunk = Sequential(top).layers
for ll in top_trunk:
x = ll.fprop(x)
neon_out_ref = x.get()
difference = neon_out_ref - neon_out
assert np.max(np.abs(difference)) < 1e-7
print np.max(np.abs(difference))
print "Beginning Back prop"
erra = np.random.random(neon_out.shape)
err = be.array(erra)
for ll in reversed(neon_layer.layers[8:]):
err = ll.bprop(err)
neon_deltas = err.get()
for bb, errb in zip((b1, b2, b3), neon_layer.layers[8].error_views):
for ll in reversed(bb):
errb = ll.bprop(errb)
# Now sum up the deltas at the root of the branch layer and compare
ref_deltas = be.zeros_like(b1[0].deltas)
ref_deltas[:] = b1[0].deltas + b2[0].deltas + b3[0].deltas
neon_ref_deltas = ref_deltas.get()
difference = neon_deltas - neon_ref_deltas
print np.max(np.abs(difference))
assert np.max(np.abs(difference)) < 1e-8
def parse_args(self, gen_be=True):
'''
Parse the command line arguments and setup neon
runtime environment accordingly
Arguments:
gen_be (bool): if False, the arg parser will not
generate the backend
Returns:
namespace: contains the parsed arguments as attributes
'''
args = super(NeonArgparser, self).parse_args()
err_msg = None # used for relaying exception to logger
# set up the logging
# max thresh is 50 (critical only), min is 10 (debug or higher)
try:
log_thresh = max(10, 40 - args.verbose * 10)
except (AttributeError, TypeError):
# if defaults are not set or not -v given
# for latter will get type error
log_thresh = 40
args.log_thresh = log_thresh
# logging formater
fmtr = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# get the parent logger for neon
main_logger = logging.getLogger('neon')
main_logger.setLevel(log_thresh)
# setup a console stderr log handler
stderrlog = logging.StreamHandler()
stderrlog.setFormatter(fmtr)
if args.logfile:
# add log to file as well
filelog = RotatingFileHandler(filename=args.logfile,
mode='w',
maxBytes=10000000,
backupCount=5)
filelog.setFormatter(fmtr)
filelog.setLevel(log_thresh)
main_logger.addHandler(filelog)
# if a log file is specified and progress bar displayed,
# log only errors to console.
if args.no_progress_bar:
stderrlog.setLevel(log_thresh)
else:
stderrlog.setLevel(logging.ERROR)
else:
stderrlog.setLevel(log_thresh)
# add this handler instead
main_logger.propagate = False
main_logger.addHandler(stderrlog)
# need to write out float otherwise numpy
# generates type in bytes not bits (f16 == 128 bits)
args.datatype = 'float' + args.datatype[1:]
args.datatype = np.dtype(args.datatype).type
# invert no_progress_bar meaning and store in args.progress_bar
args.progress_bar = not args.no_progress_bar
if args.backend == 'cpu' and args.rounding > 0:
err_msg = 'CPU backend does not support stochastic rounding'
logger.exception(err_msg)
raise NotImplementedError(err_msg)
# done up front to avoid losing data due to incorrect path
if args.save_path:
savedir = os.path.dirname(os.path.abspath(args.save_path))
if not os.access(savedir, os.R_OK | os.W_OK):
err_msg = 'Can not write to save_path dir %s' % savedir
if os.path.exists(args.save_path):
logger.warning('save file %s exists, attempting to overwrite' %
args.save_path)
if not os.access(args.save_path, os.R_OK | os.W_OK):
err_msg = 'Can not write to save_path file %s' % args.save_path
if err_msg:
logger.exception(err_msg)
raise IOError(err_msg)
if (args.serialize > 0) and (args.save_path is None):
args.save_path = "neon_model.pkl"
logger.warn(
'No path given for model serialization, using default "%s"',
args.save_path)
if (args.save_path is not None) and (args.serialize == 0):
args.serialize = 1
logger.warn(
'No schedule given for model serialization, using default %d',
args.serialize)
if args.model_file:
err_msg = None
if not os.path.exists(args.model_file):
err_msg = 'Model file %s not present' % args.model_file
if not os.access(args.model_file, os.R_OK):
err_msg = 'No read access for model file %s' % args.model_file
if err_msg:
logger.exception(err_msg)
raise IOError(err_msg)
# extended parsers may need to generate backend after argparsing
if gen_be:
# generate the backend
gen_backend(backend=args.backend,
rng_seed=args.rng_seed,
device_id=args.device_id,
batch_size=args.batch_size,
datatype=args.datatype,
stochastic_round=args.rounding,
max_devices=args.max_devices)
# display what command line / config options were set (and from where)
logger.info(self.format_values())
self._PARSED = True
self.args = args
args.callback_args = extract_valid_args(args,
Callbacks.__init__,
startidx=1)
return args
running_error /= nprocessed
if math.isnan(float(running_error)):
running_error = tmp
break
neon_logger.display('Misclassification error = %.1f%%' %
(running_error * 100))
if __name__ == "__main__":
parser = NeonArgparser(__doc__)
args = parser.parse_args()
be = gen_backend(backend=args.backend,
batch_size=args.batch_size,
rng_seed=None,
device_id=args.device_id,
datatype=args.datatype)
# Change the path accordingly ...locating csv file
ip_file_path = "../../examples/gene"
# Change the Filename accordingly
filename = "splice_data_CAGT.csv"
names = ['class', 'C', 'A', 'G', 'T']
# Number of classes EI, IE and N
nclass = 3
# 20% of data used for validation
import numpy as np
import h5py
from neon.data import ArrayIterator,HDF5Iterator
from neon.initializers import Gaussian,GlorotUniform,Constant, Xavier
from neon.layers import GeneralizedCost, Affine, Sequential, MergeMultistream, Linear, Pooling, Conv,Dropout
from neon.models import Model
from neon.optimizers import GradientDescentMomentum, RMSProp
from neon.transforms import Rectlin, Logistic, CrossEntropyBinary, SumSquared, ObjectDetection
from neon.callbacks.callbacks import Callbacks
from neon.backends import gen_backend
# parser = NeonArgparser(__doc__)
# args = parser.parse_args(gen_be=False)
be = gen_backend(batch_size=2, backend='gpu')
traindir = 'train'
imwidth = 256
train_set = HDF5Iterator('whale_train.h5')
eval_set = HDF5Iterator('whale_eval.h5')
test_set = HDF5Iterator('whale_test.h5')
# weight initialization
init_norm = Gaussian(loc=0.0, scale=0.01)
# setup model layers
relu = Rectlin()
conv_params = {'strides': 1,
'padding': 1,
parser.add_argument('--plot_save_path',
type=str,
default='',
help='Path to save weight plots instead of displaying')
# parse the command line arguments (generates the backend)
args = parser.parse_args(gen_be=False)
print('emneon / neon options:')
print(args)
# setup backend
be_args = extract_valid_args(args, gen_backend)
# mutiple gpus accessing the cache dir for autotuning winograd was causing crashes / reboots
#be_args['cache_dir'] = tempfile.mkdtemp() # create temp dir
be_args['deterministic'] = None # xxx - why was this set?
be = gen_backend(**be_args)
# xxx - this doesn't work, interrupt is caught by neon for saving the model which then raises KeyboardInterrupt
#def signal_handler(signal, frame):
# #print('You pressed Ctrl+C!')
# shutil.rmtree(be_args['cache_dir']) # delete directory
#signal.signal(signal.SIGINT, signal_handler)
# this function modified from cuda-convnets2 shownet.py
def make_filter_fig(filters,
filter_start,
fignum,
_title,
num_filters,
combine_chans,
def main():
# Get command-line parameters
parser = get_p1b3_parser()
args = parser.parse_args()
#print('Args:', args)
# Get parameters from configuration file
fileParameters = p1b3.read_config_file(args.config_file)
#print ('Params:', fileParameters)
# Correct for arguments set by default by neon parser
# (i.e. instead of taking the neon parser default value fall back to the config file,
# if effectively the command-line was used, then use the command-line value)
# This applies to conflictive parameters: batch_size, epochs and rng_seed
if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv):
args.batch_size = fileParameters['batch_size']
if not any("--epochs" in ag or "-e" in ag for ag in sys.argv):
args.epochs = fileParameters['epochs']
if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv):
args.rng_seed = fileParameters['rng_seed']
# Consolidate parameter set. Command-line parameters overwrite file configuration
gParameters = p1_common.args_overwrite_config(args, fileParameters)
print('Params:', gParameters)
# Determine verbosity level
loggingLevel = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=loggingLevel, format='')
# Construct extension to save model
ext = p1b3.extension_from_parameters(gParameters, '.neon')
# Get default parameters for initialization and optimizer functions
kerasDefaults = p1_common.keras_default_config()
seed = gParameters['rng_seed']
# Build dataset loader object
loader = p1b3.DataLoader(
seed=seed,
dtype=gParameters['datatype'],
val_split=gParameters['validation_split'],
test_cell_split=gParameters['test_cell_split'],
cell_features=gParameters['cell_features'],
drug_features=gParameters['drug_features'],
feature_subsample=gParameters['feature_subsample'],
scaling=gParameters['scaling'],
scramble=gParameters['scramble'],
min_logconc=gParameters['min_logconc'],
max_logconc=gParameters['max_logconc'],
subsample=gParameters['subsample'],
category_cutoffs=gParameters['category_cutoffs'])
# Re-generate the backend after consolidating parsing and file config
gen_backend(backend=args.backend,
rng_seed=seed,
device_id=args.device_id,
batch_size=gParameters['batch_size'],
datatype=gParameters['datatype'],
max_devices=args.max_devices,
compat_mode=args.compat_mode)
# Initialize weights and learning rule
initializer_weights = p1_common_neon.build_initializer(
gParameters['initialization'], kerasDefaults, seed)
initializer_bias = p1_common_neon.build_initializer(
'constant', kerasDefaults, 0.)
activation = p1_common_neon.get_function(gParameters['activation'])()
# Define model architecture
layers = []
reshape = None
if 'dense' in gParameters: # Build dense layers
for layer in gParameters['dense']:
if layer:
layers.append(
Affine(nout=layer,
init=initializer_weights,
bias=initializer_bias,
activation=activation))
if gParameters['drop']:
layers.append(Dropout(keep=(1 - gParameters['drop'])))
else: # Build convolutional layers
reshape = (1, loader.input_dim, 1)
layer_list = list(range(0, len(gParameters['conv']), 3))
for l, i in enumerate(layer_list):
nb_filter = gParameters['conv'][i]
filter_len = gParameters['conv'][i + 1]
stride = gParameters['conv'][i + 2]
# print(nb_filter, filter_len, stride)
# fshape: (height, width, num_filters).
layers.append(
Conv((1, filter_len, nb_filter),
strides={
'str_h': 1,
'str_w': stride
},
init=initializer_weights,
activation=activation))
if gParameters['pool']:
layers.append(Pooling((1, gParameters['pool'])))
layers.append(
Affine(nout=1,
init=initializer_weights,
bias=initializer_bias,
activation=neon.transforms.Identity()))
# Build model
model = Model(layers=layers)
# Define neon data iterators
train_samples = int(loader.n_train)
val_samples = int(loader.n_val)
if 'train_samples' in gParameters:
train_samples = gParameters['train_samples']
if 'val_samples' in gParameters:
val_samples = gParameters['val_samples']
train_iter = ConcatDataIter(loader,
ndata=train_samples,
lshape=reshape,
datatype=gParameters['datatype'])
val_iter = ConcatDataIter(loader,
partition='val',
ndata=val_samples,
lshape=reshape,
datatype=gParameters['datatype'])
# Define cost and optimizer
cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])())
optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
callbacks = Callbacks(model, eval_set=val_iter,
eval_freq=1) #**args.callback_args)
model.fit(train_iter,
optimizer=optimizer,
num_epochs=gParameters['epochs'],
cost=cost,
callbacks=callbacks)
model_path = 'Googlenet_791113_192patch.prm'
model_URL = 'http://degas.ecs.soton.ac.uk/~productizer/Googlenet_791113_192patch.prm'
parser = NeonArgparser(__doc__)
parser.add_argument('--image', dest='image',
help="A string path to the location of an image readable by scipy's imread")
parser.add_argument('--prm-name', dest='prm_name', default= model_path,
help="The name of the prm to use as a model")
parser.add_argument('--layer', dest='layer_index', default=-4,
help="The index of the layer to extract the activations from")
args = parser.parse_args()
# load the cnn model
gen_backend(batch_size=1, backend='cpu')
# gen_backend(batch_size=32, backend='gpu')
model_dict = load_obj(args.prm_name)
model = Model(model_dict)
# now we are going to extract the middle patch from the image,
# based on the size used to train the model
patch_height = model_dict['train_input_shape'][1]
patch_width = model_dict['train_input_shape'][2]
# initialise the model so that internally the arrays are allocated to the correct size
model.initialize(model_dict['train_input_shape'])
# load the image
