def test_softmax_cputensor():
sftmx = Softmax()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
temp = be.zeros((3, 1))
outputs = np.exp(inputs-1) / np.sum(np.exp(inputs-1))
sftmx.apply_function(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_logistic_cputensor():
lgstc = Logistic()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
temp = be.zeros((3, 1))
outputs = 1.0 / (1.0 + np.exp(-inputs))
lgstc.apply_function(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_softmax_cputensor():
sftmx = Softmax()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
temp = be.zeros((3, 1))
outputs = np.exp(inputs - 1) / np.sum(np.exp(inputs - 1))
sftmx.apply_function(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_cross_entropy_derivative_cputensor():
be = CPU(rng_seed=0)
outputs = CPUTensor([0.5, 0.9, 0.1, 0.0001])
targets = CPUTensor([0.5, 0.99, 0.01, 0.2])
temp = [be.zeros(outputs.shape), be.zeros(outputs.shape)]
expected_result = ((outputs.asnumpyarray() - targets.asnumpyarray()) /
(outputs.asnumpyarray() * (1 - outputs.asnumpyarray())))
assert_tensor_near_equal(
expected_result, cross_entropy_derivative(be, outputs, targets, temp))
def test_tanh_derivative_cputensor():
tntest = Tanh()
inputs = np.array([0, 1, -2])
be = CPU(rng_seed=0)
outputs = np.array(
[1 - true_tanh(0)**2, 1 - true_tanh(1)**2, 1 - true_tanh(-2)**2])
temp = be.zeros(inputs.shape)
tntest.apply_derivative(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_logistic_derivative_cputensor():
lgstc = Logistic()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
outputs = 1.0 / (1.0 + np.exp(-inputs))
outputs = outputs * (1.0 - outputs)
temp = be.zeros(inputs.shape)
lgstc.apply_derivative(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_tanh_cputensor():
tntest = Tanh()
be = CPU(rng_seed=0)
CPUTensor([0, 1, -2])
inputs = np.array([0, 1, -2])
temp = be.zeros(inputs.shape)
outputs = np.array([true_tanh(0), true_tanh(1), true_tanh(-2)])
tntest.apply_function(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_tanh_cputensor():
tntest = Tanh()
be = CPU(rng_seed=0)
CPUTensor([0, 1, -2])
inputs = np.array([0, 1, -2])
temp = be.zeros(inputs.shape)
outputs = np.array([true_tanh(0), true_tanh(1), true_tanh(-2)])
tntest.apply_function(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_tanh_derivative_cputensor():
tntest = Tanh()
inputs = np.array([0, 1, -2])
be = CPU(rng_seed=0)
outputs = np.array([1 - true_tanh(0) ** 2,
1 - true_tanh(1) ** 2,
1 - true_tanh(-2) ** 2])
temp = be.zeros(inputs.shape)
tntest.apply_derivative(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_cross_entropy_derivative_cputensor():
be = CPU(rng_seed=0)
outputs = CPUTensor([0.5, 0.9, 0.1, 0.0001])
targets = CPUTensor([0.5, 0.99, 0.01, 0.2])
temp = [be.zeros(outputs.shape), be.zeros(outputs.shape)]
expected_result = ((outputs.asnumpyarray() - targets.asnumpyarray()) /
(outputs.asnumpyarray() * (1 - outputs.asnumpyarray())))
assert_tensor_near_equal(expected_result,
cross_entropy_derivative(be, outputs,
targets, temp))
def test_softmax_derivative_cputensor():
sftmx = Softmax()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
outputs = np.exp(inputs) / np.sum(np.exp(inputs))
errmat = np.ones(inputs.shape)
a = np.einsum('ij,ji->i', errmat.T, outputs)
outputs = outputs * (errmat - a[np.newaxis, :])
temp = be.zeros(inputs.shape)
sftmx.apply_derivative(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_cross_entropy_cputensor():
be = CPU(rng_seed=0)
outputs = CPUTensor([0.5, 0.9, 0.1, 0.0001])
targets = CPUTensor([0.5, 0.99, 0.01, 0.2])
temp = [be.zeros(outputs.shape), be.zeros(outputs.shape)]
expected_result = np.sum((- targets.asnumpyarray()) *
np.log(outputs.asnumpyarray()) -
(1 - targets.asnumpyarray()) *
np.log(1 - outputs.asnumpyarray()), keepdims=True)
assert_tensor_near_equal(expected_result, cross_entropy(be, outputs,
targets, temp))
def test_cross_entropy_cputensor():
be = CPU(rng_seed=0)
outputs = CPUTensor([0.5, 0.9, 0.1, 0.0001])
targets = CPUTensor([0.5, 0.99, 0.01, 0.2])
temp = [be.zeros(outputs.shape), be.zeros(outputs.shape)]
expected_result = np.sum(
(-targets.asnumpyarray()) * np.log(outputs.asnumpyarray()) -
(1 - targets.asnumpyarray()) * np.log(1 - outputs.asnumpyarray()),
keepdims=True)
assert_tensor_near_equal(expected_result,
cross_entropy(be, outputs, targets, temp))
def test_softmax_derivative_cputensor():
sftmx = Softmax()
inputs = np.array([0, 1, -2]).reshape((3, 1))
be = CPU(rng_seed=0)
outputs = np.exp(inputs) / np.sum(np.exp(inputs))
errmat = np.ones(inputs.shape)
a = np.einsum('ij,ji->i', errmat.T, outputs)
outputs = outputs * (errmat - a[np.newaxis, :])
temp = be.zeros(inputs.shape)
sftmx.apply_derivative(be, CPUTensor(inputs), temp)
assert_tensor_near_equal(CPUTensor(outputs), temp)
def test_rectleaky_slope_zero_rectlin_equiv():
be = CPU()
inputs = be.uniform(low=-5.0, high=10.0, size=(10, 10))
lin_buf = be.empty(inputs.shape)
leaky_buf = be.empty(inputs.shape)
be.rectlin(inputs, out=lin_buf)
be.rectleaky(inputs, slope=0.0, out=leaky_buf)
assert_tensor_equal(lin_buf, leaky_buf)
def __setstate__(self, state):
"""
Defines how we go about deserializing and loading an instance of this
class from a specified state.
Arguments:
state (dict): attribute values to be loaded.
"""
self.__dict__.update(state)
if self.backend is None:
# use CPU as a default backend
self.backend = CPU()
def test_cross_entropy_limits():
be = CPU(rng_seed=0)
outputs = CPUTensor([0.5, 1.0, 0.0, 0.0001])
targets = CPUTensor([0.5, 0.0, 1.0, 0.2])
eps = 2**-23
temp = [be.zeros(outputs.shape), be.zeros(outputs.shape)]
expected_result = np.sum(
(-targets.asnumpyarray()) * np.log(outputs.asnumpyarray() + eps) -
(1 - targets.asnumpyarray()) *
np.log(1 - outputs.asnumpyarray() + eps),
keepdims=True)
assert_tensor_near_equal(expected_result,
cross_entropy(be, outputs, targets, temp, eps))
def test_coarse_labels(self):
data = CIFAR100(coarse=True, repo_path=self.tmp_repo)
data.backend = CPU(rng_seed=0)
data.backend.actual_batch_size = 128
data.load()
assert len(data.inputs['train']) == 50000
assert len(data.targets['train'][0]) == 20
def test_get_inputs(self):
d = MNIST(repo_path=self.tmp_repo)
d.backend = CPU(rng_seed=0)
d.backend.actual_batch_size = 128
inputs = d.get_inputs(train=True)
# TODO: make this work (numpy import errors at the moment)
assert inputs['train'] is not None
def test_xcov_cputensor():
np.random.seed(0)
n = 10
k = 8
(k1, k2) = (3, 5)
a = np.array(np.random.randn(k, n)*10, dtype='float32', order='C')
acc = xcc(a[:k1], a[k1:])
expected_result = 0.5 * (acc**2.).sum()
be = CPU(rng_seed=0)
outputs = CPUTensor(a.copy())
tempbuf1 = be.empty((k1, n))
tempbuf2 = be.empty((k2, n))
tempbuf3 = be.empty((k1, k2))
tempbuf4 = be.empty(outputs.shape)
temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4]
my_result = xcov_cost(be, outputs, [], temp, k1)
assert_tensor_near_equal(expected_result, my_result)
def test_xcov_cputensor():
np.random.seed(0)
n = 10
k = 8
(k1, k2) = (3, 5)
a = np.array(np.random.randn(k, n) * 10, dtype='float32', order='C')
acc = xcc(a[:k1], a[k1:])
expected_result = 0.5 * (acc**2.).sum()
be = CPU(rng_seed=0)
outputs = CPUTensor(a.copy())
tempbuf1 = be.empty((k1, n))
tempbuf2 = be.empty((k2, n))
tempbuf3 = be.empty((k1, k2))
tempbuf4 = be.empty(outputs.shape)
temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4]
my_result = xcov_cost(be, outputs, [], temp, k1)
assert_tensor_near_equal(expected_result, my_result)
def test_fine_labels(self):
data = CIFAR100(coarse=False, repo_path=self.tmp_repo)
data.backend = CPU(rng_seed=0)
data.backend.actual_batch_size = 128
par = NoPar()
par.associate(data.backend)
data.load()
assert len(data.inputs['train']) == 50000
assert len(data.targets['train'][0]) == 100
def __setstate__(self, state):
"""
Defines how we go about deserializing and loading an instance of this
class from a specified state.
Arguments:
state (dict): attribute values to be loaded.
"""
self.__dict__.update(state)
if self.backend is None:
# use CPU as a default backend
self.backend = CPU()
def test_xcov_derivative_cputensor():
np.random.seed(0)
n = 10
k = 8
(k1, k2) = (3, 5)
a = np.array(np.random.randn(k, n), dtype='float32', order='C')
s = np.zeros_like(a)
acc = xcc(a[:k1], a[k1:]) # k1 x k2
c1 = a[k1:] - a[k1:].mean(1, keepdims=True) # k2 x n
c2 = a[:k1] - a[:k1].mean(1, keepdims=True) # k1 x n
s[:k1] = acc.dot(c1) / n
s[k1:] = acc.T.dot(c2) / n
be = CPU(rng_seed=0)
outputs = CPUTensor(a.copy())
tempbuf1 = be.empty((k1, n))
tempbuf2 = be.empty((k2, n))
tempbuf3 = be.empty((k1, k2))
tempbuf4 = be.empty(outputs.shape)
temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4]
my_result = xcov_cost_derivative(be, outputs, [], temp, k1)
expected_result = CPUTensor(s)
assert_tensor_near_equal(expected_result, my_result)
def test_xcov_derivative_cputensor():
np.random.seed(0)
n = 10
k = 8
(k1, k2) = (3, 5)
a = np.array(np.random.randn(k, n), dtype='float32', order='C')
s = np.zeros_like(a)
acc = xcc(a[:k1], a[k1:]) # k1 x k2
c1 = a[k1:] - a[k1:].mean(1, keepdims=True) # k2 x n
c2 = a[:k1] - a[:k1].mean(1, keepdims=True) # k1 x n
s[:k1] = acc.dot(c1)/n
s[k1:] = acc.T.dot(c2)/n
be = CPU(rng_seed=0)
outputs = CPUTensor(a.copy())
tempbuf1 = be.empty((k1, n))
tempbuf2 = be.empty((k2, n))
tempbuf3 = be.empty((k1, k2))
tempbuf4 = be.empty(outputs.shape)
temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4]
my_result = xcov_cost_derivative(be, outputs, [], temp, k1)
expected_result = CPUTensor(s)
assert_tensor_near_equal(expected_result, my_result)
def compare_cpu_tensors(inputs, outputs, deriv=False):
rlin = RectLeaky()
be = CPU()
temp = be.zeros(inputs.shape)
if deriv is True:
rlin.apply_derivative(be, CPUTensor(inputs), temp)
else:
rlin.apply_function(be, CPUTensor(inputs), temp)
be.subtract(temp, CPUTensor(outputs), temp)
assert_tensor_equal(temp, be.zeros(inputs.shape))
def test_round_split(self):
split = 10
batch_size = 32
ntrain, ntest, nin, nout = 100, 10, 10, 5
data = UniformRandom(ntrain, ntest, nin, nout, validation_pct=split)
data.backend = CPU(rng_seed=0)
data.backend.batch_size = batch_size
data.load()
split /= 100.0
nb_batches = ntrain // batch_size
expected_nb_train = floor((1.0 - split) * nb_batches)
expected_nb_valid = floor(split * nb_batches)
assert expected_nb_train == len(data.inputs['train'])
assert expected_nb_train == len(data.targets['train'])
assert expected_nb_valid == len(data.inputs['validation'])
assert expected_nb_valid == len(data.targets['validation'])
def gen_backend(model=None,
gpu=None,
nrv=False,
datapar=False,
modelpar=False,
flexpoint=False,
rng_seed=None,
numerr_handling=None,
half=False,
stochastic_round=0,
device_id=None):
"""
Construct and return a backend instance of the appropriate type based on
the arguments given. With no parameters, a single CPU core, float32
backend is returned.
Arguments:
model (neon.models.model.Model): The instantiated model upon which we
will utilize this backend.
gpu (string, optional): Attempt to utilize a CUDA capable GPU if
installed in the system. Defaults to None which
implies a CPU based backend. If 'cudanet',
utilize a cuda-convnet2 based backed, which
supports Kepler and Maxwell GPUs with single
precision. If 'nervanagpu', attempt to utilize
the NervanaGPU Maxwell backend with float16 and
float32 support.
nrv (bool, optional): If True, attempt to utilize the Nervana Engine
for computation (must be installed on the
system). Defaults to False which implies a CPU
based backend.
datapar (bool, optional): Set to True to ensure that data is
partitioned and each chunk is processed in
parallel on different compute cores. Requires
mpi4py. Defaults to False which implies that
all data will be processed sequentially on a
single compute core.
modelpar (bool, optional): Set to True to ensure that the nodes in each
model layer are partitioned and distributed
across multiple compute cores. Requires
mpi4py. Defaults to False which implies
that all nodes in all model layers will be
processed by the same single compute core.
flexpoint (bool, optional): If True, attempt to use FlexPoint(TM)
element typed data instead of the default
float32 which is in place if set to False.
rng_seed (numeric, optional): Set this to a numeric value which can be
used to seed the random number generator
of the instantiated backend. Defaults to
None, which doesn't explicitly seed (so
each run will be different)
stochastic_round (numeric, optional): Only affects the max backend. If
1, perform stochastic rounding.
If 0, round to nearest.
numerr_handling (dict, optional): Dictate how numeric errors are
displayed and handled. The keys and
values permissible for this dict
match that seen in numpy.seterr.
If set to None (the default),
behavior is equivalent to
{'all': 'warn'}
device_id (numeric, optional): Set this to a numeric value which can be
used to select which device to run the
process on
Returns:
Backend: newly constructed backend instance of the specifed type.
Notes:
* Attempts to construct a GPU instance without a CUDA capable card or
without cudanet or nervanagpu package installed will cause the
program to display an error message and exit.
* Attempts to construct a parallel instance without mpi4py installed
will cause the program to display an error message and exit.
* The returned backend will still need to call its par.init_model()
at some point after the model has been linked, in order for parallel
training to proceed.
"""
logger = logging.getLogger(__name__)
gpuflag = False
if datapar and modelpar:
raise NotImplementedError('Hybrid parallelization scheme not '
'implemented yet. Try with at most one of'
'datapar or modelpar')
if modelpar:
par = ModelPar()
elif datapar:
par = DataPar()
else:
par = NoPar()
if par.device_id is not None:
if device_id is not None:
logger.warn('Ignoring device id specified in command line.')
device_id = par.device_id
if gpu is not None:
gpu = gpu.lower()
if sys.platform.startswith("linux"):
gpuflag = (os.system("nvidia-smi > /dev/null 2>&1") == 0)
elif sys.platform.startswith("darwin"):
gpuflag = (
os.system("kextstat | grep -i cuda > /dev/null 2>&1") == 0)
if gpuflag and gpu == 'cudanet':
try:
import cudanet # noqa
from neon.backends.cc2 import GPU
be_name = 'Cudanet'
be = GPU(rng_seed=rng_seed, device_id=device_id)
except ImportError:
logger.warning("cudanet not found, can't run via GPU")
gpuflag = False
elif gpuflag and gpu == 'nervanagpu':
try:
import nervanagpu # noqa
try:
# import pycuda.autoinit
import pycuda.driver as drv
drv.init()
device_id = device_id if device_id is not None else 0
global ctx
ctx = drv.Device(device_id).make_context()
import atexit
atexit.register(ctx.pop)
from neon.backends.gpu import GPU
be_name = 'NervanaGPU'
be = GPU(rng_seed=rng_seed,
stochastic_round=stochastic_round,
device_id=device_id)
except ImportError:
logger.warning("pycuda error, can't run via GPU")
gpuflag = False
except ImportError:
logger.warning("nervanagpu not found, can't run via GPU")
gpuflag = False
if gpuflag is False:
raise RuntimeError("Can't find CUDA capable GPU")
elif nrv:
nrv = False
try:
from umd.nrv_backend import NRVBackend
nrv = True
except ImportError:
logger.warning("Nervana Engine system software not found")
if flexpoint:
logger.warning("Flexpoint(TM) backend not currently available")
if nrv:
be_name = 'NRV'
be = NRVBackend(rng_seed=rng_seed,
seterr_handling=numerr_handling,
device_id=device_id)
elif not gpuflag:
be_name = 'CPU'
be = CPU(rng_seed=rng_seed, seterr_handling=numerr_handling)
logger.info("{} backend, RNG seed: {}, numerr: {}".format(
be_name, rng_seed, numerr_handling))
par.associate(be)
return be
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
class Dataset(object):
"""
Base dataset class. Defines interface operations.
"""
backend = None
inputs = {'train': None, 'test': None, 'validation': None}
targets = {'train': None, 'test': None, 'validation': None}
def __getstate__(self):
"""
Defines what and how we go about serializing an instance of this class.
In this case we also want to include any loaded datasets and backend
references.
Returns:
dict: keyword args, plus current inputs, targets, backend
"""
self.__dict__['backend'] = self.backend
self.__dict__['inputs'] = self.inputs
self.__dict__['targets'] = self.targets
return self.__dict__
def __setstate__(self, state):
"""
Defines how we go about deserializing and loading an instance of this
class from a specified state.
Arguments:
state (dict): attribute values to be loaded.
"""
self.__dict__.update(state)
if self.backend is None:
# use CPU as a default backend
self.backend = CPU()
def set_batch_size(self, batch_size):
self.batch_size = batch_size
def load(self, backend=None, experiment=None):
"""
Makes the dataset data available for use.
Needs to be implemented in every concrete Dataset child class.
Arguments:
backend (neon.backends.backend.Backend, optional): The
underlying data structure type used to hold this
data once loaded. If None will use
`neon.backends.cpu.CPU`
experiment (neon.experiments.experiment.Experiment, optional): The
object that loads this dataset.
Raises:
NotImplementedError: should be overridden in child class
"""
raise NotImplementedError()
def unload(self):
"""
Perform cleanup tasks if any are required.
"""
pass
def download_to_repo(self, url, repo_path):
"""
Fetches the dataset to a local repository for future use.
Arguments:
url (str): The external URI to a specific dataset
repo_path (str): The local path to write the fetched dataset to.
"""
repo_path = os.path.expandvars(os.path.expanduser(repo_path))
logger.info(
"fetching: %s, saving to: %s (this may take some time "
"depending on dataset size)", url, repo_path)
urllib.urlretrieve(url, os.path.join(repo_path, os.path.basename(url)))
def get_inputs(self,
backend=None,
train=True,
test=False,
validation=False):
"""
Loads and returns one or more input datasets.
Arguments:
backend (neon.backends.backend.Backend, optional): The underlying
data structure type used to hold this data once loaded.
If None will use whatever is set for this class
train (bool, optional): load a training target outcome dataset.
Defaults to True.
test (bool, optional): load a hold-out test target outcome dataset.
Defaults to False.
validation (bool, optional): Load a separate validation target
outcome dataset. Defaults to False.
Returns:
dict: of loaded datasets with keys train, test, validation
based on what was requested. Each dataset is a
neon.backends.backend.Tensor instance.
"""
res = dict()
if self.inputs['train'] is None:
if backend is not None:
self.load(backend)
else:
self.load()
if train and self.inputs['train'] is not None:
res['train'] = self.inputs['train']
if test and self.inputs['test'] is not None:
res['test'] = self.inputs['test']
if validation and self.inputs['validation'] is not None:
res['validation'] = self.inputs['validation']
return res
def get_targets(self,
backend=None,
train=True,
test=False,
validation=False):
"""
Loads and returns one or more labelled outcome datasets.
Arguments:
backend (neon.backends.backend.Backend, None): The underlying
data structure type used to hold this data once loaded.
If None will use whatever is set for this class
train (bool, optional): load a training target outcome dataset.
Defaults to True.
test (bool, optional): load a hold-out test target outcome dataset.
Defaults to False.
validation (bool, optional): Load a separate validation target
outcome dataset. Defaults to False.
Returns:
dict: of loaded datasets with keys train, test, validation
based on what was requested. Each dataset is a
neon.backends.backend.Tensor instance.
"""
# can't have targets without inputs, ensure these are loaded
res = dict()
if self.inputs['train'] is None:
self.load()
if train and self.inputs['train'] is not None:
res['train'] = self.targets['train']
if test and self.inputs['test'] is not None:
res['test'] = self.targets['test']
if validation and self.inputs['validation'] is not None:
res['validation'] = self.targets['validation']
return res
def sample_training_data(self):
"""
Carries out actual downsampling of data, to the percentage specified in
self.sample_pct.
"""
if self.sample_pct != 100:
train_idcs = np.arange(self.inputs['train'].shape[0])
ntrain_actual = (self.inputs['train'].shape[0] *
int(self.sample_pct) / 100)
np.random.seed(self.backend.rng_seed)
np.random.shuffle(train_idcs)
train_idcs = train_idcs[0:ntrain_actual]
self.inputs['train'] = self.inputs['train'][train_idcs]
self.targets['train'] = self.targets['train'][train_idcs]
def transpose_batches(self, data):
"""
Transpose and distribute each minibatch within a dataset.
Arguments:
data (ndarray): Dataset to be sliced into mini batches,
transposed, and loaded to appropriate device
memory.
Returns:
list: List of device loaded mini-batches of data.
"""
bs = self.backend.actual_batch_size
if data.shape[0] % bs != 0:
logger.warning('Incompatible batch size. Discarding %d samples...',
data.shape[0] % bs)
nbatches = data.shape[0] / bs
batchwise = []
for batch in range(nbatches):
batchdata = np.empty((data.shape[1], bs))
batchdata[...] = data[batch * bs:(batch + 1) * bs].transpose()
dev_batchdata = self.backend.distribute(batchdata)
batchwise.append(dev_batchdata)
return batchwise
def format(self):
"""
Transforms the loaded data into the format expected by the
backend. If a hardware accelerator device is being used,
this function also copies the data to the device memory.
"""
assert self.backend is not None
for dataset in (self.inputs, self.targets):
for key in dataset:
item = dataset[key]
if item is not None:
dataset[key] = self.transpose_batches(item)
def get_batch(self, data, batch):
"""
Extract and return a single batch from the data specified.
Arguments:
data (list): List of device loaded batches of data
batch (int): 0-based index specifying the batch number to get
Returns:
neon.backends.Tensor: Single batch of data
See Also:
transpose_batches
"""
return data[batch]
def has_set(self, setname):
"""
Indicate whether the specified setname type is part of this dataset.
Arguments:
setname (str): The type of data to look for. Typically this is one
of 'train', 'test', 'validation'.
Returns:
bool: True if this dataset contains setname type of data, and False
otherwise.
"""
inputs_dic = self.get_inputs(train=True, validation=True, test=True)
return True if (setname in inputs_dic) else False
def init_mini_batch_producer(self, batch_size, setname, predict):
"""
Setup the ability to generate mini-batches.
Arguments:
batch_size (int): The number of data examples will be contained in
each mini-batch
setname (str): The type of data to produce mini-batches for:
'train', 'test', 'validation'
predict (bool): Set this to False when training a model, or True
when generating batches to be used for prediction.
Returns:
int: The total number of examples to be mini-batched.
Notes:
This is the implementation for non-macro batched data.
macro-batched datasets will override this (e.g. ImageNet)
"""
self.cur_inputs = self.get_inputs(train=True,
validation=True,
test=True)[setname]
self.cur_tgts = self.get_targets(train=True,
validation=True,
test=True)[setname]
self.predict_mode = predict
return len(self.inputs[setname])
def get_mini_batch(self, batch_idx):
"""
Return the specified mini-batch of input and target data.
Arguments:
batch_idx (int): 0-based index specifying the mini-batch number to
retrieve.
Returns:
tuple: 2-tuple of neon.backend.Tensor objects containing the
corresponding input, and target mini-batches.
Notes:
This is the implementation for non-macro batched data.
macro-batched datasets will override this (e.g. ImageNet)
"""
return self.get_batch(self.cur_inputs, batch_idx), self.get_batch(
self.cur_tgts, batch_idx)
def del_mini_batch_producer(self):
"""
Perform any cleanup needed once all mini-batches have been produced.
Notes:
macro-batched datasets will likely override this
"""
pass
class TestCPU(object):
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
def test_empty_creation(self):
tns = self.be.empty((4, 3))
assert tns.shape == (4, 3)
def test_array_creation(self):
tns = self.be.array([[1, 2], [3, 4]])
assert tns.shape == (2, 2)
assert_tensor_equal(tns, CPUTensor([[1, 2], [3, 4]]))
def test_zeros_creation(self):
tns = self.be.zeros([3, 1])
assert tns.shape == (3, 1)
assert_tensor_equal(tns, CPUTensor([[0], [0], [0]]))
def test_ones_creation(self):
tns = self.be.ones([1, 4])
assert tns.shape == (1, 4)
assert_tensor_equal(tns, CPUTensor([[1, 1, 1, 1]]))
def test_all_equal(self):
left = self.be.ones([2, 2])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]]))
def test_some_equal(self):
left = self.be.ones([2, 2])
right = self.be.array([[0, 1], [0, 1]])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 1], [0, 1]]))
def test_none_equal(self):
left = self.be.ones([2, 2])
right = self.be.zeros([2, 2])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]]))
def test_all_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.zeros([2, 2])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]]))
def test_some_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.array([[0, 1], [0, 1]])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 0], [1, 0]]))
def test_none_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]]))
def test_greater(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.greater(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 1]]))
def test_greater_equal(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.greater_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [1, 1]]))
def test_less(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.less(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [0, 0]]))
def test_less_equal(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.less_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 0]]))
def test_argmin_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmin(tsr, None, out)
assert_tensor_equal(out, CPUTensor([[0]]))
def test_argmin_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((1, 2))
be.argmin(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([[0, 0]]))
def test_argmin_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, 1))
be.argmin(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([0, 1]))
def test_argmax_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmax(tsr, None, out)
assert_tensor_equal(out, CPUTensor(3))
def test_argmax_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((2, ))
be.argmax(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([1, 1]))
def test_argmax_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, ))
be.argmax(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([1, 0]))
def test_2norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
rpow = 1. / 2
# -> sum([[1, 0], [1, 9]], axis=0)**.5 -> sqrt([2, 9])
assert_tensor_equal(self.be.norm(tsr, order=2, axis=0),
CPUTensor([[2**rpow, 9**rpow]]))
# -> sum([[1, 0], [1, 9]], axis=1)**.5 -> sqrt([1, 10])
assert_tensor_equal(self.be.norm(tsr, order=2, axis=1),
CPUTensor([1**rpow, 10**rpow]))
def test_1norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> sum([[1, 0], [1, 3]], axis=0)**1 -> [2, 3]
assert_tensor_equal(self.be.norm(tsr, order=1, axis=0),
CPUTensor([[2, 3]]))
# -> sum([[1, 0], [1, 3]], axis=1)**1 -> [1, 4]
assert_tensor_equal(self.be.norm(tsr, order=1, axis=1),
CPUTensor([1, 4]))
def test_0norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> sum(tsr != 0, axis=0) -> [2, 1]
assert_tensor_equal(self.be.norm(tsr, order=0, axis=0),
CPUTensor([[2, 1]]))
# -> sum(tsr != 0, axis=1) -> [1, 2]
assert_tensor_equal(self.be.norm(tsr, order=0, axis=1),
CPUTensor([1, 2]))
def test_infnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> max(abs(tsr), axis=0) -> [1, 3]
assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=0),
CPUTensor([[1, 3]]))
# -> max(abs(tsr), axis=1) -> [1, 3]
assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=1),
CPUTensor([1, 3]))
def test_neginfnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> min(abs(tsr), axis=0) -> [1, 0]
assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=0),
CPUTensor([[1, 0]]))
# -> min(abs(tsr), axis=1) -> [0, 1]
assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=1),
CPUTensor([0, 1]))
def test_lrgnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
rpow = 1. / 5
# -> sum([[1, 0], [1, 243]], axis=0)**rpow -> rpow([2, 243])
assert_tensor_equal(self.be.norm(tsr, order=5, axis=0),
CPUTensor([[2**rpow, 243**rpow]]))
# -> sum([[1, 0], [1, 243]], axis=1)**rpow -> rpow([1, 244])
# 244**.2 == ~3.002465 hence the near_equal test
assert_tensor_near_equal(self.be.norm(tsr, order=5, axis=1),
CPUTensor([1**rpow, 244**rpow]), 1e-6)
def test_negnorm(self):
tsr = self.be.array([[-1, -2], [1, 3]])
rpow = -1. / 3
# -> sum([[1, .125], [1, .037037]], axis=0)**rpow -> rpow([2, .162037])
assert_tensor_equal(self.be.norm(tsr, order=-3, axis=0),
CPUTensor([[2**rpow, .162037037037**rpow]]))
# -> sum([[1, .125], [1, .037037]], axis=1)**rpow ->
# rpow([1.125, 1.037037])
assert_tensor_near_equal(self.be.norm(tsr, order=-3, axis=1),
CPUTensor([1.125**rpow, 1.037037**rpow]),
1e-6)
class TestValInit(object):
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
def test_uni_basics(self):
uni = UniformValGen(backend=self.be)
assert str(uni) == ("UniformValGen utilizing CPU backend\n\t"
"low: 0.0, high: 1.0")
def test_uni_gen(self):
uni = UniformValGen(backend=self.be)
res = uni.generate(shape=[1, 1])
assert res.shape == (1, 1)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
assert out.asnumpyarray() >= 0.0
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < 1.0
def test_uni_params(self):
low = -5.5
high = 10.2
uni = UniformValGen(backend=self.be, low=low, high=high)
assert str(uni) == ("UniformValGen utilizing CPU backend\n\t"
"low: {low}, high: {high}".format(low=low,
high=high))
res = uni.generate(shape=[4, 4])
assert res.shape == (4, 4)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
assert out.asnumpyarray() >= low
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < high
def test_autouni_gen(self):
autouni = AutoUniformValGen(backend=self.be, relu=True)
assert autouni.relu is True
assert str(autouni) == ("AutoUniformValGen utilizing CPU backend\n\t"
"low: nan, high: nan")
res = autouni.generate([3, 3])
assert res.shape == (3, 3)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
expected_val = math.sqrt(2) * (1.0 / math.sqrt(3))
assert out.asnumpyarray() >= - expected_val
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < expected_val
def test_gaussian_gen(self):
loc = 5
scale = 2.0
gauss = GaussianValGen(backend=self.be, loc=loc, scale=scale)
assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t"
"loc: {}, scale: {}".format(loc, scale))
res = gauss.generate([5, 10])
assert res.shape == (5, 10)
# TODO: test distribution of vals to ensure ~gaussian dist
def test_normal_gen(self):
loc = -2.5
scale = 3.0
gauss = NormalValGen(backend=self.be, loc=loc, scale=scale)
assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t"
"loc: {}, scale: {}".format(loc, scale))
res = gauss.generate([9, 3])
assert res.shape == (9, 3)
# TODO: test distribution of vals to ensure ~gaussian dist
def test_sparseeig_gen(self):
sparseness = 10
eigenvalue = 3.1
eig = SparseEigenValGen(backend=self.be, sparseness=sparseness,
eigenvalue=eigenvalue)
assert str(eig) == ("SparseEigenValGen utilizing CPU backend\n\t"
"sparseness: {}, eigenvalue: "
"{}".format(sparseness, eigenvalue))
res = eig.generate([20, 20])
assert res.shape == (20, 20)
# TODO: test distribution of vals
def test_nodenorm_gen(self):
scale = 3.0
nodenorm = NodeNormalizedValGen(backend=self.be, scale=scale)
assert str(nodenorm) == ("NodeNormalizedValGen utilizing CPU backend"
"\n\tscale: {}".format(scale))
res = nodenorm.generate([8, 9])
assert res.shape == (8, 9)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
expected_val = scale * math.sqrt(6) / math.sqrt(8 + 9.)
assert out.asnumpyarray() >= - expected_val
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < expected_val
def test_argmin_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((1, 2))
be.argmin(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([[0, 0]]))
def test_argmax_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmax(tsr, None, out)
assert_tensor_equal(out, CPUTensor(3))
def test_argmin_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((1, 2))
be.argmin(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([[0, 0]]))
class Dataset(object):
"""
Base dataset class. Defines interface operations.
"""
backend = None
inputs = {'train': None, 'test': None, 'validation': None}
targets = {'train': None, 'test': None, 'validation': None}
def __getstate__(self):
"""
Defines what and how we go about serializing an instance of this class.
In this case we also want to include any loaded datasets and backend
references.
Returns:
dict: keyword args, plus current inputs, targets, backend
"""
self.__dict__['backend'] = self.backend
self.__dict__['inputs'] = self.inputs
self.__dict__['targets'] = self.targets
return self.__dict__
def __setstate__(self, state):
"""
Defines how we go about deserializing and loading an instance of this
class from a specified state.
Arguments:
state (dict): attribute values to be loaded.
"""
self.__dict__.update(state)
if self.backend is None:
# use CPU as a default backend
self.backend = CPU()
def set_batch_size(self, batch_size):
self.batch_size = batch_size
def load(self, backend=None, experiment=None):
"""
Makes the dataset data available for use.
Needs to be implemented in every concrete Dataset child class.
Arguments:
backend (neon.backends.backend.Backend, optional): The
underlying data structure type used to hold this
data once loaded. If None will use
`neon.backends.cpu.CPU`
experiment (neon.experiments.experiment.Experiment, optional): The
object that loads this dataset.
Raises:
NotImplementedError: should be overridden in child class
"""
raise NotImplementedError()
def unload(self):
"""
Perform cleanup tasks if any are required.
"""
pass
def process_result(self, result):
"""
Accept and process results of running inference.
Arguments:
result (ndarray): Array containing predictions obtained by
processing a minibatch of input data.
"""
pass
def download_to_repo(self, url, repo_path):
"""
Fetches the dataset to a local repository for future use.
Arguments:
url (str): The external URI to a specific dataset
repo_path (str): The local path to write the fetched dataset to.
"""
repo_path = os.path.expandvars(os.path.expanduser(repo_path))
logger.info("fetching: %s, saving to: %s (this may take some time "
"depending on dataset size)", url, repo_path)
urllib.urlretrieve(url, os.path.join(repo_path,
os.path.basename(url)))
def get_inputs(self, backend=None, train=True, test=False,
validation=False):
"""
Loads and returns one or more input datasets.
Arguments:
backend (neon.backends.backend.Backend, optional): The underlying
data structure type used to hold this data once loaded.
If None will use whatever is set for this class
train (bool, optional): load a training target outcome dataset.
Defaults to True.
test (bool, optional): load a hold-out test target outcome dataset.
Defaults to False.
validation (bool, optional): Load a separate validation target
outcome dataset. Defaults to False.
Returns:
dict: of loaded datasets with keys train, test, validation
based on what was requested. Each dataset is a
neon.backends.backend.Tensor instance.
"""
res = dict()
if self.inputs['train'] is None:
if backend is not None:
self.load(backend)
else:
self.load()
if train and self.inputs['train'] is not None:
res['train'] = self.inputs['train']
if test and self.inputs['test'] is not None:
res['test'] = self.inputs['test']
if validation and self.inputs['validation'] is not None:
res['validation'] = self.inputs['validation']
return res
def get_targets(self, backend=None, train=True, test=False,
validation=False):
"""
Loads and returns one or more labelled outcome datasets.
Arguments:
backend (neon.backends.backend.Backend, None): The underlying
data structure type used to hold this data once loaded.
If None will use whatever is set for this class
train (bool, optional): load a training target outcome dataset.
Defaults to True.
test (bool, optional): load a hold-out test target outcome dataset.
Defaults to False.
validation (bool, optional): Load a separate validation target
outcome dataset. Defaults to False.
Returns:
dict: of loaded datasets with keys train, test, validation
based on what was requested. Each dataset is a
neon.backends.backend.Tensor instance.
"""
# can't have targets without inputs, ensure these are loaded
res = dict()
if self.inputs['train'] is None:
self.load()
if train and self.inputs['train'] is not None:
res['train'] = self.targets['train']
if test and self.inputs['test'] is not None:
res['test'] = self.targets['test']
if validation and self.inputs['validation'] is not None:
res['validation'] = self.targets['validation']
return res
def sample_training_data(self):
"""
Carries out actual downsampling of data, to the percentage specified in
self.sample_pct.
"""
if self.sample_pct != 100:
train_idcs = np.arange(self.inputs['train'].shape[0])
ntrain_actual = (self.inputs['train'].shape[0] *
int(self.sample_pct) / 100)
np.random.seed(self.backend.rng_seed)
np.random.shuffle(train_idcs)
train_idcs = train_idcs[0:ntrain_actual]
self.inputs['train'] = self.inputs['train'][train_idcs]
self.targets['train'] = self.targets['train'][train_idcs]
def transpose_batches(self, data):
"""
Transpose and distribute each minibatch within a dataset.
Arguments:
data (ndarray): Dataset to be sliced into mini batches,
transposed, and loaded to appropriate device
memory.
Returns:
list: List of device loaded mini-batches of data.
"""
bs = self.backend.actual_batch_size
if data.shape[0] % bs != 0:
logger.warning('Incompatible batch size. Discarding %d samples...',
data.shape[0] % bs)
nbatches = data.shape[0] / bs
batchwise = []
for batch in range(nbatches):
batchdata = np.empty((data.shape[1], bs))
batchdata[...] = data[batch * bs:(batch + 1) * bs].transpose()
dev_batchdata = self.backend.distribute(batchdata)
batchwise.append(dev_batchdata)
return batchwise
def format(self):
"""
Transforms the loaded data into the format expected by the
backend. If a hardware accelerator device is being used,
this function also copies the data to the device memory.
"""
assert self.backend is not None
for dataset in (self.inputs, self.targets):
for key in dataset:
item = dataset[key]
if item is not None:
dataset[key] = self.transpose_batches(item)
def get_batch(self, data, batch):
"""
Extract and return a single batch from the data specified.
Arguments:
data (list): List of device loaded batches of data
batch (int): 0-based index specifying the batch number to get
Returns:
neon.backends.Tensor: Single batch of data
See Also:
transpose_batches
"""
return data[batch]
def has_set(self, setname):
"""
Indicate whether the specified setname type is part of this dataset.
Arguments:
setname (str): The type of data to look for. Typically this is one
of 'train', 'test', 'validation'.
Returns:
bool: True if this dataset contains setname type of data, and False
otherwise.
"""
inputs_dic = self.get_inputs(train=True, validation=True,
test=True)
return True if (setname in inputs_dic) else False
def init_mini_batch_producer(self, batch_size, setname, predict):
"""
Setup the ability to generate mini-batches.
Arguments:
batch_size (int): The number of data examples will be contained in
each mini-batch
setname (str): The type of data to produce mini-batches for:
'train', 'test', 'validation'
predict (bool): Set this to False when training a model, or True
when generating batches to be used for prediction.
Returns:
int: The total number of examples to be mini-batched.
Notes:
This is the implementation for non-macro batched data.
macro-batched datasets will override this (e.g. ImageNet)
"""
self.cur_inputs = self.get_inputs(train=True, validation=True,
test=True)[setname]
self.cur_tgts = self.get_targets(train=True, validation=True,
test=True)[setname]
self.predict_mode = predict
return len(self.inputs[setname])
def get_mini_batch(self, batch_idx):
"""
Return the specified mini-batch of input and target data.
Arguments:
batch_idx (int): 0-based index specifying the mini-batch number to
retrieve.
Returns:
tuple: 2-tuple of neon.backend.Tensor objects containing the
corresponding input, and target mini-batches.
Notes:
This is the implementation for non-macro batched data.
macro-batched datasets will override this (e.g. ImageNet)
"""
return self.get_batch(self.cur_inputs, batch_idx), self.get_batch(
self.cur_tgts, batch_idx)
def del_mini_batch_producer(self):
"""
Perform any cleanup needed once all mini-batches have been produced.
Notes:
macro-batched datasets will likely override this
"""
pass
def test_argmax_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, ))
be.argmax(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([1, 0]))
def gen_backend(model=None,
gpu=None,
nrv=False,
flexpoint=False,
rng_seed=None,
numerr_handling=None,
half=False,
stochastic_round=0,
device_id=None):
"""
Construct and return a backend instance of the appropriate type based on
the arguments given. With no parameters, a single CPU core, float32
backend is returned.
Arguments:
model (neon.models.model.Model): The instantiated model upon which we
will utilize this backend.
gpu (string, optional): Attempt to utilize a CUDA capable GPU if
installed in the system. Defaults to None which
implies a CPU based backend. If 'cudanet',
utilize a cuda-convnet2 based backed, which
supports Kepler and Maxwell GPUs with single
precision. If 'nervanagpu', attempt to utilize
the NervanaGPU Maxwell backend with float16 and
float32 support.
nrv (bool, optional): If True, attempt to utilize the Nervana Engine
for computation (must be installed on the
system). Defaults to False which implies a CPU
based backend.
rng_seed (numeric, optional): Set this to a numeric value which can be
used to seed the random number generator
of the instantiated backend. Defaults to
None, which doesn't explicitly seed (so
each run will be different)
stochastic_round (numeric, optional): Only affects the max backend. If
1, perform stochastic rounding.
If 0, round to nearest.
numerr_handling (dict, optional): Dictate how numeric errors are
displayed and handled. The keys and
values permissible for this dict
match that seen in numpy.seterr.
If set to None (the default),
behavior is equivalent to
{'all': 'warn'}
device_id (numeric, optional): Set this to a numeric value which can be
used to select which device to run the
process on
Returns:
Backend: newly constructed backend instance of the specifed type.
Notes:
* Attempts to construct a GPU instance without a CUDA capable card or
without cudanet or nervanagpu package installed will cause the
program to display an error message and exit.
* Attempts to construct a parallel instance without mpi4py installed
will cause the program to display an error message and exit.
* The returned backend will still need to call its par.init_model()
at some point after the model has been linked, in order for parallel
training to proceed.
"""
logger = logging.getLogger(__name__)
gpuflag = False
if gpu is not None:
gpu = gpu.lower()
if sys.platform.startswith("linux"):
gpuflag = (os.system("nvcc --version > /dev/null 2>&1") == 0)
elif sys.platform.startswith("darwin"):
gpuflag = (
os.system("kextstat | grep -i cuda > /dev/null 2>&1") == 0)
if gpuflag and gpu == 'cudanet':
try:
import cudanet # noqa
from neon.backends.cc2 import GPU
be_name = 'Cudanet'
be = GPU(rng_seed=rng_seed, device_id=device_id)
except ImportError:
raise RuntimeError("cudanet not found, can't run via GPU")
elif gpuflag and gpu.startswith('nervanagpu'):
try:
import nervanagpu # noqa
try:
be_name = 'NervanaGPU'
if gpu == 'nervanagpu':
device_id = 0 if device_id is None else device_id[0]
from neon.backends.gpu import GPU
be = GPU(rng_seed=rng_seed,
stochastic_round=stochastic_round,
device_id=device_id)
else:
from neon.backends.mgpu import MGPU
try:
num_dev = int(gpu.strip('nervanagpu'))
except (ValueError):
raise ValueError("invalid number of GPUs" +
" specified")
if not device_id:
device_id = range(num_dev)
if len(device_id) != num_dev:
raise RuntimeError(
"Incorrect number of devices"
" specified ", device_id, num_dev)
be = MGPU(rng_seed=rng_seed,
stochastic_round=stochastic_round,
device_id=device_id,
num_dev=num_dev)
except ImportError:
logger.warning("pycuda error, can't run via GPU")
gpuflag = False
except ImportError:
logger.warning("nervanagpu not found, can't run via GPU")
gpuflag = False
if gpuflag is False:
raise RuntimeError("Can't find CUDA capable GPU")
elif nrv:
nrv = False
try:
from umd.nrv_backend import NRVBackend
nrv = True
except ImportError:
logger.warning("Nervana Engine system software not found")
if flexpoint:
logger.warning("Flexpoint(TM) backend not currently available")
if nrv:
be_name = 'NRV'
be = NRVBackend(rng_seed=rng_seed,
seterr_handling=numerr_handling,
device_id=device_id)
elif not gpuflag:
be_name = 'CPU'
be = CPU(rng_seed=rng_seed, seterr_handling=numerr_handling)
logger.info("{} backend, RNG seed: {}, numerr: {}".format(
be_name, rng_seed, numerr_handling))
return be
class TestValInit(object):
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
def test_uni_basics(self):
uni = UniformValGen(backend=self.be)
assert str(uni) == ("UniformValGen utilizing CPU backend\n\t"
"low: 0.0, high: 1.0")
def test_uni_gen(self):
uni = UniformValGen(backend=self.be)
res = uni.generate(shape=[1, 1])
assert res.shape == (1, 1)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
assert out.asnumpyarray() >= 0.0
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < 1.0
def test_uni_params(self):
low = -5.5
high = 10.2
uni = UniformValGen(backend=self.be, low=low, high=high)
assert str(uni) == ("UniformValGen utilizing CPU backend\n\t"
"low: {low}, high: {high}".format(low=low,
high=high))
res = uni.generate(shape=[4, 4])
assert res.shape == (4, 4)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
assert out.asnumpyarray() >= low
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < high
def test_autouni_gen(self):
autouni = AutoUniformValGen(backend=self.be, relu=True)
assert autouni.relu is True
assert str(autouni) == ("AutoUniformValGen utilizing CPU backend\n\t"
"low: nan, high: nan")
res = autouni.generate([3, 3])
assert res.shape == (3, 3)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
expected_val = math.sqrt(2) * (1.0 / math.sqrt(3))
assert out.asnumpyarray() >= -expected_val
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < expected_val
def test_gaussian_gen(self):
loc = 5
scale = 2.0
gauss = GaussianValGen(backend=self.be, loc=loc, scale=scale)
assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t"
"loc: {}, scale: {}".format(loc, scale))
res = gauss.generate([5, 10])
assert res.shape == (5, 10)
# TODO: test distribution of vals to ensure ~gaussian dist
def test_normal_gen(self):
loc = -2.5
scale = 3.0
gauss = NormalValGen(backend=self.be, loc=loc, scale=scale)
assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t"
"loc: {}, scale: {}".format(loc, scale))
res = gauss.generate([9, 3])
assert res.shape == (9, 3)
# TODO: test distribution of vals to ensure ~gaussian dist
def test_sparseeig_gen(self):
sparseness = 10
eigenvalue = 3.1
eig = SparseEigenValGen(backend=self.be,
sparseness=sparseness,
eigenvalue=eigenvalue)
assert str(eig) == ("SparseEigenValGen utilizing CPU backend\n\t"
"sparseness: {}, eigenvalue: "
"{}".format(sparseness, eigenvalue))
res = eig.generate([20, 20])
assert res.shape == (20, 20)
# TODO: test distribution of vals
def test_nodenorm_gen(self):
scale = 3.0
nodenorm = NodeNormalizedValGen(backend=self.be, scale=scale)
assert str(nodenorm) == ("NodeNormalizedValGen utilizing CPU backend"
"\n\tscale: {}".format(scale))
res = nodenorm.generate([8, 9])
assert res.shape == (8, 9)
out = self.be.empty((1, 1))
self.be.min(res, axes=None, out=out)
expected_val = scale * math.sqrt(6) / math.sqrt(8 + 9.)
assert out.asnumpyarray() >= -expected_val
self.be.max(res, axes=None, out=out)
assert out.asnumpyarray() < expected_val
def test_cpu_bprop(self):
backend = CPU(rng_seed=0)
layer = self.create_layer(backend=backend)
check_bprop(layer, backend)
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
class TestCPU(object):
def __init__(self):
# this code gets called prior to each test
self.be = CPU()
def test_empty_creation(self):
tns = self.be.empty((4, 3))
assert tns.shape == (4, 3)
def test_array_creation(self):
tns = self.be.array([[1, 2], [3, 4]])
assert tns.shape == (2, 2)
assert_tensor_equal(tns, CPUTensor([[1, 2], [3, 4]]))
def test_zeros_creation(self):
tns = self.be.zeros([3, 1])
assert tns.shape == (3, 1)
assert_tensor_equal(tns, CPUTensor([[0], [0], [0]]))
def test_ones_creation(self):
tns = self.be.ones([1, 4])
assert tns.shape == (1, 4)
assert_tensor_equal(tns, CPUTensor([[1, 1, 1, 1]]))
def test_all_equal(self):
left = self.be.ones([2, 2])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]]))
def test_some_equal(self):
left = self.be.ones([2, 2])
right = self.be.array([[0, 1], [0, 1]])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 1], [0, 1]]))
def test_none_equal(self):
left = self.be.ones([2, 2])
right = self.be.zeros([2, 2])
out = self.be.empty([2, 2])
self.be.equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]]))
def test_all_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.zeros([2, 2])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]]))
def test_some_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.array([[0, 1], [0, 1]])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 0], [1, 0]]))
def test_none_not_equal(self):
left = self.be.ones([2, 2])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.not_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]]))
def test_greater(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.greater(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [0, 1]]))
def test_greater_equal(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.greater_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[0, 0], [1, 1]]))
def test_less(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.less(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [0, 0]]))
def test_less_equal(self):
left = self.be.array([[-1, 0], [1, 92]])
right = self.be.ones([2, 2])
out = self.be.empty([2, 2])
self.be.less_equal(left, right, out)
assert out.shape == (2, 2)
assert_tensor_equal(out, CPUTensor([[1, 1], [1, 0]]))
def test_argmin_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmin(tsr, None, out)
assert_tensor_equal(out, CPUTensor([[0]]))
def test_argmin_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((1, 2))
be.argmin(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([[0, 0]]))
def test_argmin_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, 1))
be.argmin(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([0, 1]))
def test_argmax_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmax(tsr, None, out)
assert_tensor_equal(out, CPUTensor(3))
def test_argmax_axis0(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty((2, ))
be.argmax(tsr, 0, out)
assert_tensor_equal(out, CPUTensor([1, 1]))
def test_argmax_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, ))
be.argmax(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([1, 0]))
def test_2norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
rpow = 1. / 2
# -> sum([[1, 0], [1, 9]], axis=0)**.5 -> sqrt([2, 9])
assert_tensor_equal(self.be.norm(tsr, order=2, axis=0),
CPUTensor([[2**rpow, 9**rpow]]))
# -> sum([[1, 0], [1, 9]], axis=1)**.5 -> sqrt([1, 10])
assert_tensor_equal(self.be.norm(tsr, order=2, axis=1),
CPUTensor([1**rpow, 10**rpow]))
def test_1norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> sum([[1, 0], [1, 3]], axis=0)**1 -> [2, 3]
assert_tensor_equal(self.be.norm(tsr, order=1, axis=0),
CPUTensor([[2, 3]]))
# -> sum([[1, 0], [1, 3]], axis=1)**1 -> [1, 4]
assert_tensor_equal(self.be.norm(tsr, order=1, axis=1),
CPUTensor([1, 4]))
def test_0norm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> sum(tsr != 0, axis=0) -> [2, 1]
assert_tensor_equal(self.be.norm(tsr, order=0, axis=0),
CPUTensor([[2, 1]]))
# -> sum(tsr != 0, axis=1) -> [1, 2]
assert_tensor_equal(self.be.norm(tsr, order=0, axis=1),
CPUTensor([1, 2]))
def test_infnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> max(abs(tsr), axis=0) -> [1, 3]
assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=0),
CPUTensor([[1, 3]]))
# -> max(abs(tsr), axis=1) -> [1, 3]
assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=1),
CPUTensor([1, 3]))
def test_neginfnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
# -> min(abs(tsr), axis=0) -> [1, 0]
assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=0),
CPUTensor([[1, 0]]))
# -> min(abs(tsr), axis=1) -> [0, 1]
assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=1),
CPUTensor([0, 1]))
def test_lrgnorm(self):
tsr = self.be.array([[-1, 0], [1, 3]])
rpow = 1. / 5
# -> sum([[1, 0], [1, 243]], axis=0)**rpow -> rpow([2, 243])
assert_tensor_equal(self.be.norm(tsr, order=5, axis=0),
CPUTensor([[2**rpow, 243**rpow]]))
# -> sum([[1, 0], [1, 243]], axis=1)**rpow -> rpow([1, 244])
# 244**.2 == ~3.002465 hence the near_equal test
assert_tensor_near_equal(self.be.norm(tsr, order=5, axis=1),
CPUTensor([1**rpow, 244**rpow]), 1e-6)
def test_negnorm(self):
tsr = self.be.array([[-1, -2], [1, 3]])
rpow = -1. / 3
# -> sum([[1, .125], [1, .037037]], axis=0)**rpow -> rpow([2, .162037])
assert_tensor_equal(self.be.norm(tsr, order=-3, axis=0),
CPUTensor([[2**rpow, .162037037037**rpow]]))
# -> sum([[1, .125], [1, .037037]], axis=1)**rpow ->
# rpow([1.125, 1.037037])
assert_tensor_near_equal(self.be.norm(tsr, order=-3, axis=1),
CPUTensor([1.125**rpow, 1.037037**rpow]),
1e-6)
def test_argmax_noaxis(self):
be = CPU()
tsr = be.array([[-1, 0], [1, 92]])
out = be.empty([1, 1])
be.argmax(tsr, None, out)
assert_tensor_equal(out, CPUTensor(3))
def test_argmax_axis1(self):
be = CPU()
tsr = be.array([[-1, 10], [11, 9]])
out = be.empty((2, ))
be.argmax(tsr, 1, out)
assert_tensor_equal(out, CPUTensor([1, 0]))
