def test_rectleaky_slope_zero_rectlin_equiv(): be = CPU() inputs = be.uniform(low=-5.0, high=10.0, size=(10, 10)) lin_buf = be.empty(inputs.shape) leaky_buf = be.empty(inputs.shape) be.rectlin(inputs, out=lin_buf) be.rectleaky(inputs, slope=0.0, out=leaky_buf) assert_tensor_equal(lin_buf, leaky_buf)
def test_xcov_cputensor(): np.random.seed(0) n = 10 k = 8 (k1, k2) = (3, 5) a = np.array(np.random.randn(k, n)*10, dtype='float32', order='C') acc = xcc(a[:k1], a[k1:]) expected_result = 0.5 * (acc**2.).sum() be = CPU(rng_seed=0) outputs = CPUTensor(a.copy()) tempbuf1 = be.empty((k1, n)) tempbuf2 = be.empty((k2, n)) tempbuf3 = be.empty((k1, k2)) tempbuf4 = be.empty(outputs.shape) temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4] my_result = xcov_cost(be, outputs, [], temp, k1) assert_tensor_near_equal(expected_result, my_result)
def test_xcov_cputensor(): np.random.seed(0) n = 10 k = 8 (k1, k2) = (3, 5) a = np.array(np.random.randn(k, n) * 10, dtype='float32', order='C') acc = xcc(a[:k1], a[k1:]) expected_result = 0.5 * (acc**2.).sum() be = CPU(rng_seed=0) outputs = CPUTensor(a.copy()) tempbuf1 = be.empty((k1, n)) tempbuf2 = be.empty((k2, n)) tempbuf3 = be.empty((k1, k2)) tempbuf4 = be.empty(outputs.shape) temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4] my_result = xcov_cost(be, outputs, [], temp, k1) assert_tensor_near_equal(expected_result, my_result)
def test_xcov_derivative_cputensor(): np.random.seed(0) n = 10 k = 8 (k1, k2) = (3, 5) a = np.array(np.random.randn(k, n), dtype='float32', order='C') s = np.zeros_like(a) acc = xcc(a[:k1], a[k1:]) # k1 x k2 c1 = a[k1:] - a[k1:].mean(1, keepdims=True) # k2 x n c2 = a[:k1] - a[:k1].mean(1, keepdims=True) # k1 x n s[:k1] = acc.dot(c1) / n s[k1:] = acc.T.dot(c2) / n be = CPU(rng_seed=0) outputs = CPUTensor(a.copy()) tempbuf1 = be.empty((k1, n)) tempbuf2 = be.empty((k2, n)) tempbuf3 = be.empty((k1, k2)) tempbuf4 = be.empty(outputs.shape) temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4] my_result = xcov_cost_derivative(be, outputs, [], temp, k1) expected_result = CPUTensor(s) assert_tensor_near_equal(expected_result, my_result)
def test_xcov_derivative_cputensor(): np.random.seed(0) n = 10 k = 8 (k1, k2) = (3, 5) a = np.array(np.random.randn(k, n), dtype='float32', order='C') s = np.zeros_like(a) acc = xcc(a[:k1], a[k1:]) # k1 x k2 c1 = a[k1:] - a[k1:].mean(1, keepdims=True) # k2 x n c2 = a[:k1] - a[:k1].mean(1, keepdims=True) # k1 x n s[:k1] = acc.dot(c1)/n s[k1:] = acc.T.dot(c2)/n be = CPU(rng_seed=0) outputs = CPUTensor(a.copy()) tempbuf1 = be.empty((k1, n)) tempbuf2 = be.empty((k2, n)) tempbuf3 = be.empty((k1, k2)) tempbuf4 = be.empty(outputs.shape) temp = [tempbuf1, tempbuf2, tempbuf3, tempbuf4] my_result = xcov_cost_derivative(be, outputs, [], temp, k1) expected_result = CPUTensor(s) assert_tensor_near_equal(expected_result, my_result)
class TestValInit(object): def __init__(self): # this code gets called prior to each test self.be = CPU() def test_uni_basics(self): uni = UniformValGen(backend=self.be) assert str(uni) == ("UniformValGen utilizing CPU backend\n\t" "low: 0.0, high: 1.0") def test_uni_gen(self): uni = UniformValGen(backend=self.be) res = uni.generate(shape=[1, 1]) assert res.shape == (1, 1) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) assert out.asnumpyarray() >= 0.0 self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < 1.0 def test_uni_params(self): low = -5.5 high = 10.2 uni = UniformValGen(backend=self.be, low=low, high=high) assert str(uni) == ("UniformValGen utilizing CPU backend\n\t" "low: {low}, high: {high}".format(low=low, high=high)) res = uni.generate(shape=[4, 4]) assert res.shape == (4, 4) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) assert out.asnumpyarray() >= low self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < high def test_autouni_gen(self): autouni = AutoUniformValGen(backend=self.be, relu=True) assert autouni.relu is True assert str(autouni) == ("AutoUniformValGen utilizing CPU backend\n\t" "low: nan, high: nan") res = autouni.generate([3, 3]) assert res.shape == (3, 3) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) expected_val = math.sqrt(2) * (1.0 / math.sqrt(3)) assert out.asnumpyarray() >= - expected_val self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < expected_val def test_gaussian_gen(self): loc = 5 scale = 2.0 gauss = GaussianValGen(backend=self.be, loc=loc, scale=scale) assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t" "loc: {}, scale: {}".format(loc, scale)) res = gauss.generate([5, 10]) assert res.shape == (5, 10) # TODO: test distribution of vals to ensure ~gaussian dist def test_normal_gen(self): loc = -2.5 scale = 3.0 gauss = NormalValGen(backend=self.be, loc=loc, scale=scale) assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t" "loc: {}, scale: {}".format(loc, scale)) res = gauss.generate([9, 3]) assert res.shape == (9, 3) # TODO: test distribution of vals to ensure ~gaussian dist def test_sparseeig_gen(self): sparseness = 10 eigenvalue = 3.1 eig = SparseEigenValGen(backend=self.be, sparseness=sparseness, eigenvalue=eigenvalue) assert str(eig) == ("SparseEigenValGen utilizing CPU backend\n\t" "sparseness: {}, eigenvalue: " "{}".format(sparseness, eigenvalue)) res = eig.generate([20, 20]) assert res.shape == (20, 20) # TODO: test distribution of vals def test_nodenorm_gen(self): scale = 3.0 nodenorm = NodeNormalizedValGen(backend=self.be, scale=scale) assert str(nodenorm) == ("NodeNormalizedValGen utilizing CPU backend" "\n\tscale: {}".format(scale)) res = nodenorm.generate([8, 9]) assert res.shape == (8, 9) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) expected_val = scale * math.sqrt(6) / math.sqrt(8 + 9.) assert out.asnumpyarray() >= - expected_val self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < expected_val
class TestCPU(object): def __init__(self): # this code gets called prior to each test self.be = CPU() def test_empty_creation(self): tns = self.be.empty((4, 3)) assert tns.shape == (4, 3) def test_array_creation(self): tns = self.be.array([[1, 2], [3, 4]]) assert tns.shape == (2, 2) assert_tensor_equal(tns, CPUTensor([[1, 2], [3, 4]])) def test_zeros_creation(self): tns = self.be.zeros([3, 1]) assert tns.shape == (3, 1) assert_tensor_equal(tns, CPUTensor([[0], [0], [0]])) def test_ones_creation(self): tns = self.be.ones([1, 4]) assert tns.shape == (1, 4) assert_tensor_equal(tns, CPUTensor([[1, 1, 1, 1]])) def test_all_equal(self): left = self.be.ones([2, 2]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]])) def test_some_equal(self): left = self.be.ones([2, 2]) right = self.be.array([[0, 1], [0, 1]]) out = self.be.empty([2, 2]) self.be.equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[0, 1], [0, 1]])) def test_none_equal(self): left = self.be.ones([2, 2]) right = self.be.zeros([2, 2]) out = self.be.empty([2, 2]) self.be.equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]])) def test_all_not_equal(self): left = self.be.ones([2, 2]) right = self.be.zeros([2, 2]) out = self.be.empty([2, 2]) self.be.not_equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[1, 1], [1, 1]])) def test_some_not_equal(self): left = self.be.ones([2, 2]) right = self.be.array([[0, 1], [0, 1]]) out = self.be.empty([2, 2]) self.be.not_equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[1, 0], [1, 0]])) def test_none_not_equal(self): left = self.be.ones([2, 2]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.not_equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[0, 0], [0, 0]])) def test_greater(self): left = self.be.array([[-1, 0], [1, 92]]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.greater(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[0, 0], [0, 1]])) def test_greater_equal(self): left = self.be.array([[-1, 0], [1, 92]]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.greater_equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[0, 0], [1, 1]])) def test_less(self): left = self.be.array([[-1, 0], [1, 92]]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.less(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[1, 1], [0, 0]])) def test_less_equal(self): left = self.be.array([[-1, 0], [1, 92]]) right = self.be.ones([2, 2]) out = self.be.empty([2, 2]) self.be.less_equal(left, right, out) assert out.shape == (2, 2) assert_tensor_equal(out, CPUTensor([[1, 1], [1, 0]])) def test_argmin_noaxis(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty([1, 1]) be.argmin(tsr, None, out) assert_tensor_equal(out, CPUTensor([[0]])) def test_argmin_axis0(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty((1, 2)) be.argmin(tsr, 0, out) assert_tensor_equal(out, CPUTensor([[0, 0]])) def test_argmin_axis1(self): be = CPU() tsr = be.array([[-1, 10], [11, 9]]) out = be.empty((2, 1)) be.argmin(tsr, 1, out) assert_tensor_equal(out, CPUTensor([0, 1])) def test_argmax_noaxis(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty([1, 1]) be.argmax(tsr, None, out) assert_tensor_equal(out, CPUTensor(3)) def test_argmax_axis0(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty((2, )) be.argmax(tsr, 0, out) assert_tensor_equal(out, CPUTensor([1, 1])) def test_argmax_axis1(self): be = CPU() tsr = be.array([[-1, 10], [11, 9]]) out = be.empty((2, )) be.argmax(tsr, 1, out) assert_tensor_equal(out, CPUTensor([1, 0])) def test_2norm(self): tsr = self.be.array([[-1, 0], [1, 3]]) rpow = 1. / 2 # -> sum([[1, 0], [1, 9]], axis=0)**.5 -> sqrt([2, 9]) assert_tensor_equal(self.be.norm(tsr, order=2, axis=0), CPUTensor([[2**rpow, 9**rpow]])) # -> sum([[1, 0], [1, 9]], axis=1)**.5 -> sqrt([1, 10]) assert_tensor_equal(self.be.norm(tsr, order=2, axis=1), CPUTensor([1**rpow, 10**rpow])) def test_1norm(self): tsr = self.be.array([[-1, 0], [1, 3]]) # -> sum([[1, 0], [1, 3]], axis=0)**1 -> [2, 3] assert_tensor_equal(self.be.norm(tsr, order=1, axis=0), CPUTensor([[2, 3]])) # -> sum([[1, 0], [1, 3]], axis=1)**1 -> [1, 4] assert_tensor_equal(self.be.norm(tsr, order=1, axis=1), CPUTensor([1, 4])) def test_0norm(self): tsr = self.be.array([[-1, 0], [1, 3]]) # -> sum(tsr != 0, axis=0) -> [2, 1] assert_tensor_equal(self.be.norm(tsr, order=0, axis=0), CPUTensor([[2, 1]])) # -> sum(tsr != 0, axis=1) -> [1, 2] assert_tensor_equal(self.be.norm(tsr, order=0, axis=1), CPUTensor([1, 2])) def test_infnorm(self): tsr = self.be.array([[-1, 0], [1, 3]]) # -> max(abs(tsr), axis=0) -> [1, 3] assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=0), CPUTensor([[1, 3]])) # -> max(abs(tsr), axis=1) -> [1, 3] assert_tensor_equal(self.be.norm(tsr, order=float('inf'), axis=1), CPUTensor([1, 3])) def test_neginfnorm(self): tsr = self.be.array([[-1, 0], [1, 3]]) # -> min(abs(tsr), axis=0) -> [1, 0] assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=0), CPUTensor([[1, 0]])) # -> min(abs(tsr), axis=1) -> [0, 1] assert_tensor_equal(self.be.norm(tsr, order=float('-inf'), axis=1), CPUTensor([0, 1])) def test_lrgnorm(self): tsr = self.be.array([[-1, 0], [1, 3]]) rpow = 1. / 5 # -> sum([[1, 0], [1, 243]], axis=0)**rpow -> rpow([2, 243]) assert_tensor_equal(self.be.norm(tsr, order=5, axis=0), CPUTensor([[2**rpow, 243**rpow]])) # -> sum([[1, 0], [1, 243]], axis=1)**rpow -> rpow([1, 244]) # 244**.2 == ~3.002465 hence the near_equal test assert_tensor_near_equal(self.be.norm(tsr, order=5, axis=1), CPUTensor([1**rpow, 244**rpow]), 1e-6) def test_negnorm(self): tsr = self.be.array([[-1, -2], [1, 3]]) rpow = -1. / 3 # -> sum([[1, .125], [1, .037037]], axis=0)**rpow -> rpow([2, .162037]) assert_tensor_equal(self.be.norm(tsr, order=-3, axis=0), CPUTensor([[2**rpow, .162037037037**rpow]])) # -> sum([[1, .125], [1, .037037]], axis=1)**rpow -> # rpow([1.125, 1.037037]) assert_tensor_near_equal(self.be.norm(tsr, order=-3, axis=1), CPUTensor([1.125**rpow, 1.037037**rpow]), 1e-6)
def test_argmax_axis1(self): be = CPU() tsr = be.array([[-1, 10], [11, 9]]) out = be.empty((2, )) be.argmax(tsr, 1, out) assert_tensor_equal(out, CPUTensor([1, 0]))
def test_argmax_noaxis(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty([1, 1]) be.argmax(tsr, None, out) assert_tensor_equal(out, CPUTensor(3))
def test_argmin_axis0(self): be = CPU() tsr = be.array([[-1, 0], [1, 92]]) out = be.empty((1, 2)) be.argmin(tsr, 0, out) assert_tensor_equal(out, CPUTensor([[0, 0]]))
class TestValInit(object): def __init__(self): # this code gets called prior to each test self.be = CPU() def test_uni_basics(self): uni = UniformValGen(backend=self.be) assert str(uni) == ("UniformValGen utilizing CPU backend\n\t" "low: 0.0, high: 1.0") def test_uni_gen(self): uni = UniformValGen(backend=self.be) res = uni.generate(shape=[1, 1]) assert res.shape == (1, 1) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) assert out.asnumpyarray() >= 0.0 self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < 1.0 def test_uni_params(self): low = -5.5 high = 10.2 uni = UniformValGen(backend=self.be, low=low, high=high) assert str(uni) == ("UniformValGen utilizing CPU backend\n\t" "low: {low}, high: {high}".format(low=low, high=high)) res = uni.generate(shape=[4, 4]) assert res.shape == (4, 4) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) assert out.asnumpyarray() >= low self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < high def test_autouni_gen(self): autouni = AutoUniformValGen(backend=self.be, relu=True) assert autouni.relu is True assert str(autouni) == ("AutoUniformValGen utilizing CPU backend\n\t" "low: nan, high: nan") res = autouni.generate([3, 3]) assert res.shape == (3, 3) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) expected_val = math.sqrt(2) * (1.0 / math.sqrt(3)) assert out.asnumpyarray() >= -expected_val self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < expected_val def test_gaussian_gen(self): loc = 5 scale = 2.0 gauss = GaussianValGen(backend=self.be, loc=loc, scale=scale) assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t" "loc: {}, scale: {}".format(loc, scale)) res = gauss.generate([5, 10]) assert res.shape == (5, 10) # TODO: test distribution of vals to ensure ~gaussian dist def test_normal_gen(self): loc = -2.5 scale = 3.0 gauss = NormalValGen(backend=self.be, loc=loc, scale=scale) assert str(gauss) == ("GaussianValGen utilizing CPU backend\n\t" "loc: {}, scale: {}".format(loc, scale)) res = gauss.generate([9, 3]) assert res.shape == (9, 3) # TODO: test distribution of vals to ensure ~gaussian dist def test_sparseeig_gen(self): sparseness = 10 eigenvalue = 3.1 eig = SparseEigenValGen(backend=self.be, sparseness=sparseness, eigenvalue=eigenvalue) assert str(eig) == ("SparseEigenValGen utilizing CPU backend\n\t" "sparseness: {}, eigenvalue: " "{}".format(sparseness, eigenvalue)) res = eig.generate([20, 20]) assert res.shape == (20, 20) # TODO: test distribution of vals def test_nodenorm_gen(self): scale = 3.0 nodenorm = NodeNormalizedValGen(backend=self.be, scale=scale) assert str(nodenorm) == ("NodeNormalizedValGen utilizing CPU backend" "\n\tscale: {}".format(scale)) res = nodenorm.generate([8, 9]) assert res.shape == (8, 9) out = self.be.empty((1, 1)) self.be.min(res, axes=None, out=out) expected_val = scale * math.sqrt(6) / math.sqrt(8 + 9.) assert out.asnumpyarray() >= -expected_val self.be.max(res, axes=None, out=out) assert out.asnumpyarray() < expected_val
class Dataset(object): """ Base dataset class. Defines interface operations. """ backend = None inputs = {'train': None, 'test': None, 'validation': None} targets = {'train': None, 'test': None, 'validation': None} def __getstate__(self): """ Defines what and how we go about serializing an instance of this class. In this case we also want to include any loaded datasets and backend references. Returns: dict: keyword args, plus current inputs, targets, backend """ self.__dict__['backend'] = self.backend self.__dict__['inputs'] = self.inputs self.__dict__['targets'] = self.targets return self.__dict__ def __setstate__(self, state): """ Defines how we go about deserializing and loading an instance of this class from a specified state. Arguments: state (dict): attribute values to be loaded. """ self.__dict__.update(state) if self.backend is None: # use CPU as a default backend self.backend = CPU() def set_distributed_batch_size(self, model): self.batch_size = model.batch_size self.fragment_data = False if self.backend.is_dist: if model.layers[1].is_local and self.backend.num_dev > 1: self.fragment_data = True def load(self, backend=None, experiment=None): """ Makes the dataset data available for use. Needs to be implemented in every concrete Dataset child class. Arguments: backend (neon.backends.backend.Backend, optional): The underlying data structure type used to hold this data once loaded. If None will use `neon.backends.cpu.CPU` experiment (neon.experiments.experiment.Experiment, optional): The object that loads this dataset. Raises: NotImplementedError: should be overridden in child class """ raise NotImplementedError() def unload(self): """ Perform cleanup tasks if any are required. """ pass def process_result(self, result): """ Accept and process results of running inference. Arguments: result (ndarray): Array containing predictions obtained by processing a minibatch of input data. """ pass def download_to_repo(self, url, repo_path): """ Fetches the dataset to a local repository for future use. Arguments: url (str): The external URI to a specific dataset repo_path (str): The local path to write the fetched dataset to. """ repo_path = os.path.expandvars(os.path.expanduser(repo_path)) logger.info("fetching: %s, saving to: %s (this may take some time " "depending on dataset size)", url, repo_path) urllib.urlretrieve(url, os.path.join(repo_path, os.path.basename(url))) def get_inputs(self, backend=None, train=True, test=False, validation=False): """ Loads and returns one or more input datasets. Arguments: backend (neon.backends.backend.Backend, optional): The underlying data structure type used to hold this data once loaded. If None will use whatever is set for this class train (bool, optional): load a training target outcome dataset. Defaults to True. test (bool, optional): load a hold-out test target outcome dataset. Defaults to False. validation (bool, optional): Load a separate validation target outcome dataset. Defaults to False. Returns: dict: of loaded datasets with keys train, test, validation based on what was requested. Each dataset is a neon.backends.backend.Tensor instance. """ res = dict() if self.inputs['train'] is None: if backend is not None: self.load(backend) else: self.load() if train and self.inputs['train'] is not None: res['train'] = self.inputs['train'] if test and self.inputs['test'] is not None: res['test'] = self.inputs['test'] if validation and self.inputs['validation'] is not None: res['validation'] = self.inputs['validation'] return res def get_targets(self, backend=None, train=True, test=False, validation=False): """ Loads and returns one or more labelled outcome datasets. Arguments: backend (neon.backends.backend.Backend, None): The underlying data structure type used to hold this data once loaded. If None will use whatever is set for this class train (bool, optional): load a training target outcome dataset. Defaults to True. test (bool, optional): load a hold-out test target outcome dataset. Defaults to False. validation (bool, optional): Load a separate validation target outcome dataset. Defaults to False. Returns: dict: of loaded datasets with keys train, test, validation based on what was requested. Each dataset is a neon.backends.backend.Tensor instance. """ # can't have targets without inputs, ensure these are loaded res = dict() if self.inputs['train'] is None: self.load() if train and self.inputs['train'] is not None: res['train'] = self.targets['train'] if test and self.inputs['test'] is not None: res['test'] = self.targets['test'] if validation and self.inputs['validation'] is not None: res['validation'] = self.targets['validation'] return res def sample_training_data(self): """ Carries out actual downsampling of data, to the percentage specified in self.sample_pct. """ if self.sample_pct != 100: train_idcs = np.arange(self.inputs['train'].shape[0]) ntrain_actual = (self.inputs['train'].shape[0] * int(self.sample_pct) / 100) np.random.seed(self.backend.rng_seed) np.random.shuffle(train_idcs) train_idcs = train_idcs[0:ntrain_actual] self.inputs['train'] = self.inputs['train'][train_idcs] self.targets['train'] = self.targets['train'][train_idcs] def transpose_batches(self, data, dtype, is_target=False): """ Transpose and distribute each minibatch within a dataset. Arguments: data (ndarray): Dataset to be sliced into mini batches, transposed, and loaded to appropriate device memory. Returns: list: List of device loaded mini-batches of data. """ bs = self.backend.batch_size sba = self.backend.array if data.shape[0] % bs != 0: logger.warning('Incompatible batch size. Discarding %d samples...', data.shape[0] % bs) nbatches = data.shape[0] // bs batchwise = [] if not self.backend.is_dist: batchwise = [sba(data[idx * bs:(idx + 1) * bs].transpose().copy()) for idx in range(nbatches)] else: batchwise = [] if is_target or not self.fragment_data: devshape = (bs, data.shape[1]) ptype = 'replica' else: devshape = (bs / self.backend.num_dev, data.shape[1]) ptype = 'fragment' dev_batchdata_t = self.backend.empty(devshape) dev_batchdata_t.ptype = ptype for batch in range(nbatches): self.backend.set(dev_batchdata_t, data[batch * bs:(batch + 1) * bs]) dev_batchdata = self.backend.empty(dev_batchdata_t.shape[::-1]) dev_batchdata[:] = dev_batchdata_t.T dev_batchdata.ptype = dev_batchdata_t.ptype batchwise.append(dev_batchdata) return batchwise def format(self, dtype=np.float32): """ Transforms the loaded data into the format expected by the backend. If a hardware accelerator device is being used, this function also copies the data to the device memory. """ assert self.backend is not None for dsname in ('inputs', 'targets'): dataset = getattr(self, dsname) for key in dataset: item = dataset[key] if item is not None: dataset[key] = self.transpose_batches(item, dtype, dsname == 'targets') def get_batch(self, data, batch): """ Extract and return a single batch from the data specified. Arguments: data (list): List of device loaded batches of data batch (int): 0-based index specifying the batch number to get Returns: neon.backends.Tensor: Single batch of data See Also: transpose_batches """ return data[batch] def has_set(self, setname): """ Indicate whether the specified setname type is part of this dataset. Arguments: setname (str): The type of data to look for. Typically this is one of 'train', 'test', 'validation'. Returns: bool: True if this dataset contains setname type of data, and False otherwise. """ inputs_dic = self.get_inputs(train=True, validation=True, test=True) return True if (setname in inputs_dic) else False def split_set(self, pct, from_set='train', to_set='validation'): """ Splits the specified percentage amount of from_set and places it into to_set. Any existing data in to_set will be dropped. Arguments: pct (float): The percentage of data to transfer, in [0, 100]. from_set (str): Where the data will be transfered from. to_set (str): The set to be created with the transfered data. """ if pct != 0: from_idcs = np.arange(self.inputs[from_set].shape[0]) nto_actual = (self.inputs[from_set].shape[0] * int(pct) / 100) np.random.seed(self.backend.rng_seed) np.random.shuffle(from_idcs) to_idcs = from_idcs[0:nto_actual] from_idcs = from_idcs[nto_actual:] self.inputs[to_set] = self.inputs[from_set][to_idcs] self.targets[to_set] = self.targets[from_set][to_idcs] self.inputs[from_set] = self.inputs[from_set][from_idcs] self.targets[from_set] = self.targets[from_set][from_idcs] def init_mini_batch_producer(self, batch_size, setname, predict): """ Setup the ability to generate mini-batches. Arguments: batch_size (int): The number of data examples will be contained in each mini-batch setname (str): The type of data to produce mini-batches for: 'train', 'test', 'validation' predict (bool): Set this to False when training a model, or True when generating batches to be used for prediction. Returns: int: The total number of examples to be mini-batched. Notes: This is the implementation for non-macro batched data. macro-batched datasets will override this (e.g. ImageNet) """ self.cur_inputs = self.get_inputs(train=True, validation=True, test=True)[setname] self.cur_tgts = self.get_targets(train=True, validation=True, test=True)[setname] self.predict_mode = predict return len(self.inputs[setname]) def get_mini_batch(self, batch_idx): """ Return the specified mini-batch of input and target data. Arguments: batch_idx (int): 0-based index specifying the mini-batch number to retrieve. Returns: tuple: 2-tuple of neon.backend.Tensor objects containing the corresponding input, and target mini-batches. Notes: This is the implementation for non-macro batched data. macro-batched datasets will override this (e.g. ImageNet) """ return self.get_batch(self.cur_inputs, batch_idx), self.get_batch( self.cur_tgts, batch_idx) def del_mini_batch_producer(self): """ Perform any cleanup needed once all mini-batches have been produced. Notes: macro-batched datasets will likely override this """ pass