def main():
h5file = '/root/data/pcallier/amazon/temp.hd5'
amzn_path = '/root/data/pcallier/amazon/reviews_Health_and_Personal_Care.json.gz'
#azbw = AmazonBatchWriter(amzn_path, h5file)
#azbw.run()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = DiskDataIterator(lambda: batcher(load_data('/root/data/amazon/test_amazon.json.gz')), 3000, 128, nvocab=67)
# random examples from each
for bidx, (X_batch, y_batch) in enumerate(train_set):
print "Batch {}:".format(bidx)
#print X_batch.get().T.sum(axis=1)
reviewnum = input("Pick review index to fetch and decode: ")
review = from_one_hot(X_batch.get().T[reviewnum].reshape(67, -1))
print ''.join(review)[::-1]
def main():
h5file = '/root/data/pcallier/amazon/temp.hd5'
amzn_path = '/root/data/pcallier/amazon/reviews_Health_and_Personal_Care.json.gz'
#azbw = AmazonBatchWriter(amzn_path, h5file)
#azbw.run()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = DiskDataIterator(
lambda: batcher(load_data('/root/data/amazon/test_amazon.json.gz')),
3000,
128,
nvocab=67)
# random examples from each
for bidx, (X_batch, y_batch) in enumerate(train_set):
print "Batch {}:".format(bidx)
#print X_batch.get().T.sum(axis=1)
reviewnum = input("Pick review index to fetch and decode: ")
review = from_one_hot(X_batch.get().T[reviewnum].reshape(67, -1))
print ''.join(review)[::-1]
yield (inputs, targets)
class DataIterator(ArrayIterator):
"""
This class has been renamed to ArrayIterator and deprecated.
This is just a place holder until the class is removed. Please
use the ArrayIterator class.
"""
def __init__(self, *args, **kwargs):
logger.error('DataIterator class has been deprecated and renamed'
'"ArrayIterator" please use that name.')
super(DataIterator, self).__init__(*args, **kwargs)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test) = load_mnist()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = ArrayIterator(
[X_test[:1000], X_test[:1000]], y_test[:1000], nclass=10)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
print bidx, train_set.start
pass
axis=0)
if self.be.bsz > bsz:
self.ybuf[:, bsz:] = self.be.onehot(
self.ydev[:(self.be.bsz - bsz)], axis=0)
else:
self.ybuf[:, :bsz] = self.ydev[i1:i2].T
if self.be.bsz > bsz:
self.ybuf[:, bsz:] = self.ydev[:(self.be.bsz - bsz)].T
inputs = self.Xbuf[0] if len(self.Xbuf) == 1 else self.Xbuf
targets = self.ybuf if self.ybuf else inputs
yield (inputs, targets)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test) = load_mnist()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = DataIterator([X_test[:1000], X_test[:1000]],
y_test[:1000],
nclass=10)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
print bidx, train_set.start
pass
