def main(): h5file = '/root/data/pcallier/amazon/temp.hd5' amzn_path = '/root/data/pcallier/amazon/reviews_Health_and_Personal_Care.json.gz' #azbw = AmazonBatchWriter(amzn_path, h5file) #azbw.run() from neon.backends.nervanagpu import NervanaGPU ng = NervanaGPU(0, device_id=1) NervanaObject.be = ng ng.bsz = 128 train_set = DiskDataIterator(lambda: batcher(load_data('/root/data/amazon/test_amazon.json.gz')), 3000, 128, nvocab=67) # random examples from each for bidx, (X_batch, y_batch) in enumerate(train_set): print "Batch {}:".format(bidx) #print X_batch.get().T.sum(axis=1) reviewnum = input("Pick review index to fetch and decode: ") review = from_one_hot(X_batch.get().T[reviewnum].reshape(67, -1)) print ''.join(review)[::-1]
def main(): h5file = '/root/data/pcallier/amazon/temp.hd5' amzn_path = '/root/data/pcallier/amazon/reviews_Health_and_Personal_Care.json.gz' #azbw = AmazonBatchWriter(amzn_path, h5file) #azbw.run() from neon.backends.nervanagpu import NervanaGPU ng = NervanaGPU(0, device_id=1) NervanaObject.be = ng ng.bsz = 128 train_set = DiskDataIterator( lambda: batcher(load_data('/root/data/amazon/test_amazon.json.gz')), 3000, 128, nvocab=67) # random examples from each for bidx, (X_batch, y_batch) in enumerate(train_set): print "Batch {}:".format(bidx) #print X_batch.get().T.sum(axis=1) reviewnum = input("Pick review index to fetch and decode: ") review = from_one_hot(X_batch.get().T[reviewnum].reshape(67, -1)) print ''.join(review)[::-1]
yield (inputs, targets) class DataIterator(ArrayIterator): """ This class has been renamed to ArrayIterator and deprecated. This is just a place holder until the class is removed. Please use the ArrayIterator class. """ def __init__(self, *args, **kwargs): logger.error('DataIterator class has been deprecated and renamed' '"ArrayIterator" please use that name.') super(DataIterator, self).__init__(*args, **kwargs) if __name__ == '__main__': from neon.data import load_mnist (X_train, y_train), (X_test, y_test) = load_mnist() from neon.backends.nervanagpu import NervanaGPU ng = NervanaGPU(0, device_id=1) NervanaObject.be = ng ng.bsz = 128 train_set = ArrayIterator( [X_test[:1000], X_test[:1000]], y_test[:1000], nclass=10) for i in range(3): for bidx, (X_batch, y_batch) in enumerate(train_set): print bidx, train_set.start pass
axis=0) if self.be.bsz > bsz: self.ybuf[:, bsz:] = self.be.onehot( self.ydev[:(self.be.bsz - bsz)], axis=0) else: self.ybuf[:, :bsz] = self.ydev[i1:i2].T if self.be.bsz > bsz: self.ybuf[:, bsz:] = self.ydev[:(self.be.bsz - bsz)].T inputs = self.Xbuf[0] if len(self.Xbuf) == 1 else self.Xbuf targets = self.ybuf if self.ybuf else inputs yield (inputs, targets) if __name__ == '__main__': from neon.data import load_mnist (X_train, y_train), (X_test, y_test) = load_mnist() from neon.backends.nervanagpu import NervanaGPU ng = NervanaGPU(0, device_id=1) NervanaObject.be = ng ng.bsz = 128 train_set = DataIterator([X_test[:1000], X_test[:1000]], y_test[:1000], nclass=10) for i in range(3): for bidx, (X_batch, y_batch) in enumerate(train_set): print bidx, train_set.start pass