def data(data_dir, batch_size, num_parts=1, part_index=0):
if data_dir == 'data/':
sys.path.insert(0, "../../tests/python/common")
import get_data
get_data.GetMNIST_ubyte()
train_dataiter = mx.io.MNISTIter(
image=data_dir + "train-images-idx3-ubyte",
label=data_dir + "train-labels-idx1-ubyte",
data_shape=(1, 28, 28),
batch_size=batch_size,
shuffle=True,
flat=False,
silent=False,
num_parts=num_parts,
part_index=part_index)
val_dataiter = mx.io.MNISTIter(image=data_dir + "t10k-images-idx3-ubyte",
label=data_dir + "t10k-labels-idx1-ubyte",
data_shape=(1, 28, 28),
batch_size=batch_size,
shuffle=True,
flat=False,
silent=False)
return (train_dataiter, val_dataiter)
def get_iter(data_dir):
if data_dir == 'data':
sysath.append(os.path.join(curr_path, '../common/'))
import get_data
get_data.GetMNIST_ubyte()
batch_size = 100
train_dataiter = mx.io.MNISTIter(
image = data_dir + "/train-images-idx3-ubyte",
label = data_dir + "/train-labels-idx1-ubyte",
data_shape=(1, 28, 28),
batch_size=batch_size, shuffle=True, flat=False, silent=False)
val_dataiter = mx.io.MNISTIter(
image = data_dir + "/t10k-images-idx3-ubyte",
label = data_dir + "/t10k-labels-idx1-ubyte",
data_shape=(1, 28, 28),
batch_size=batch_size, shuffle=True, flat=False, silent=False)
return (train_dataiter, val_dataiter)
def mnist_iterator(batch_size, input_shape):
"""return train and val iterators for mnist"""
# download data
get_data.GetMNIST_ubyte()
flat = False if len(input_shape) == 3 else True
train_dataiter = mx.io.MNISTIter(image="data/train-images-idx3-ubyte",
label="data/train-labels-idx1-ubyte",
input_shape=input_shape,
batch_size=batch_size,
shuffle=True,
flat=flat)
val_dataiter = mx.io.MNISTIter(image="data/t10k-images-idx3-ubyte",
label="data/t10k-labels-idx1-ubyte",
input_shape=input_shape,
batch_size=batch_size,
flat=flat)
return (train_dataiter, val_dataiter)
def mnist(batch_size, input_shape, num_parts=1, part_index=0):
"""return mnist iters"""
get_data.GetMNIST_ubyte()
flat = len(input_shape) == 1
train = mx.io.MNISTIter(image="data/train-images-idx3-ubyte",
label="data/train-labels-idx1-ubyte",
data_shape=input_shape,
batch_size=batch_size,
num_parts=num_parts,
part_index=part_index,
shuffle=False,
flat=flat,
silent=False)
val = mx.io.MNISTIter(image="data/t10k-images-idx3-ubyte",
label="data/t10k-labels-idx1-ubyte",
data_shape=input_shape,
batch_size=batch_size,
shuffle=False,
flat=flat,
silent=False)
return (train, val)
def get_iterator_impl_mnist(args, kv):
"""return train and val iterators for mnist"""
# download data
get_data.GetMNIST_ubyte()
flat = False if len(data_shape) != 1 else True
train = mx.io.MNISTIter(image="data/train-images-idx3-ubyte",
label="data/train-labels-idx1-ubyte",
input_shape=data_shape,
batch_size=args.batch_size,
shuffle=True,
flat=flat,
num_parts=kv.num_workers,
part_index=kv.rank)
val = mx.io.MNISTIter(image="data/t10k-images-idx3-ubyte",
label="data/t10k-labels-idx1-ubyte",
input_shape=data_shape,
batch_size=args.batch_size,
flat=flat,
num_parts=kv.num_workers,
part_index=kv.rank)
return (train, val)
softmax = mx.symbol.Softmax(fc3, name='sm')
def accuracy(label, pred):
py = np.argmax(pred, axis=1)
return np.sum(py == label) / float(label.size)
num_round = 4
prefix = './mlp'
kv = mx.kvstore.create('dist')
batch_size /= kv.get_num_workers()
#check data
get_data.GetMNIST_ubyte()
train_dataiter = mx.io.MNISTIter(image="data/train-images-idx3-ubyte",
label="data/train-labels-idx1-ubyte",
data_shape=(784, ),
num_parts=kv.get_num_workers(),
part_index=kv.get_rank(),
batch_size=batch_size,
shuffle=True,
flat=True,
silent=False,
seed=10)
val_dataiter = mx.io.MNISTIter(image="data/t10k-images-idx3-ubyte",
label="data/t10k-labels-idx1-ubyte",
data_shape=(784, ),
batch_size=batch_size,
