def get_fake_data(dataset, batch_size, n_iter):
x_train, y_train = generate_data(dataset, batch_size)
train_data = {'image': {'data': x_train, 'axes': ('batch', 'C', 'height', 'width')},
'label': {'data': y_train, 'axes': ('batch',)}}
train_set = ArrayIterator(train_data, batch_size, total_iterations=n_iter)
inputs = train_set.make_placeholders(include_iteration=True)
return inputs, train_data, train_set
def get_fake_data(dataset, batch_size, num_iterations, seed=None):
x_train, y_train = generate_data(dataset, batch_size, rand_seed=seed)
train_data = {
'image': {
'data': x_train,
'axes': ('batch', 'C', 'H', 'W')
},
'label': {
'data': y_train,
'axes': ('batch', )
}
}
train_set = ArrayIterator(train_data,
batch_size,
total_iterations=num_iterations)
inputs = train_set.make_placeholders(include_iteration=True)
return inputs, train_data, train_set
def get_fake_cifar(batch_size, n_iter):
cifar = FakeCIFAR()
cifar.reset(0)
batch_xs, batch_ys = cifar.train.next_batch(batch_size)
x_train = np.vstack(batch_xs).reshape(-1, 3, 32, 32)
y_train = np.vstack(batch_ys).ravel()
train_data = {
'image': {
'data': x_train,
'axes': ('batch', 'C', 'height', 'width')
},
'label': {
'data': y_train,
'axes': ('batch', )
}
}
train_set = ArrayIterator(train_data, batch_size, total_iterations=n_iter)
inputs = train_set.make_placeholders(include_iteration=True)
return inputs, train_data, train_set
from data import make_aeon_loaders
train_set, valid_set = make_aeon_loaders(args.data_dir,
args.batch_size,
args.num_iterations)
else:
from ngraph.frontends.neon import ArrayIterator # noqa
from ngraph.frontends.neon import CIFAR10 # noqa
train_data, valid_data = CIFAR10(args.data_dir).load_data()
train_set = ArrayIterator(train_data,
args.batch_size,
total_iterations=args.num_iterations)
valid_set = ArrayIterator(valid_data, args.batch_size)
# we need to ask the dataset to create an iteration
# placeholder for our learning rate schedule
inputs = train_set.make_placeholders(include_iteration=True)
ax.Y.length = 10
resnet = residual_network(args.stage_depth)
learning_rate_policy = {
'name': 'schedule',
'schedule': [32000, 48000],
'gamma': 0.1,
'base_lr': 0.1
}
optimizer = GradientDescentMomentum(learning_rate=learning_rate_policy,
momentum_coef=0.9,
wdecay=0.0001,
iteration=inputs['iteration'])
parser = NgraphArgparser(description='Train simple CNN on cifar10 dataset')
parser.add_argument('--use_batch_norm',
action='store_true',
help='whether to use batch normalization')
args = parser.parse_args()
np.random.seed(args.rng_seed)
# Create the dataloader
train_data, valid_data = CIFAR10(args.data_dir).load_data()
train_set = ArrayIterator(train_data,
args.batch_size,
total_iterations=args.num_iterations)
valid_set = ArrayIterator(valid_data, args.batch_size)
inputs = train_set.make_placeholders()
ax.Y.length = 10
######################
# Model specification
def cifar_mean_subtract(x):
bgr_mean = ng.persistent_tensor(axes=[x.axes.channel_axis()],
initial_value=np.array([104., 119., 127.]))
return (x - bgr_mean) / 255.
init_uni = UniformInit(-0.1, 0.1)
print("creating training Set ")
train = get_data_array_squad_ngraph(params_dict, data_train, set_val='train')
dev = get_data_array_squad_ngraph(params_dict, data_dev, set_val='dev')
print('Train Set Size is', len(train['para']['data']))
print('Dev set size is', len(dev['para']['data']))
# Use Array Iterator for training set
train_set = ArrayIterator(train, batch_size=params_dict['batch_size'],
total_iterations=params_dict['num_iterations'])
# Use Array Iterator for validation set
valid_set = ArrayIterator(dev, batch_size=params_dict['batch_size'],
total_iterations=params_dict['num_iterations'])
# Make placeholderds for training
inputs = train_set.make_placeholders(include_iteration=True)
# Encoding Layer
rlayer_1 = LSTM(hidden_size, init, activation=Tanh(), reset_cells=True,
gate_activation=Logistic(), return_sequence=True)
# Embedding Layer
embed_layer = LookupTable(
params_dict['vocab_size'],
params_dict['embed_size'],
embeddings,
update=False,
pad_idx=params_dict['pad_idx'])
oov=args.use_oov,
use_match_type=args.use_match_type,
cache_match_type=args.cache_match_type,
cache_vectorized=args.cache_vectorized)
weight_saver = Saver()
# Set num iterations to 1 epoch since we loop over epochs & shuffle
ndata = babi.data_dict['train']['memory']['data'].shape[0]
num_iterations = ndata // args.batch_size
train_set = ArrayIterator(babi.data_dict['train'], batch_size=args.batch_size,
total_iterations=num_iterations)
dev_set = ArrayIterator(babi.data_dict['dev'], batch_size=args.batch_size)
test_set = ArrayIterator(babi.data_dict['test'], batch_size=args.batch_size)
inputs = train_set.make_placeholders()
memn2n = MemN2N_Dialog(
babi.cands,
babi.num_cands,
babi.max_cand_len,
babi.memory_size,
babi.max_utt_len,
babi.vocab_size,
args.emb_size,
args.batch_size,
use_match_type=args.use_match_type,
kb_ents_to_type=babi.kb_ents_to_type,
kb_ents_to_cand_idxs=babi.kb_ents_to_cand_idxs,
match_type_idxs=babi.match_type_idxs,
nhops=args.nhops,
def train_mnist_mlp(transformer_name,
data_dir=None,
rng_seed=12,
batch_size=128,
train_iter=10,
eval_iter=10):
assert transformer_name in ['cpu', 'hetr']
assert isinstance(rng_seed, int)
# Apply this metadata to graph regardless of transformer,
# but it is ignored for non-HeTr case
hetr_device_ids = (0, 1)
# use consistent rng seed between runs
np.random.seed(rng_seed)
# Data
train_data, valid_data = MNIST(path=data_dir).load_data()
train_set = ArrayIterator(train_data,
batch_size,
total_iterations=train_iter)
valid_set = ArrayIterator(valid_data, batch_size)
inputs = train_set.make_placeholders()
ax.Y.length = 10
# Model
with ng.metadata(device_id=hetr_device_ids, parallel=ax.N):
seq1 = Sequential([
Preprocess(functor=lambda x: x / 255.),
Affine(nout=100, weight_init=GaussianInit(), activation=Rectlin()),
Affine(axes=ax.Y,
weight_init=GaussianInit(),
activation=Logistic())
])
train_prob = seq1(inputs['image'])
train_loss = ng.cross_entropy_binary(
train_prob, ng.one_hot(inputs['label'], axis=ax.Y))
optimizer = GradientDescentMomentum(0.1, 0.9)
batch_cost = ng.sequential(
[optimizer(train_loss),
ng.mean(train_loss, out_axes=())])
train_outputs = dict(batch_cost=batch_cost)
with Layer.inference_mode_on():
inference_prob = seq1(inputs['image'])
errors = ng.not_equal(ng.argmax(inference_prob, out_axes=[ax.N]),
inputs['label'])
eval_loss = ng.cross_entropy_binary(
inference_prob, ng.one_hot(inputs['label'], axis=ax.Y))
eval_outputs = dict(cross_ent_loss=eval_loss, misclass_pct=errors)
# Runtime
with closing(
ngt.make_transformer_factory(transformer_name)()) as transformer:
train_computation = make_bound_computation(transformer, train_outputs,
inputs)
loss_computation = make_bound_computation(transformer, eval_outputs,
inputs)
train_costs = list()
for step in range(train_iter):
out = train_computation(next(train_set))
train_costs.append(float(out['batch_cost']))
ce_loss = list()
for step in range(eval_iter):
out = loss_computation(next(valid_set))
ce_loss.append(np.mean(out['cross_ent_loss']))
return train_costs, ce_loss