def get_fake_data(dataset, batch_size, n_iter): x_train, y_train = generate_data(dataset, batch_size) train_data = {'image': {'data': x_train, 'axes': ('batch', 'C', 'height', 'width')}, 'label': {'data': y_train, 'axes': ('batch',)}} train_set = ArrayIterator(train_data, batch_size, total_iterations=n_iter) inputs = train_set.make_placeholders(include_iteration=True) return inputs, train_data, train_set
def get_fake_data(dataset, batch_size, num_iterations, seed=None): x_train, y_train = generate_data(dataset, batch_size, rand_seed=seed) train_data = { 'image': { 'data': x_train, 'axes': ('batch', 'C', 'H', 'W') }, 'label': { 'data': y_train, 'axes': ('batch', ) } } train_set = ArrayIterator(train_data, batch_size, total_iterations=num_iterations) inputs = train_set.make_placeholders(include_iteration=True) return inputs, train_data, train_set
def get_fake_cifar(batch_size, n_iter): cifar = FakeCIFAR() cifar.reset(0) batch_xs, batch_ys = cifar.train.next_batch(batch_size) x_train = np.vstack(batch_xs).reshape(-1, 3, 32, 32) y_train = np.vstack(batch_ys).ravel() train_data = { 'image': { 'data': x_train, 'axes': ('batch', 'C', 'height', 'width') }, 'label': { 'data': y_train, 'axes': ('batch', ) } } train_set = ArrayIterator(train_data, batch_size, total_iterations=n_iter) inputs = train_set.make_placeholders(include_iteration=True) return inputs, train_data, train_set
from data import make_aeon_loaders train_set, valid_set = make_aeon_loaders(args.data_dir, args.batch_size, args.num_iterations) else: from ngraph.frontends.neon import ArrayIterator # noqa from ngraph.frontends.neon import CIFAR10 # noqa train_data, valid_data = CIFAR10(args.data_dir).load_data() train_set = ArrayIterator(train_data, args.batch_size, total_iterations=args.num_iterations) valid_set = ArrayIterator(valid_data, args.batch_size) # we need to ask the dataset to create an iteration # placeholder for our learning rate schedule inputs = train_set.make_placeholders(include_iteration=True) ax.Y.length = 10 resnet = residual_network(args.stage_depth) learning_rate_policy = { 'name': 'schedule', 'schedule': [32000, 48000], 'gamma': 0.1, 'base_lr': 0.1 } optimizer = GradientDescentMomentum(learning_rate=learning_rate_policy, momentum_coef=0.9, wdecay=0.0001, iteration=inputs['iteration'])
parser = NgraphArgparser(description='Train simple CNN on cifar10 dataset') parser.add_argument('--use_batch_norm', action='store_true', help='whether to use batch normalization') args = parser.parse_args() np.random.seed(args.rng_seed) # Create the dataloader train_data, valid_data = CIFAR10(args.data_dir).load_data() train_set = ArrayIterator(train_data, args.batch_size, total_iterations=args.num_iterations) valid_set = ArrayIterator(valid_data, args.batch_size) inputs = train_set.make_placeholders() ax.Y.length = 10 ###################### # Model specification def cifar_mean_subtract(x): bgr_mean = ng.persistent_tensor(axes=[x.axes.channel_axis()], initial_value=np.array([104., 119., 127.])) return (x - bgr_mean) / 255. init_uni = UniformInit(-0.1, 0.1)
print("creating training Set ") train = get_data_array_squad_ngraph(params_dict, data_train, set_val='train') dev = get_data_array_squad_ngraph(params_dict, data_dev, set_val='dev') print('Train Set Size is', len(train['para']['data'])) print('Dev set size is', len(dev['para']['data'])) # Use Array Iterator for training set train_set = ArrayIterator(train, batch_size=params_dict['batch_size'], total_iterations=params_dict['num_iterations']) # Use Array Iterator for validation set valid_set = ArrayIterator(dev, batch_size=params_dict['batch_size'], total_iterations=params_dict['num_iterations']) # Make placeholderds for training inputs = train_set.make_placeholders(include_iteration=True) # Encoding Layer rlayer_1 = LSTM(hidden_size, init, activation=Tanh(), reset_cells=True, gate_activation=Logistic(), return_sequence=True) # Embedding Layer embed_layer = LookupTable( params_dict['vocab_size'], params_dict['embed_size'], embeddings, update=False, pad_idx=params_dict['pad_idx'])
oov=args.use_oov, use_match_type=args.use_match_type, cache_match_type=args.cache_match_type, cache_vectorized=args.cache_vectorized) weight_saver = Saver() # Set num iterations to 1 epoch since we loop over epochs & shuffle ndata = babi.data_dict['train']['memory']['data'].shape[0] num_iterations = ndata // args.batch_size train_set = ArrayIterator(babi.data_dict['train'], batch_size=args.batch_size, total_iterations=num_iterations) dev_set = ArrayIterator(babi.data_dict['dev'], batch_size=args.batch_size) test_set = ArrayIterator(babi.data_dict['test'], batch_size=args.batch_size) inputs = train_set.make_placeholders() memn2n = MemN2N_Dialog( babi.cands, babi.num_cands, babi.max_cand_len, babi.memory_size, babi.max_utt_len, babi.vocab_size, args.emb_size, args.batch_size, use_match_type=args.use_match_type, kb_ents_to_type=babi.kb_ents_to_type, kb_ents_to_cand_idxs=babi.kb_ents_to_cand_idxs, match_type_idxs=babi.match_type_idxs, nhops=args.nhops,
def train_mnist_mlp(transformer_name, data_dir=None, rng_seed=12, batch_size=128, train_iter=10, eval_iter=10): assert transformer_name in ['cpu', 'hetr'] assert isinstance(rng_seed, int) # Apply this metadata to graph regardless of transformer, # but it is ignored for non-HeTr case hetr_device_ids = (0, 1) # use consistent rng seed between runs np.random.seed(rng_seed) # Data train_data, valid_data = MNIST(path=data_dir).load_data() train_set = ArrayIterator(train_data, batch_size, total_iterations=train_iter) valid_set = ArrayIterator(valid_data, batch_size) inputs = train_set.make_placeholders() ax.Y.length = 10 # Model with ng.metadata(device_id=hetr_device_ids, parallel=ax.N): seq1 = Sequential([ Preprocess(functor=lambda x: x / 255.), Affine(nout=100, weight_init=GaussianInit(), activation=Rectlin()), Affine(axes=ax.Y, weight_init=GaussianInit(), activation=Logistic()) ]) train_prob = seq1(inputs['image']) train_loss = ng.cross_entropy_binary( train_prob, ng.one_hot(inputs['label'], axis=ax.Y)) optimizer = GradientDescentMomentum(0.1, 0.9) batch_cost = ng.sequential( [optimizer(train_loss), ng.mean(train_loss, out_axes=())]) train_outputs = dict(batch_cost=batch_cost) with Layer.inference_mode_on(): inference_prob = seq1(inputs['image']) errors = ng.not_equal(ng.argmax(inference_prob, out_axes=[ax.N]), inputs['label']) eval_loss = ng.cross_entropy_binary( inference_prob, ng.one_hot(inputs['label'], axis=ax.Y)) eval_outputs = dict(cross_ent_loss=eval_loss, misclass_pct=errors) # Runtime with closing( ngt.make_transformer_factory(transformer_name)()) as transformer: train_computation = make_bound_computation(transformer, train_outputs, inputs) loss_computation = make_bound_computation(transformer, eval_outputs, inputs) train_costs = list() for step in range(train_iter): out = train_computation(next(train_set)) train_costs.append(float(out['batch_cost'])) ce_loss = list() for step in range(eval_iter): out = loss_computation(next(valid_set)) ce_loss.append(np.mean(out['cross_ent_loss'])) return train_costs, ce_loss