def transformer_factory(request):
def set_and_get_factory(transformer_name):
factory = ngt.make_transformer_factory(transformer_name)
ngt.set_transformer_factory(factory)
return factory
transformer_name = request.param
if pytest.config.getoption("--enable_flex"):
if transformer_name == flex_gpu_transformer_name:
if flex_gpu_transformer_name in ngt.transformer_choices():
yield set_and_get_factory(transformer_name)
else:
raise ValueError("GPU not found, should not set --enable_flex"
"flag for py.test.")
else:
pytest.skip(
'Skip all other transformers since --enable_flex is set.')
else:
if transformer_name == flex_gpu_transformer_name:
pytest.skip('Skip flex test since --enable_flex is not set.')
else:
yield set_and_get_factory(transformer_name)
# Reset transformer factory to default
ngt.set_transformer_factory(ngt.make_transformer_factory("numpy"))
def transformer_factory(request):
factory = ngt.make_transformer_factory(request.param)
ngt.set_transformer_factory(factory)
yield factory
# Reset transformer factory to default
ngt.set_transformer_factory(ngt.make_transformer_factory("numpy"))
def parse_args(self, gen_be=True):
args = super(NgraphArgparser, self).parse_args()
factory = ngt.make_transformer_factory(args.backend)
ngt.set_transformer_factory(factory)
# invert no_progress_bar meaning and store in args.progress_bar
args.progress_bar = not args.no_progress_bar
return args
def transformer_factory(request):
def set_and_get_factory(transformer_name):
factory = ngt.make_transformer_factory(transformer_name)
ngt.set_transformer_factory(factory)
return factory
name = request.config.getoption("--transformer")
yield set_and_get_factory(name)
# Reset transformer factory to default
ngt.set_transformer_factory(ngt.make_transformer_factory("cpu"))
def make_and_set_transformer_factory(args):
flex_args = ('fixed_point', 'flex_verbose', 'collect_flex_data')
# default value for all flex args if not given, confusing with store_true in add_argument
default = False
if args.backend == flex_gpu_transformer_name:
flex_args_dict = dict(
(a, getattr(args, a, default)) for a in flex_args)
factory = ngt.make_transformer_factory(args.backend,
**flex_args_dict)
else:
factory = ngt.make_transformer_factory(args.backend)
ngt.set_transformer_factory(factory)
dtype=np.int32)
utt_lens_val = ((2 * lbl_lens_val + 1) / float(max_utt_len)) * 100
utt_lens_val = utt_lens_val.astype(np.int32)
fprop_ctc = costfun(activs_val, lbls_val, utt_lens_val,
lbl_lens_val).copy()
bprop_ctc = grad_costfun(activs_val, lbls_val, utt_lens_val,
lbl_lens_val).copy()
# compare with reference values
costs_ref, grads_ref = ctc_ref(activs_val, lbls_val, utt_lens_val,
lbl_lens_val)
ng.testing.assert_allclose(fprop_ctc,
costs_ref,
rtol=1.0e-5,
atol=1.0e-5)
ng.testing.assert_allclose(bprop_ctc[0],
grads_ref,
rtol=1.0e-5,
atol=1.0e-5)
if __name__ == '__main__':
factory = ngt.make_transformer_factory('cpu')
ngt.set_transformer_factory(factory)
bsz = 4
nout = np.random.randint(5, 10)
max_lbl_len = np.random.randint(5, 20)
max_utt_len = 2 * max_lbl_len + 1
data_args = (nout, bsz, max_utt_len, max_lbl_len)
test_ctc(factory, data_args)
def test_gdm(args, transformer_factory):
"""
Test the ngraph GradientDescentMomentum against the neon version across 10 update steps.
"""
# set up parameters
C = ng.make_axis(20, name="C")
N = ng.make_axis(32, name="N", batch=True)
be = gen_backend(backend='cpu', batch_size=N.length)
# restrict to numpy transformer for now
factory = ngt.make_transformer_factory('numpy')
ngt.set_transformer_factory(factory)
ngt.make_transformer()
# generate dummy data (to initialize values)
w_init = np.random.rand(C.length).astype('float32')
# set up nervana graph
X = ng.placeholder([C, N]).named('X')
Y = ng.placeholder([N]).named('Y')
W = ng.variable([C - 1], initial_value=w_init).named('W')
ex = ExecutorFactory()
transformer = ex.transformer
lrate, mom, wdecay = args
gdm = GradientDescentMomentum(learning_rate=lrate, momentum_coef=mom, wdecay=wdecay)
cost = ng.sum(Y - ng.dot(W, X), out_axis=())
# to call ngraph gdm, use (ngraph_W, _) = ngraph_optimize(x, y)
# where (x, y) are nparrays that fill the placeholders X and Y
updates = gdm(cost)
ngraph_optimize = transformer.computation([W, updates], X, Y)
transformer.initialize()
# set up the neon gdm
neon_gdm = NeonGradientDescentMomentum(learning_rate=lrate, momentum_coef=mom, wdecay=wdecay)
# dev_v0 = be.zeros((C.length, 1)) # velocities are zero at the beginning
dev_dw = be.zeros((C.length, 1)) # we fill the gradient info in the below
dev_w_init = be.array(w_init) # copy w_init to device
param_list = [((dev_w_init, dev_dw), [])]
# store the weights with each minibatch for debugging
ng_Ws = []
be_Ws = []
# run for 20 minibatches
for i, (x, y) in enumerate([generate_data(C.length, N.length) for _ in range(20)]):
# obtain ngraph results
(ng_W, _) = ngraph_optimize(x, y)
ng_Ws.append(copy.deepcopy(ng_W))
# obtain neon results
dw = -1 * x.sum(axis=1) # the gradients we compute analytically
param_list[0][0][1].set(dw) # fill the gradient
neon_gdm.optimize([DummyLayer(param_list)], epoch=0)
(param, grad), states = param_list[0]
be_W = param.get()[:, 0]
be_Ws.append(be_W)
np.testing.assert_allclose(be_W, ng_W, rtol=1e-3)
def make_and_set_transformer_factory(self, args):
factory = ngt.make_transformer_factory(args.backend)
ngt.set_transformer_factory(factory)
def set_and_get_factory(transformer_name):
factory = ngt.make_transformer_factory(transformer_name)
ngt.set_transformer_factory(factory)
return factory
def make_and_set_transformer_factory(self, args):
if args.backend == flex_gpu_transformer_name:
FlexNgraphArgparser.make_and_set_transformer_factory(args)
else:
factory = ngt.make_transformer_factory(args.backend)
ngt.set_transformer_factory(factory)
