def test_native_user_function(tmpdir): if not C.cntk_py.is_native_user_function_registered('NativeUserTimesOp'): C.ops.register_native_user_function( 'NativeUserTimesOp', 'Cntk.ExtensibilityExamples-' + C.__version__.rstrip('+'), 'CreateUserTimesFunction') dev = C.cpu() x = C.input_variable((2)) w = C.parameter((2, 2), init=np.asarray([[0.5, 2], [-0.5, 1.5]], dtype=np.float32), device=dev) attributes = { 'param_rank': 2, 'padding': True, 'none': None, 'nested lists': [[1, 2, 3], [4, 5, 6]], 'string': 'string', 'some data': np.arange(1, 10, dtype=np.float32).reshape((3, 3)) } def verify_attributes(udf): for k, v in attributes.items(): if not isinstance(v, np.ndarray): assert udf.attributes[k] == v else: assert (udf.attributes[k] == v).all() op = C.ops.native_user_function('NativeUserTimesOp', [w, x], attributes, 'native_user_times_function') verify_attributes(op.owner) filepath = str(tmpdir / 'test_native_user_function.dat') op.save(filepath) op_reloaded = Function.load(filepath, device=dev) x_data = C.NDArrayView.from_dense(np.asarray([[0.1, 0.2], [-0.1, 0.3]], dtype=np.float32), device=dev) result = op_reloaded.eval({op_reloaded.arguments[0]: x_data}, device=dev) assert np.allclose(result, [[-0.05, 0.5], [-0.2, 0.25]]) native_times_primitive = op_reloaded.find_by_name( 'native_user_times_function') verify_attributes(native_times_primitive)
def test_override_serialize(tmpdir): dev = C.cpu() a, b = 1.2322341, -0.29084 op = MyPlusPlus([C.constant(a), C.constant(b)], '++') op = MyPlusPlus([op, op], '+++') op = MyPlusPlus([op, op], '++++') op = C.user_function(op) result1 = op.eval({}, device=dev) filepath = str(tmpdir / 'test_udf_with_renamed_deserialize.dat') op.save(filepath) op_reloaded = Function.load(filepath, device=dev) assert result1 == op_reloaded.eval({}, device=dev)
def test_both_flavors_of_user_functions(tmpdir): dev, w_value, c1_value, c2_value, op = build_test_function() filepath = str(tmpdir / 'test_native_user_function.dat') op.save(filepath) op_reloaded = Function.load(filepath, device=dev) np.random.seed(1) for i in range(5): x_value = np.random.random((2, 2)).astype(np.float32) x_data = C.NDArrayView.from_dense(x_value, device=dev) result = op_reloaded.eval({op_reloaded.arguments[0]: x_data}, device=dev) expected = np.matmul((x_value + c1_value), w_value) + c2_value assert np.allclose(result, expected)
def test_ext_train(tmpdir): dim = 4 p = C.parameter(shape=(dim, ), init=10) i = C.sequence.input_variable(dim, needs_gradient=True, name='i_var') m = MyPlus(i, C.constant(3), 'my_plus') # keeping m unwrapped since we need to access its member variables z = C.user_function(m) + p momentum_time_constant = C.momentum_as_time_constant_schedule(1100) lr_per_sample = C.learning_parameter_schedule(0.007, minibatch_size=1) trainer = C.Trainer(z, (z + 0, z + 0), [ C.momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant, True, minibatch_size=0) ]) i = 0 while i < 100: i += 1 input_data = np.random.rand(dim) trainer.train_minibatch([input_data]) assert m.forward_calls == m.backward_calls == 100 filepath = str(tmpdir / 'test_ext_train.dat') z.save(filepath) buf = open(filepath, 'rb').read() # this is only need for Python 2.7 # (which does not distinguish between bytes and strings) if isinstance(buf, str): buf = bytearray(buf) z1 = Function.load(buf) m1 = z1.find_by_name('my_plus') # m1 is an instance of UserFunction, cannot directly downcast it to MyPlus, # using serialize as workaround: state = m1.serialize()['state'] assert state['forward_calls'] == state['backward_calls'] == 100
def test_ext_lambdafunc(tmpdir): dim = 4 class CallbackCounter(object): def __init__(self): self.count = 0 def inc(self, arg): self.count += 1 cb = CallbackCounter() p = C.parameter(shape=(dim,), init=1) i = C.input_variable(dim, needs_gradient=True, name='i_var') k = i * p m = LambdaFunc(k, when=lambda arg: np.sum(arg) > 1, execute=cb.inc) m = C.user_function(m) z0 = m + 0 filepath = str(tmpdir / 'test_ext_lambdafunc.dat') z0.save(filepath) Function.register_udf_deserialize_callback('conditional_exec_lambda', lambda x, *unused: LambdaFunc(x, when=lambda arg: np.sum(arg) > 1, execute=cb.inc)) z = Function.load(filepath) momentum_time_constant = C.momentum_as_time_constant_schedule(1100) lr_per_sample = C.learning_rate_schedule(0.007, C.UnitType.sample) trainer = C.Trainer(z, (z + 0, z + 0), [C.momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant, True)]) i = 0 input_data = 0.1 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 0 input_data = 0.3 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 1
def test_ext_lambdafunc(tmpdir): dim = 4 class CallbackCounter(object): def __init__(self): self.count = 0 def inc(self, arg): self.count += 1 cb = CallbackCounter() p = C.parameter(shape=(dim,), init=1) i = C.input_variable(dim, needs_gradient=True, name='i_var') k = i * p m = LambdaFunc(k, when=lambda arg: np.sum(arg) > 1, execute=cb.inc) m = C.user_function(m) z0 = m + 0 filepath = str(tmpdir / 'test_ext_lambdafunc.dat') z0.save(filepath) Function.register_udf_deserialize_callback('conditional_exec_lambda', lambda x, *unused: LambdaFunc(x, when=lambda arg: np.sum(arg) > 1, execute=cb.inc)) z = Function.load(filepath) momentum_time_constant = C.momentum_as_time_constant_schedule(1100) lr_per_sample = C.learning_parameter_schedule(0.007, minibatch_size = 1) trainer = C.Trainer(z, (z + 0, z + 0), [C.momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant, True)]) i = 0 input_data = 0.1 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 0 input_data = 0.3 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 1
def test_override_deserialize(tmpdir): dev, w_value, c1_value, c2_value, op = build_test_function() filepath = str(tmpdir / 'test_override_deserialize.dat') op.save(filepath) Function.register_udf_deserialize_callback(MyPlus._op_name(), lambda *x: MyPlusPlus(*x)) op_reloaded = Function.load(filepath, device=dev) np.random.seed(1) for i in range(5): x_value = np.random.random((2, 2)).astype(np.float32) x_data = C.NDArrayView.from_dense(x_value, device=dev) result = op_reloaded.eval({op_reloaded.arguments[0]: x_data}, device=dev) expected = 2 * (np.matmul(2 * (x_value + c1_value), w_value) + c2_value) assert np.allclose(result, expected)
def test_ext_lambdafunc(tmpdir): dim = 4 class CallbackCounter(object): def __init__(self): self.count = 0 def inc(self, arg): self.count += 1 cb = CallbackCounter() p = parameter(shape=(dim, ), init=1) i = input(dim, needs_gradient=True, name='i_var') k = i * p m = LambdaFunc(k, when=lambda arg: np.sum(arg) > 1, execute=cb.inc) m = user_function(m) z0 = m + 0 filepath = str(tmpdir / 'test_ext_lambdafunc.dat') z0.save(filepath) z = Function.load( filepath, udf_factory_callback_map={ 'conditional_exec_lambda': lambda x, *unused: LambdaFunc( x, when=lambda arg: np.sum(arg) > 1, execute=cb.inc) }) momentum_time_constant = momentum_as_time_constant_schedule(1100) lr_per_sample = learning_rate_schedule(0.007, UnitType.sample) trainer = Trainer(z, (z+0, z+0), \ [momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant, True)]) i = 0 input_data = 0.1 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 0 input_data = 0.3 * np.ones(dim) trainer.train_minibatch([input_data]) assert cb.count == 1
def test_ext_train(tmpdir): dim = 4 p = C.parameter(shape=(dim,), init=10) i = C.sequence.input_variable(dim, needs_gradient=True, name='i_var') m = MyPlus(i, C.constant(3), 'my_plus') # keeping m unwrapped since we need to access its member variables z = C.user_function(m) + p momentum_time_constant = C.momentum_as_time_constant_schedule(1100) lr_per_sample = C.learning_rate_schedule(0.007, C.UnitType.sample) trainer = C.Trainer(z, (z + 0, z + 0), [C.momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant, True)]) i = 0 while i < 100: i += 1 input_data = np.random.rand(dim) trainer.train_minibatch([input_data]) assert m.forward_calls == m.backward_calls == 100 filepath = str(tmpdir / 'test_ext_train.dat') z.save(filepath) buf = open(filepath, 'rb').read() # this is only need for Python 2.7 # (which does not distinguish between bytes and strings) if isinstance(buf, str): buf = bytearray(buf) z1 = Function.load(buf) m1 = z1.find_by_name('my_plus') # m1 is an instance of UserFunction, cannot directly downcast it to MyPlus, # using serialize as workaround: state = m1.serialize()['state'] assert state['forward_calls'] == state['backward_calls'] == 100
def test_native_user_function(tmpdir): if not C.cntk_py.is_native_user_function_registered('NativeUserTimesOp'): C.ops.register_native_user_function('NativeUserTimesOp', 'Cntk.ExtensibilityExamples-' + C.__version__.rstrip('+'), 'CreateUserTimesFunction') dev = C.cpu() x = C.input_variable((2)) w = C.parameter((2, 2), init=np.asarray([[0.5, 2], [-0.5, 1.5]], dtype=np.float32), device=dev) attributes = {'param_rank': 2, 'padding': True, 'none': None, 'nested lists': [[1, 2, 3], [4, 5, 6]], 'string': 'string', 'some data': np.arange(1, 10, dtype=np.float32).reshape((3, 3)) } def verify_attributes(udf): for k, v in attributes.items(): if not isinstance(v, np.ndarray): assert udf.attributes[k] == v else: assert (udf.attributes[k] == v).all() op = C.ops.native_user_function('NativeUserTimesOp', [w, x], attributes, 'native_user_times_function') verify_attributes(op.owner) filepath = str(tmpdir / 'test_native_user_function.dat') op.save(filepath) op_reloaded = Function.load(filepath, device=dev) x_data = C.NDArrayView.from_dense(np.asarray([[0.1, 0.2], [-0.1, 0.3]], dtype=np.float32), device=dev) result = op_reloaded.eval({op_reloaded.arguments[0]: x_data}, device=dev) assert np.allclose(result, [[-0.05, 0.5], [-0.2, 0.25]]) native_times_primitive = op_reloaded.find_by_name('native_user_times_function') verify_attributes(native_times_primitive)
# hook up data vocab, i2w, w2i = get_vocab(os.path.join(DATA_DIR, VOCAB_FILE)) # create inputs and create model model = create_model() # train train_reader = create_reader(os.path.join(DATA_DIR, TRAINING_DATA), True) valid_reader = create_reader(os.path.join(DATA_DIR, VALIDATION_DATA), True) train(train_reader, valid_reader, vocab, i2w, model, max_epochs=30, epoch_size=908241) test_epoch = 10 model = Function.load(model_path(test_epoch)) # test string error rate on decoded output test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False) evaluate_decoding(test_reader, model, i2w) # test same metric same as in training on test set test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False) evaluate_metric(test_reader, model) # try the model out in an interactive session interactive_session(model, vocab, i2w, show_attention=True)
if __name__ == '__main__': #try_set_default_device(cpu()) from _cntk_py import set_fixed_random_seed set_fixed_random_seed(1) # hook up data vocab, i2w, w2i = get_vocab(os.path.join(DATA_DIR, VOCAB_FILE)) # create inputs and create model model = create_model() # train train_reader = create_reader(os.path.join(DATA_DIR, TRAINING_DATA), True) valid_reader = create_reader(os.path.join(DATA_DIR, VALIDATION_DATA), True) train(train_reader, valid_reader, vocab, i2w, model, max_epochs=30, epoch_size=908241) test_epoch = 10 model = Function.load(model_path(test_epoch)) # test string error rate on decoded output test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False) evaluate_decoding(test_reader, model, i2w) # test same metric same as in training on test set test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False) evaluate_metric(test_reader, model) # try the model out in an interactive session interactive_session(model, vocab, i2w, show_attention=True)