def test_model_not_criterion_subset():
input_dim = 2
proj_dim = 11
model1_dim = 3
model2_dim = 4
x = sequence.input((input_dim, ))
core = Embedding(proj_dim)
model1 = Dense(model1_dim)(sequence.last(core(x)))
model1_label = input((model1_dim, ))
ce_model1 = cross_entropy_with_softmax(model1, model1_label)
pe_model1 = classification_error(model1, model1_label)
model2 = Dense(model2_dim)(core(x))
model2_label = sequence.input((model2_dim, ))
ce_model2 = cross_entropy_with_softmax(model2, model2_label)
pe_model2 = classification_error(model2, model2_label)
ce = 0.5 * sequence.reduce_sum(ce_model2) + 0.5 * ce_model1
lr_schedule = learning_rate_schedule(0.003, UnitType.sample)
trainer_multitask = Trainer(model1, (ce, pe_model1),
sgd(ce.parameters, lr=lr_schedule))
x_data = np.asarray([[2., 1.], [1., 2.]], np.float32)
model1_label_data = np.asarray([1., 0., 0.], np.float32)
model2_label_data = np.asarray([[0., 1., 0., 0.], [0., 0., 0., 1.]],
np.float32)
trainer_multitask.train_minibatch({
x: [x_data],
model1_label: [model1_label_data],
model2_label: [model2_label_data]
})
def test_model_not_criterion_subset():
input_dim = 2
proj_dim = 11
model1_dim = 3
model2_dim = 4
x = input_variable((input_dim,))
core = Embedding(proj_dim)
model1 = Dense(model1_dim)(sequence.last(core(x)))
model1_label = input_variable((model1_dim,), dynamic_axes=[Axis.default_batch_axis()])
ce_model1 = cross_entropy_with_softmax(model1, model1_label)
pe_model1 = classification_error(model1, model1_label)
model2 = Dense(model2_dim)(core(x))
model2_label = input_variable((model2_dim,))
ce_model2 = cross_entropy_with_softmax(model2, model2_label)
pe_model2 = classification_error(model2, model2_label)
ce = 0.5 * sequence.reduce_sum(ce_model2) + 0.5 * ce_model1
lr_schedule = learning_rate_schedule(0.003, UnitType.sample)
trainer_multitask = Trainer(model1, (ce, pe_model1), sgd(ce.parameters, lr=lr_schedule))
x_data = np.asarray([[2., 1.], [1., 2.]], np.float32)
model1_label_data = np.asarray([1., 0., 0.], np.float32)
model2_label_data = np.asarray([[0., 1., 0., 0.], [0., 0., 0., 1.]], np.float32)
trainer_multitask.train_minibatch({x : [x_data], model1_label : [model1_label_data], model2_label : [model2_label_data]})
def test_op_times_reduce_sequence_axis(device_id, precision):
dt_precision = PRECISION_TO_TYPE[precision]
from cntk import times, Value, TIMES_REDUCE_SEQUENCE_AXIS_WITHOUT_INFERRED_INPUT_RANK
from cntk import sequence
dim = 10
seq = [[0,1,2], [3], [4,5,6,7,8,9]]
right_data = Value.one_hot(seq, dim, dtype=dt_precision)
right_var = sequence.input_variable(shape=(dim), is_sparse=True, dtype=dt_precision)
left_data = [AA([1,1,1],dtype=dt_precision), AA([1],dtype=dt_precision), AA([1,1,1,1,1,1],dtype=dt_precision)]
left_var = sequence.input_variable(shape=(1), dtype=dt_precision)
func = times(left_var, right_var, infer_input_rank_to_map=TIMES_REDUCE_SEQUENCE_AXIS_WITHOUT_INFERRED_INPUT_RANK)
func2 = sequence.reduce_sum(times(left_var, right_var))
assert func.dynamic_axes == func2.dynamic_axes
_, forward_output = func.forward({left_var:left_data, right_var:right_data})
actual_forward = forward_output[func.output]
expected_forward = AA([[[1,1,1,0,0,0,0,0,0,0]],
[[0,0,0,1,0,0,0,0,0,0]],
[[0,0,0,0,1,1,1,1,1,1]]])
assert np.allclose(actual_forward, expected_forward)
def test_usermbsource_training(tmpdir):
input_dim = 1000
num_output_classes = 5
mbs = MyDataSource(input_dim, num_output_classes)
from cntk import sequence, parameter, plus, cross_entropy_with_softmax, \
classification_error, learning_rate_schedule, sgd, Trainer, \
training_session, times, UnitType
feature = sequence.input_variable(shape=(input_dim, ))
label = C.input_variable(shape=(num_output_classes, ))
p = parameter(shape=(input_dim, num_output_classes), init=10)
z = times(sequence.reduce_sum(feature), p, name='z')
ce = cross_entropy_with_softmax(z, label)
errs = classification_error(z, label)
lr_per_sample = learning_rate_schedule([0.3, 0.2, 0.1, 0.0],
UnitType.sample)
learner = sgd(z.parameters, lr_per_sample)
trainer = Trainer(z, (ce, errs), [learner])
input_map = {feature: mbs.fsi, label: mbs.lsi}
session = training_session(trainer=trainer,
mb_source=mbs,
model_inputs_to_streams=input_map,
mb_size=4,
max_samples=20)
session.train()
assert trainer.total_number_of_samples_seen == 20
def test_op_times_reduce_sequence_axis(device_id, precision):
dt_precision = PRECISION_TO_TYPE[precision]
from cntk import times, Value, TIMES_REDUCE_SEQUENCE_AXIS_WITHOUT_INFERRED_INPUT_RANK
from cntk import sequence
dim = 10
seq = [[0,1,2], [3], [4,5,6,7,8,9]]
right_data = Value.one_hot(seq, dim, dtype=dt_precision)
right_var = sequence.input(shape=(dim), is_sparse=True, dtype=dt_precision)
left_data = [AA([1,1,1],dtype=dt_precision), AA([1],dtype=dt_precision), AA([1,1,1,1,1,1],dtype=dt_precision)]
left_var = sequence.input(shape=(1), dtype=dt_precision)
func = times(left_var, right_var, infer_input_rank_to_map=TIMES_REDUCE_SEQUENCE_AXIS_WITHOUT_INFERRED_INPUT_RANK)
func2 = sequence.reduce_sum(times(left_var, right_var))
assert func.dynamic_axes == func2.dynamic_axes
_, forward_output = func.forward({left_var:left_data, right_var:right_data})
actual_forward = forward_output[func.output]
expected_forward = AA([[[1,1,1,0,0,0,0,0,0,0]],
[[0,0,0,1,0,0,0,0,0,0]],
[[0,0,0,0,1,1,1,1,1,1]]])
assert np.allclose(actual_forward, expected_forward)
def test_usermbsource_training(tmpdir, with_checkpoint_impl):
input_dim = 1000
num_output_classes = 5
mbs = MyDataSource(input_dim, num_output_classes)
# Using this for testing the UserMinibatchSource checkpointing
if with_checkpoint_impl:
MBS_CV_CLASS = MyDataSourceWithCheckpoint
else:
MBS_CV_CLASS = MyDataSource
mbs_cv = MBS_CV_CLASS(input_dim, num_output_classes)
from cntk import sequence, parameter, plus, cross_entropy_with_softmax, \
classification_error, learning_rate_schedule, sgd, Trainer, \
training_session, times, UnitType
feature = sequence.input_variable(shape=(input_dim,))
label = C.input_variable(shape=(num_output_classes,))
p = parameter(shape=(input_dim, num_output_classes), init=10)
z = times(sequence.reduce_sum(feature), p, name='z')
ce = cross_entropy_with_softmax(z, label)
errs = classification_error(z, label)
#having a large learning rate to prevent the model from converging earlier where not all the intended samples are fed
#note that training session can end earlier if there is no updates
lr_per_sample = learning_rate_schedule(0.3, UnitType.sample)
learner = sgd(z.parameters, lr_per_sample)
trainer = Trainer(z, (ce, errs), [learner])
input_map = {
feature: mbs.fsi,
label: mbs.lsi
}
session = training_session(
trainer=trainer, mb_source=mbs,
model_inputs_to_streams=input_map,
mb_size=4, max_samples=20,
cv_config = C.CrossValidationConfig(minibatch_source=mbs_cv, max_samples=10,
minibatch_size=2)
)
session.train()
assert trainer.total_number_of_samples_seen == 20
if with_checkpoint_impl:
assert mbs_cv._restore_from_checkpoint_calls == 1
def test_usermbsource_training(tmpdir, with_checkpoint_impl):
input_dim = 1000
num_output_classes = 5
mbs = MyDataSource(input_dim, num_output_classes)
# Using this for testing the UserMinibatchSource checkpointing
if with_checkpoint_impl:
MBS_CV_CLASS = MyDataSourceWithCheckpoint
else:
MBS_CV_CLASS = MyDataSource
mbs_cv = MBS_CV_CLASS(input_dim, num_output_classes)
from cntk import sequence, parameter, plus, cross_entropy_with_softmax, \
classification_error, learning_parameter_schedule_per_sample, sgd, Trainer, \
training_session, times
feature = sequence.input_variable(shape=(input_dim,))
label = C.input_variable(shape=(num_output_classes,))
p = parameter(shape=(input_dim, num_output_classes), init=10)
z = times(sequence.reduce_sum(feature), p, name='z')
ce = cross_entropy_with_softmax(z, label)
errs = classification_error(z, label)
#having a large learning rate to prevent the model from converging earlier where not all the intended samples are fed
#note that training session can end earlier if there is no updates
lr_per_sample = learning_parameter_schedule_per_sample(0.3)
learner = sgd(z.parameters, lr_per_sample)
trainer = Trainer(z, (ce, errs), [learner])
input_map = {
feature: mbs.fsi,
label: mbs.lsi
}
session = training_session(
trainer=trainer, mb_source=mbs,
model_inputs_to_streams=input_map,
mb_size=4, max_samples=20,
cv_config = C.CrossValidationConfig(minibatch_source=mbs_cv, max_samples=10,
minibatch_size=2)
)
session.train()
assert trainer.total_number_of_samples_seen == 20
if with_checkpoint_impl:
assert mbs_cv._restore_from_checkpoint_calls == 1
