def seq_hardmax(logits):
seq_max = C.layers.Fold(C.element_max,
initial_state=C.constant(-1e+30,
logits.shape))(logits)
s = C.equal(logits, C.sequence.broadcast_as(seq_max, logits))
s_acc = C.layers.Recurrence(C.plus)(s)
return s * C.equal(s_acc, 1) # only pick the first one
def seq_hardmax(logits):
# [#][dim=1]
seq_max = C.layers.Fold(C.element_max, initial_state=C.constant(-1e+30, logits.shape))(logits)
# [#,c][dim] 找到最大单词的位置
s = C.equal(logits, C.sequence.broadcast_as(seq_max, logits))
# [#,c][dim] 找到第一个出现的最大单词的位置
s_acc = C.layers.Recurrence(C.plus)(s)
# 除了最大单词为其logits外，其他都为0
return s * C.equal(s_acc, 1) # only pick the first one
def implementing_1d_convnet_cntk():
max_features = 10000 # number of words to consider as features
max_len = 500 # cut texts after this number of words (among top max_features most common words)
x_train, y_train, x_test, y_test = load_data(max_features, max_len)
model = build_model_cntk(max_features, max_len)
x = cntk.input_variable(shape=(max_len, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model.replace_placeholders({model.placeholders[0]: x})
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 10
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def learning_word_embeddings_with_the_embedding_layer_cntk():
x_train, y_train, x_test, y_test = load_from_files()
max_features = 10000
maxlen = 20
embedding_dim = 8
x = cntk.input_variable(shape=(maxlen, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model = cntk.one_hot(x, num_classes=max_features, sparse_output=True)
model = cntk.layers.Embedding(embedding_dim)(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 30
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def triangular_matrix_seq(mode: int = 1):
X = C.placeholder(1)
ones = C.ones_like(X[0])
perm_1 = C.layers.Recurrence(C.plus, return_full_state=True)(ones)
perm_2 = C.layers.Recurrence(C.plus,
go_backwards=True,
return_full_state=True)(ones)
arr_1 = C.sequence.unpack(perm_1, 0, True)
arr_2 = C.sequence.unpack(perm_2, 0, True)
mat = C.times_transpose(arr_1, arr_2)
mat_c = arr_1 * arr_2
diagonal_mat = mat - mat_c
final_mat = diagonal_mat
if mode == 0:
final_mat = C.equal(final_mat, 0)
elif mode == 1:
final_mat = C.less_equal(final_mat, 0)
elif mode == 2:
final_mat = C.less(final_mat, 0)
elif mode == -1:
final_mat = C.greater_equal(final_mat, 0)
elif mode == -2:
final_mat = C.greater(final_mat, 0)
result = C.as_block(final_mat, [(X, X)], 'triangular_matrix')
return C.stop_gradient(result)
def model(self):
c1_axis = C.Axis.new_unique_dynamic_axis('c1_axis')
c2_axis = C.Axis.new_unique_dynamic_axis('c2_axis')
b = C.Axis.default_batch_axis()
c1 = C.input_variable(self.word_dim,
dynamic_axes=[b, c1_axis],
name='c1')
c2 = C.input_variable(self.word_dim,
dynamic_axes=[b, c2_axis],
name='c2')
y = C.input_variable(1, dynamic_axes=[b], name='y')
c1_processed, c2_processed = self.input_layer(c1, c2).outputs
att_context = self.attention_layer(c2_processed, c1_processed,
'attention')
c2_len = C.layers.Fold(plus1)(c2_processed)
att_len = C.layers.Fold(plus1)(att_context)
cos = C.cosine_distance(
C.sequence.reduce_sum(c2_processed) / c2_len,
C.sequence.reduce_sum(att_context) / att_len)
prob = C.sigmoid(cos)
is_context = C.greater(prob, 0.5)
loss = C.losses.binary_cross_entropy(prob, y)
acc = C.equal(is_context, y)
return cos, loss, acc
def inner(a):
p = position(a)
quotient = p / s # every s sequence item will be an integer
decimals = quotient - C.floor(
quotient) # every s sequence item will be a zero
valid = C.equal(decimals, 0)
result = C.sequence.gather(a, valid)
return result
def equal(left, right, name=''):
'''
Elementwise 'equal' comparison of two tensors. Result is 1 if values are equal 0 otherwise.
Example:
>>> C.eval(C.equal([41., 42., 43.], [42., 42., 42.]))
[array([[0., 1., 0.]])]
>>> C.eval(C.eq([-1,0,1], [1]))
[array([[0., 1., 0.]])]
Args:
left: left side tensor
right: right side tensor
name (str): the name of the node in the network
Returns:
:class:`cntk.Function`
'''
from cntk import equal
left = sanitize_input(left, get_data_type(right))
right = sanitize_input(right, get_data_type(left))
return equal(left, right, name).output()
def equal(left, right, name=''):
'''
Elementwise 'equal' comparison of two tensors. Result is 1 if values are equal 0 otherwise.
Example:
>>> C.eval(C.equal([41., 42., 43.], [42., 42., 42.]))
[array([[0., 1., 0.]])]
>>> C.eval(C.eq([-1,0,1], [1]))
[array([[0., 1., 0.]])]
Args:
left: left side tensor
right: right side tensor
name (str): the name of the node in the network
Returns:
:class:`cntk.Function`
'''
from cntk import equal
left = sanitize_input(left, get_data_type(right))
right = sanitize_input(right, get_data_type(left))
return equal(left, right, name).output()
def use_glove_word_embeddings_cntk(preload_weights=False):
tokenizer, x_train, y_train, x_val, y_val = from_raw_text_to_word_embeddings(
)
x = cntk.input_variable(shape=(Constants.maxlen, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model = cntk.one_hot(x,
num_classes=Constants.max_words,
sparse_output=True)
if preload_weights is True:
embedding_matrix = compute_embedding_matrix(tokenizer)
assert (Constants.embedding_dim
== embedding_matrix.shape[0]) or (Constants.embedding_dim
== embedding_matrix.shape[1])
model = cntk.layers.Embedding(weights=embedding_matrix)(model)
else:
model = cntk.layers.Embedding(Constants.embedding_dim)(model)
model = cntk.layers.Dense(32, activation=cntk.relu)(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 10
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def test_Equal(tmpdir):
data0 = np.asarray([41., 42., 43.], dtype=np.float32)
data1 = np.asarray([42., 42., 42.], dtype=np.float32)
model = C.equal(data0, data1)
verify_no_input(model, tmpdir, 'Equal_0')
def test_Equal(tmpdir, dtype):
with C.default_options(dtype=dtype):
data0 = np.asarray([41., 42., 43.], dtype=dtype)
data1 = np.asarray([42., 42., 42.], dtype=dtype)
model = C.equal(data0, data1)
verify_no_input(model, tmpdir, 'Equal_0')
def test_Equal(tmpdir, dtype):
with C.default_options(dtype = dtype):
data0 = np.asarray([41., 42., 43.], dtype=dtype)
data1 = np.asarray([42., 42., 42.], dtype=dtype)
model = C.equal(data0, data1)
verify_no_input(model, tmpdir, 'Equal_0')
def test_Equal(tmpdir):
data0 = np.asarray([41., 42., 43.], dtype=np.float32)
data1 = np.asarray([42., 42., 42.], dtype=np.float32)
model = C.equal(data0, data1)
verify_no_input(model, tmpdir, 'Equal_0')
def seq_hardmax(logits):
seq_max = C.layers.Fold(C.element_max, initial_state=C.constant(-1e+30, logits.shape))(logits)
s = C.equal(logits, C.sequence.broadcast_as(seq_max, logits))
s_acc = C.layers.Recurrence(C.plus)(s)
return s * C.equal(s_acc, 1) # only pick the first one
def run_experiment_cntk():
if os.path.isfile('x_train_imdb.bin'):
print('Loading from .bin files')
x_train, y_train, x_test, y_test = load_from_files(x_shape=(25000,
500),
y_shape=(25000, ))
else:
print('Loading data...')
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(
num_words=Constants.max_words)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
print('Pad sequences (samples x time)')
x_train = keras.preprocessing.sequence.pad_sequences(
x_train, maxlen=Constants.maxlen)
x_test = keras.preprocessing.sequence.pad_sequences(
x_test, maxlen=Constants.maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Saving to .bin files')
save_to_files(x_train, y_train, x_test, y_test)
x = cntk.sequence.input_variable(shape=(), dtype=np.float32)
y = cntk.input_variable(shape=(), dtype=np.float32)
x_placeholder = cntk.placeholder(shape=(),
dynamic_axes=[
cntk.Axis.default_batch_axis(),
cntk.Axis.default_dynamic_axis()
])
model = cntk.one_hot(x_placeholder,
num_classes=Constants.max_words,
sparse_output=True)
model = cntk.layers.Embedding(Constants.embedding_dim)(model)
model = cntk.layers.Recurrence(cntk.layers.LSTM(32))(model)
model = cntk.sequence.last(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
model.save('ch6-2.cntk.model')
model = None
model = cntk.load_model('ch6-2.cntk.model')
model.replace_placeholders({model.placeholders[0]: x})
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements,
axis=cntk.Axis.all_static_axes())
max_epochs = 10
batch_size = 128
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.01),
cntk.learning_parameter_schedule_per_sample(0.9))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def inner(a):
return C.equal(C.reduce_max(a, axis=axis), a)
def inner(a):
perturbed = a + C.random.gumbel_like(a)
sampled = C.equal(C.reduce_max(perturbed, axis=axis),
perturbed,
name=name) # equivalent to hardmax(perturbed_x)
return sampled
import cntk
A = [1, 3, 4]
B = [4, 3, 2]
print("less(A,B):")
less = cntk.less(A, B).eval()
print("{}\n".format(less))
print("equal(A,B):")
equal = cntk.equal(A, B).eval()
print("{}\n".format(equal))
print("greater(A,B)")
greater = cntk.greater(A, B).eval()
print("{}\n".format(greater))
print("greater_equal(A,B):")
greater_equal = cntk.greater_equal(A, B).eval()
print("{}\n".format(greater_equal))
print("not_equal(A,B):")
not_equal = cntk.not_equal(A, B).eval()
print("{}\n".format(not_equal))
print("less_equal(A,B):")
less_equal = cntk.less_equal(A, B).eval()
print("{}\n".format(less_equal))
