def seq_hardmax(logits): seq_max = C.layers.Fold(C.element_max, initial_state=C.constant(-1e+30, logits.shape))(logits) s = C.equal(logits, C.sequence.broadcast_as(seq_max, logits)) s_acc = C.layers.Recurrence(C.plus)(s) return s * C.equal(s_acc, 1) # only pick the first one
def seq_hardmax(logits): # [#][dim=1] seq_max = C.layers.Fold(C.element_max, initial_state=C.constant(-1e+30, logits.shape))(logits) # [#,c][dim] 找到最大单词的位置 s = C.equal(logits, C.sequence.broadcast_as(seq_max, logits)) # [#,c][dim] 找到第一个出现的最大单词的位置 s_acc = C.layers.Recurrence(C.plus)(s) # 除了最大单词为其logits外,其他都为0 return s * C.equal(s_acc, 1) # only pick the first one
def implementing_1d_convnet_cntk(): max_features = 10000 # number of words to consider as features max_len = 500 # cut texts after this number of words (among top max_features most common words) x_train, y_train, x_test, y_test = load_data(max_features, max_len) model = build_model_cntk(max_features, max_len) x = cntk.input_variable(shape=(max_len, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model.replace_placeholders({model.placeholders[0]: x}) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 10 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def learning_word_embeddings_with_the_embedding_layer_cntk(): x_train, y_train, x_test, y_test = load_from_files() max_features = 10000 maxlen = 20 embedding_dim = 8 x = cntk.input_variable(shape=(maxlen, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model = cntk.one_hot(x, num_classes=max_features, sparse_output=True) model = cntk.layers.Embedding(embedding_dim)(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 30 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def triangular_matrix_seq(mode: int = 1): X = C.placeholder(1) ones = C.ones_like(X[0]) perm_1 = C.layers.Recurrence(C.plus, return_full_state=True)(ones) perm_2 = C.layers.Recurrence(C.plus, go_backwards=True, return_full_state=True)(ones) arr_1 = C.sequence.unpack(perm_1, 0, True) arr_2 = C.sequence.unpack(perm_2, 0, True) mat = C.times_transpose(arr_1, arr_2) mat_c = arr_1 * arr_2 diagonal_mat = mat - mat_c final_mat = diagonal_mat if mode == 0: final_mat = C.equal(final_mat, 0) elif mode == 1: final_mat = C.less_equal(final_mat, 0) elif mode == 2: final_mat = C.less(final_mat, 0) elif mode == -1: final_mat = C.greater_equal(final_mat, 0) elif mode == -2: final_mat = C.greater(final_mat, 0) result = C.as_block(final_mat, [(X, X)], 'triangular_matrix') return C.stop_gradient(result)
def model(self): c1_axis = C.Axis.new_unique_dynamic_axis('c1_axis') c2_axis = C.Axis.new_unique_dynamic_axis('c2_axis') b = C.Axis.default_batch_axis() c1 = C.input_variable(self.word_dim, dynamic_axes=[b, c1_axis], name='c1') c2 = C.input_variable(self.word_dim, dynamic_axes=[b, c2_axis], name='c2') y = C.input_variable(1, dynamic_axes=[b], name='y') c1_processed, c2_processed = self.input_layer(c1, c2).outputs att_context = self.attention_layer(c2_processed, c1_processed, 'attention') c2_len = C.layers.Fold(plus1)(c2_processed) att_len = C.layers.Fold(plus1)(att_context) cos = C.cosine_distance( C.sequence.reduce_sum(c2_processed) / c2_len, C.sequence.reduce_sum(att_context) / att_len) prob = C.sigmoid(cos) is_context = C.greater(prob, 0.5) loss = C.losses.binary_cross_entropy(prob, y) acc = C.equal(is_context, y) return cos, loss, acc
def inner(a): p = position(a) quotient = p / s # every s sequence item will be an integer decimals = quotient - C.floor( quotient) # every s sequence item will be a zero valid = C.equal(decimals, 0) result = C.sequence.gather(a, valid) return result
def equal(left, right, name=''): ''' Elementwise 'equal' comparison of two tensors. Result is 1 if values are equal 0 otherwise. Example: >>> C.eval(C.equal([41., 42., 43.], [42., 42., 42.])) [array([[0., 1., 0.]])] >>> C.eval(C.eq([-1,0,1], [1])) [array([[0., 1., 0.]])] Args: left: left side tensor right: right side tensor name (str): the name of the node in the network Returns: :class:`cntk.Function` ''' from cntk import equal left = sanitize_input(left, get_data_type(right)) right = sanitize_input(right, get_data_type(left)) return equal(left, right, name).output()
def use_glove_word_embeddings_cntk(preload_weights=False): tokenizer, x_train, y_train, x_val, y_val = from_raw_text_to_word_embeddings( ) x = cntk.input_variable(shape=(Constants.maxlen, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model = cntk.one_hot(x, num_classes=Constants.max_words, sparse_output=True) if preload_weights is True: embedding_matrix = compute_embedding_matrix(tokenizer) assert (Constants.embedding_dim == embedding_matrix.shape[0]) or (Constants.embedding_dim == embedding_matrix.shape[1]) model = cntk.layers.Embedding(weights=embedding_matrix)(model) else: model = cntk.layers.Embedding(Constants.embedding_dim)(model) model = cntk.layers.Dense(32, activation=cntk.relu)(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 10 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def test_Equal(tmpdir): data0 = np.asarray([41., 42., 43.], dtype=np.float32) data1 = np.asarray([42., 42., 42.], dtype=np.float32) model = C.equal(data0, data1) verify_no_input(model, tmpdir, 'Equal_0')
def test_Equal(tmpdir, dtype): with C.default_options(dtype=dtype): data0 = np.asarray([41., 42., 43.], dtype=dtype) data1 = np.asarray([42., 42., 42.], dtype=dtype) model = C.equal(data0, data1) verify_no_input(model, tmpdir, 'Equal_0')
def test_Equal(tmpdir, dtype): with C.default_options(dtype = dtype): data0 = np.asarray([41., 42., 43.], dtype=dtype) data1 = np.asarray([42., 42., 42.], dtype=dtype) model = C.equal(data0, data1) verify_no_input(model, tmpdir, 'Equal_0')
def run_experiment_cntk(): if os.path.isfile('x_train_imdb.bin'): print('Loading from .bin files') x_train, y_train, x_test, y_test = load_from_files(x_shape=(25000, 500), y_shape=(25000, )) else: print('Loading data...') (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data( num_words=Constants.max_words) print(len(x_train), 'train sequences') print(len(x_test), 'test sequences') print('Pad sequences (samples x time)') x_train = keras.preprocessing.sequence.pad_sequences( x_train, maxlen=Constants.maxlen) x_test = keras.preprocessing.sequence.pad_sequences( x_test, maxlen=Constants.maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('Saving to .bin files') save_to_files(x_train, y_train, x_test, y_test) x = cntk.sequence.input_variable(shape=(), dtype=np.float32) y = cntk.input_variable(shape=(), dtype=np.float32) x_placeholder = cntk.placeholder(shape=(), dynamic_axes=[ cntk.Axis.default_batch_axis(), cntk.Axis.default_dynamic_axis() ]) model = cntk.one_hot(x_placeholder, num_classes=Constants.max_words, sparse_output=True) model = cntk.layers.Embedding(Constants.embedding_dim)(model) model = cntk.layers.Recurrence(cntk.layers.LSTM(32))(model) model = cntk.sequence.last(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) model.save('ch6-2.cntk.model') model = None model = cntk.load_model('ch6-2.cntk.model') model.replace_placeholders({model.placeholders[0]: x}) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=cntk.Axis.all_static_axes()) max_epochs = 10 batch_size = 128 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.01), cntk.learning_parameter_schedule_per_sample(0.9)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def inner(a): return C.equal(C.reduce_max(a, axis=axis), a)
def inner(a): perturbed = a + C.random.gumbel_like(a) sampled = C.equal(C.reduce_max(perturbed, axis=axis), perturbed, name=name) # equivalent to hardmax(perturbed_x) return sampled
import cntk A = [1, 3, 4] B = [4, 3, 2] print("less(A,B):") less = cntk.less(A, B).eval() print("{}\n".format(less)) print("equal(A,B):") equal = cntk.equal(A, B).eval() print("{}\n".format(equal)) print("greater(A,B)") greater = cntk.greater(A, B).eval() print("{}\n".format(greater)) print("greater_equal(A,B):") greater_equal = cntk.greater_equal(A, B).eval() print("{}\n".format(greater_equal)) print("not_equal(A,B):") not_equal = cntk.not_equal(A, B).eval() print("{}\n".format(not_equal)) print("less_equal(A,B):") less_equal = cntk.less_equal(A, B).eval() print("{}\n".format(less_equal))