def mnist_moment_estimation_encoder(data_dim, noise_dim, noise_basis_dim, latent_dim=8): noise_input = Input(shape=(noise_basis_dim, noise_dim,), name='enc_internal_noise_input') # compute the noise basis vectors by attaching small independent fully connected networks to each noise input from # noise_basis ones in total def get_inp_row(inputs, **kwargs): row = kwargs.get('i', 0) # first axis is the batch size, so skip it return inputs[:, row] noise_basis_vectors = [] for i in range(noise_basis_dim): l = Lambda(get_inp_row, arguments={'i': i}, name='enc_noise_basis_vec_select_{}'.format(i))(noise_input) fc = Dense(128, activation='relu', name='enc_noise_basis_body_0_basis_{}'.format(i))(l) fc = Dense(128, activation='relu', name='enc_noise_basis_body_1_basis_{}'.format(i))(fc) fc = Dense(128, activation='relu', name='enc_noise_basis_body_2_basis_{}'.format(i))(fc) fc = Dense(latent_dim, activation=None, name='enc_noise_basis_body_3_basis_{}'.format(i))(fc) noise_basis_vectors.append(fc) noise_basis_vectors_model = Model(inputs=noise_input, outputs=noise_basis_vectors, name='enc_noise_basis_vector_model') data_input = Input(shape=(data_dim,), name='enc_internal_data_input') assert data_dim == 28 ** 2, "MNIST data should be flattened to 784-dimensional vectors." # center the input around 0 centered_data = Lambda(lambda x: 2 * x - 1, name='enc_centering_data_input')(data_input) # compute the data embedding using deep convolutional neural network and reshape the output to the noise dim. convnet_input = Reshape((28, 28, 1), name='enc_data_reshape')(centered_data) # add noise to the convolutions # partial_noise = Reshape((-1,), name='enc_noise_addition_conv')(noise_input) coefficients = deflating_convolution(convnet_input, n_deflation_layers=3, n_filters_init=64, name_prefix='enc_data_body') coefficients = Reshape((-1,), name='enc_data_features_reshape')(coefficients) extracted_features = Dense(800, activation='relu', name='enc_expanding_before_latent')(coefficients) latent_0 = Dense(latent_dim, name='enc_coefficients')(extracted_features) coefficients = [] for i in range(noise_basis_dim): coefficients.append(Dense(latent_dim, name='enc_coefficients_{}'.format(i))(extracted_features)) coefficients.append(latent_0) coefficients_model = Model(inputs=data_input, outputs=coefficients, name='enc_coefficients_model') return coefficients_model, noise_basis_vectors_model
# C-LSTM main_input = Input(shape=(20, ), dtype='float64') embed = Embedding(len(vocab) + 1, 256, input_length=20)(main_input) cnn = Convolution1D(256, 3, padding='same', strides=1, activation='relu')(embed) new = Reshape(target_shape=(cnn.shape[2].value, cnn.shape[1].value))(cnn) # CNN-char # Reprocess the input # get vocab all_sent = [] for sent in title.tolist(): new = [] for word in sent: for char in word: new.append(word) new_sent = " ".join(new) all_sent.append(new_sent) tokenizer = Tokenizer(filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=" ") tokenizer.fit_on_texts(all_sent) vocab = tokenizer.word_index X_train, X_test, y_train, y_test = train_test_split(all_sent, label, test_size=0.1, random_state=42) X_train_word_ids = tokenizer.texts_to_sequences(X_train) X_test_word_ids = tokenizer.texts_to_sequences(X_test) X_train_padded_seqs = pad_sequences(X_train_word_ids, maxlen=30) X_test_padded_seqs = pad_sequences(X_test_word_ids, maxlen=30)
def binet_model_fn(dataset, latent_dim=None, use_attributes=None, use_present_activity=None, use_present_attributes=None, encode=None, decode=None, use_attention=None, sparse=False, postfix=''): from keras.models import Model from keras.layers import Input from keras.layers import Embedding from keras.layers import concatenate from keras.layers import GRU from keras.layers import BatchNormalization from keras.layers import Dense from keras.layers import Reshape from keras.optimizers import Adam from april.anomalydetection.binet.attention import Attention # Check for compatibility if use_attributes and dataset.num_attributes == 1: use_attributes = False if use_present_attributes and dataset.num_attributes == 2: use_present_attributes = False use_present_activity = True from keras import losses if sparse: targets = dataset.train_targets categorical_loss = losses.sparse_categorical_crossentropy else: targets = dataset.onehot_train_targets categorical_loss = losses.categorical_crossentropy # TODO: Tune mse_weight mse_weight = 0.5 loss_functions = {AttributeType.CATEGORICAL: categorical_loss, AttributeType.NUMERICAL: lambda y_t, y_p: mse_weight * losses.mean_squared_error(y_t, y_p)} loss_map = {} if not use_attributes: features = dataset.features[:1] targets = targets[:1] else: features = dataset.features if latent_dim is None: latent_dim = min(int(dataset.max_len * 2), 64) # clipping at 64 was not part of original paper # Build inputs (and encoders if enabled) for past events embeddings = [] inputs = [] past_outputs = [] for feature, attr_dim, attr_key, attr_type in \ zip(features, dataset.attribute_dims, dataset.attribute_keys, dataset.attribute_types): # Assign loss to attribute loss_map[attr_key] = loss_functions[attr_type] i = Input(shape=(None,), name=f'past_{attr_key}{postfix}') inputs.append(i) if attr_type != AttributeType.NUMERICAL: voc_size = int(attr_dim + 1) # we start at 1, hence plus 1 emb_size = np.clip(int(voc_size / 10), 2, 16) embedding = Embedding(input_dim=voc_size, output_dim=emb_size, input_length=feature.shape[1], mask_zero=True) embeddings.append(embedding) x = embedding(i) else: x = Reshape((feature.shape[1], 1))(i) if encode: x, _ = GRU(latent_dim, return_sequences=True, return_state=True, name=f'past_encoder_{attr_key}{postfix}')(x) x = BatchNormalization()(x) past_outputs.append(x) # Build inputs (and encoders if enabled) for present event present_features = [] present_outputs = [] if use_attributes and (use_present_activity or use_present_attributes): # Generate present features, by skipping the first event and adding one padding event at the end present_features = [np.pad(f[:, 1:], ((0, 0), (0, 1)), 'constant') for f in features] if use_attributes and not use_present_attributes: # Use only the activity features present_features = present_features[:1] for feature, embedding, attr_key, attr_type in \ zip(present_features, embeddings, dataset.attribute_keys, dataset.attribute_types): i = Input(shape=(None,), name=f'present_{attr_key}{postfix}') inputs.append(i) if attr_type != AttributeType.NUMERICAL: x = embedding(i) else: x = Reshape((feature.shape[1], 1))(i) if encode: x = GRU(latent_dim, return_sequences=True, name=f'present_encoder_{attr_key}{postfix}')(x) x = BatchNormalization()(x) present_outputs.append(x) # Build output layers for each attribute to predict outputs = [] for feature, attr_dim, attr_key, attr_type in \ zip(features, dataset.attribute_dims, dataset.attribute_keys, dataset.attribute_types): if attr_key == 'name' or not use_attributes or (not use_present_activity and not use_present_attributes): x = past_outputs # Else predict the attribute else: x = present_outputs[:1] if use_present_attributes: for past_o, present_o, at_key in zip(past_outputs[1:], present_outputs[1:], dataset.attribute_keys[1:]): if attr_key == at_key: x.append(past_o) else: x.append(present_o) else: x += past_outputs[1:] if use_attention: attentions = [] for _x, at_key in zip(x, dataset.attribute_keys): a, _ = Attention(return_sequences=True, return_coefficients=True, name=f'attention_{attr_key}/{at_key}{postfix}')(_x) attentions.append(a) x = attentions if len(x) > 1: x = concatenate(x) else: x = x[0] if decode: x = GRU(latent_dim, return_sequences=True, name=f'decoder_{attr_key}{postfix}')(x) x = BatchNormalization()(x) activation = 'softmax' if attr_type != AttributeType.NUMERICAL else 'linear' o = Dense(int(attr_dim), activation=activation, name=f'out_{attr_key}{postfix}')(x) outputs.append(o) # Combine features and build model features = features + present_features model = Model(inputs=inputs, outputs=outputs) # Define custom loss def custom_loss(y_true, y_pred): attr_name = y_pred.name.split('_')[1].split('/')[0] return loss_map[attr_name](y_true, y_pred) # Register it as custom object, so it may be loaded afterwards in conjunction with the model from keras.utils.generic_utils import get_custom_objects get_custom_objects().update({"custom_loss": custom_loss}) # Compile model model.compile( optimizer=Adam(), loss='custom_loss' ) return model, features, targets