def mnist_moment_estimation_encoder(data_dim, noise_dim, noise_basis_dim, latent_dim=8):
noise_input = Input(shape=(noise_basis_dim, noise_dim,), name='enc_internal_noise_input')
# compute the noise basis vectors by attaching small independent fully connected networks to each noise input from
# noise_basis ones in total
def get_inp_row(inputs, **kwargs):
row = kwargs.get('i', 0)
# first axis is the batch size, so skip it
return inputs[:, row]
noise_basis_vectors = []
for i in range(noise_basis_dim):
l = Lambda(get_inp_row, arguments={'i': i}, name='enc_noise_basis_vec_select_{}'.format(i))(noise_input)
fc = Dense(128, activation='relu', name='enc_noise_basis_body_0_basis_{}'.format(i))(l)
fc = Dense(128, activation='relu', name='enc_noise_basis_body_1_basis_{}'.format(i))(fc)
fc = Dense(128, activation='relu', name='enc_noise_basis_body_2_basis_{}'.format(i))(fc)
fc = Dense(latent_dim, activation=None, name='enc_noise_basis_body_3_basis_{}'.format(i))(fc)
noise_basis_vectors.append(fc)
noise_basis_vectors_model = Model(inputs=noise_input, outputs=noise_basis_vectors,
name='enc_noise_basis_vector_model')
data_input = Input(shape=(data_dim,), name='enc_internal_data_input')
assert data_dim == 28 ** 2, "MNIST data should be flattened to 784-dimensional vectors."
# center the input around 0
centered_data = Lambda(lambda x: 2 * x - 1, name='enc_centering_data_input')(data_input)
# compute the data embedding using deep convolutional neural network and reshape the output to the noise dim.
convnet_input = Reshape((28, 28, 1), name='enc_data_reshape')(centered_data)
# add noise to the convolutions
# partial_noise = Reshape((-1,), name='enc_noise_addition_conv')(noise_input)
coefficients = deflating_convolution(convnet_input, n_deflation_layers=3,
n_filters_init=64, name_prefix='enc_data_body')
coefficients = Reshape((-1,), name='enc_data_features_reshape')(coefficients)
extracted_features = Dense(800, activation='relu', name='enc_expanding_before_latent')(coefficients)
latent_0 = Dense(latent_dim, name='enc_coefficients')(extracted_features)
coefficients = []
for i in range(noise_basis_dim):
coefficients.append(Dense(latent_dim, name='enc_coefficients_{}'.format(i))(extracted_features))
coefficients.append(latent_0)
coefficients_model = Model(inputs=data_input, outputs=coefficients, name='enc_coefficients_model')
return coefficients_model, noise_basis_vectors_model
# C-LSTM
main_input = Input(shape=(20, ), dtype='float64')
embed = Embedding(len(vocab) + 1, 256, input_length=20)(main_input)
cnn = Convolution1D(256, 3, padding='same', strides=1,
activation='relu')(embed)
new = Reshape(target_shape=(cnn.shape[2].value, cnn.shape[1].value))(cnn)
# CNN-char
# Reprocess the input
# get vocab
all_sent = []
for sent in title.tolist():
new = []
for word in sent:
for char in word:
new.append(word)
new_sent = " ".join(new)
all_sent.append(new_sent)
tokenizer = Tokenizer(filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
lower=True,
split=" ")
tokenizer.fit_on_texts(all_sent)
vocab = tokenizer.word_index
X_train, X_test, y_train, y_test = train_test_split(all_sent,
label,
test_size=0.1,
random_state=42)
X_train_word_ids = tokenizer.texts_to_sequences(X_train)
X_test_word_ids = tokenizer.texts_to_sequences(X_test)
X_train_padded_seqs = pad_sequences(X_train_word_ids, maxlen=30)
X_test_padded_seqs = pad_sequences(X_test_word_ids, maxlen=30)
def binet_model_fn(dataset,
latent_dim=None,
use_attributes=None,
use_present_activity=None,
use_present_attributes=None,
encode=None,
decode=None,
use_attention=None,
sparse=False,
postfix=''):
from keras.models import Model
from keras.layers import Input
from keras.layers import Embedding
from keras.layers import concatenate
from keras.layers import GRU
from keras.layers import BatchNormalization
from keras.layers import Dense
from keras.layers import Reshape
from keras.optimizers import Adam
from april.anomalydetection.binet.attention import Attention
# Check for compatibility
if use_attributes and dataset.num_attributes == 1:
use_attributes = False
if use_present_attributes and dataset.num_attributes == 2:
use_present_attributes = False
use_present_activity = True
from keras import losses
if sparse:
targets = dataset.train_targets
categorical_loss = losses.sparse_categorical_crossentropy
else:
targets = dataset.onehot_train_targets
categorical_loss = losses.categorical_crossentropy
# TODO: Tune mse_weight
mse_weight = 0.5
loss_functions = {AttributeType.CATEGORICAL: categorical_loss, AttributeType.NUMERICAL:
lambda y_t, y_p: mse_weight * losses.mean_squared_error(y_t, y_p)}
loss_map = {}
if not use_attributes:
features = dataset.features[:1]
targets = targets[:1]
else:
features = dataset.features
if latent_dim is None:
latent_dim = min(int(dataset.max_len * 2), 64) # clipping at 64 was not part of original paper
# Build inputs (and encoders if enabled) for past events
embeddings = []
inputs = []
past_outputs = []
for feature, attr_dim, attr_key, attr_type in \
zip(features, dataset.attribute_dims, dataset.attribute_keys, dataset.attribute_types):
# Assign loss to attribute
loss_map[attr_key] = loss_functions[attr_type]
i = Input(shape=(None,), name=f'past_{attr_key}{postfix}')
inputs.append(i)
if attr_type != AttributeType.NUMERICAL:
voc_size = int(attr_dim + 1) # we start at 1, hence plus 1
emb_size = np.clip(int(voc_size / 10), 2, 16)
embedding = Embedding(input_dim=voc_size,
output_dim=emb_size,
input_length=feature.shape[1],
mask_zero=True)
embeddings.append(embedding)
x = embedding(i)
else:
x = Reshape((feature.shape[1], 1))(i)
if encode:
x, _ = GRU(latent_dim,
return_sequences=True,
return_state=True,
name=f'past_encoder_{attr_key}{postfix}')(x)
x = BatchNormalization()(x)
past_outputs.append(x)
# Build inputs (and encoders if enabled) for present event
present_features = []
present_outputs = []
if use_attributes and (use_present_activity or use_present_attributes):
# Generate present features, by skipping the first event and adding one padding event at the end
present_features = [np.pad(f[:, 1:], ((0, 0), (0, 1)), 'constant') for f in features]
if use_attributes and not use_present_attributes:
# Use only the activity features
present_features = present_features[:1]
for feature, embedding, attr_key, attr_type in \
zip(present_features, embeddings, dataset.attribute_keys, dataset.attribute_types):
i = Input(shape=(None,), name=f'present_{attr_key}{postfix}')
inputs.append(i)
if attr_type != AttributeType.NUMERICAL:
x = embedding(i)
else:
x = Reshape((feature.shape[1], 1))(i)
if encode:
x = GRU(latent_dim,
return_sequences=True,
name=f'present_encoder_{attr_key}{postfix}')(x)
x = BatchNormalization()(x)
present_outputs.append(x)
# Build output layers for each attribute to predict
outputs = []
for feature, attr_dim, attr_key, attr_type in \
zip(features, dataset.attribute_dims, dataset.attribute_keys, dataset.attribute_types):
if attr_key == 'name' or not use_attributes or (not use_present_activity and not use_present_attributes):
x = past_outputs
# Else predict the attribute
else:
x = present_outputs[:1]
if use_present_attributes:
for past_o, present_o, at_key in zip(past_outputs[1:], present_outputs[1:], dataset.attribute_keys[1:]):
if attr_key == at_key:
x.append(past_o)
else:
x.append(present_o)
else:
x += past_outputs[1:]
if use_attention:
attentions = []
for _x, at_key in zip(x, dataset.attribute_keys):
a, _ = Attention(return_sequences=True,
return_coefficients=True,
name=f'attention_{attr_key}/{at_key}{postfix}')(_x)
attentions.append(a)
x = attentions
if len(x) > 1:
x = concatenate(x)
else:
x = x[0]
if decode:
x = GRU(latent_dim,
return_sequences=True,
name=f'decoder_{attr_key}{postfix}')(x)
x = BatchNormalization()(x)
activation = 'softmax' if attr_type != AttributeType.NUMERICAL else 'linear'
o = Dense(int(attr_dim), activation=activation, name=f'out_{attr_key}{postfix}')(x)
outputs.append(o)
# Combine features and build model
features = features + present_features
model = Model(inputs=inputs, outputs=outputs)
# Define custom loss
def custom_loss(y_true, y_pred):
attr_name = y_pred.name.split('_')[1].split('/')[0]
return loss_map[attr_name](y_true, y_pred)
# Register it as custom object, so it may be loaded afterwards in conjunction with the model
from keras.utils.generic_utils import get_custom_objects
get_custom_objects().update({"custom_loss": custom_loss})
# Compile model
model.compile(
optimizer=Adam(),
loss='custom_loss'
)
return model, features, targets
