def create_model(X_train, y_train, X_valid, y_valid, X_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
dp = {{uniform(0, 0.5)}}
N_CLASSES = y_train.shape[1]
nb_features = X_train.shape[1]
sequence_input = tf.keras.layers.Input(shape=nb_features, dtype='float32')
dense2 = tf.keras.layers.Dense(64, activation='relu')(
sequence_input) # Does using 2*32 layers make sense ?
drop2 = tf.keras.layers.Dropout(dp)(dense2)
dense3 = tf.keras.layers.Dense(32, activation='relu')(drop2)
drop3 = tf.keras.layers.Dropout(dp)(dense3)
dense4 = tf.keras.layers.Dense(16, activation='relu')(drop3)
drop4 = tf.keras.layers.Dropout(dp)(dense4)
predictions = tf.keras.layers.Dense(N_CLASSES, activation='softmax')(drop4)
model = tf.keras.Model(sequence_input, predictions)
#==================================================
# Specifying the optimizer
#==================================================
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)
check = ModelCheckpoint(filepath='w_cbf_hyperopt.hdf5',\
verbose = 1, save_best_only=True)
optim_ch = {{choice(['adam', 'ranger'])}}
lr = {{uniform(1e-3, 1e-2)}}
if optim_ch == 'adam':
optim = tf.keras.optimizers.Adam(lr=lr)
else:
sync_period = {{choice([2, 6, 10])}}
slow_step_size = {{normal(0.5, 0.1)}}
rad = RectifiedAdam(lr=lr)
optim = Lookahead(rad,
sync_period=sync_period,
slow_step_size=slow_step_size)
batch_size = {{choice([64 * 4, 64 * 8])}}
STEP_SIZE_TRAIN = (len(X_train) // batch_size) + 1
STEP_SIZE_VALID = 1
beta = {{choice([0.9, 0.99, 0.999, 0.9999, 0.99993])}}
gamma = {{uniform(1, 2.2)}}
print('gamma value is :', gamma)
sample_per_class = np.sum(y_train, axis=0)
model.compile(loss=[CB_loss(sample_per_class, beta=beta, gamma=gamma)],
metrics=['accuracy'],
optimizer=optim)
result = model.fit(X_train, y_train, validation_data=(X_valid, y_valid), \
steps_per_epoch = STEP_SIZE_TRAIN, validation_steps = STEP_SIZE_VALID,\
epochs = 60, shuffle=True, verbose=2, callbacks = [check, es])
#Get the highest validation accuracy of the training epochs
loss_acc = np.amin(result.history['val_loss'])
print('Min loss of epoch:', loss_acc)
model.load_weights('w_cbf_hyperopt.hdf5')
return {'loss': loss_acc, 'status': STATUS_OK, 'model': model}
def model(train_x, train_y, dev_x, dev_y, test_x, test_y, dev_y_org,
test_y_org, overal_maxlen):
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras import optimizers
import keras.backend as K
import pickle as pk
import numpy as np
from deepats.optimizers import get_optimizer
from deepats.asap_evaluator import Evaluator
import deepats.asap_reader as dataset
from deepats.config import get_args
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
import string
import random
def random_id(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
import time
ms = int(round(time.time() * 1000))
rand_seed = ms % (2**32 - 1)
random.seed(rand_seed)
args = get_args()
model_id = random_id()
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def kappa_loss(t, x):
u = K.sum(K.square(x - t))
v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1))
return u / (2 * v + u)
lr = {{lognormal(-3 * 2.3, .8)}}
lr = lr * 2
rho = {{normal(.875, .04)}}
clipnorm = {{uniform(1, 15)}}
eps = 1e-6
opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps)
loss = kappa_loss
metric = kappa_metric
dataset.set_score_range(args.data_set)
evl = Evaluator(dataset,
args.prompt_id,
args.abs_out_path,
dev_x,
test_x,
dev_y,
test_y,
dev_y_org,
test_y_org,
model_id=model_id)
abs_vocab_file = args.abs_out_path + '/vocab.pkl'
with open(abs_vocab_file, 'rb') as vocab_file:
vocab = pk.load(vocab_file)
train_y_mean = train_y.mean(axis=0)
if train_y_mean.ndim == 0:
train_y_mean = np.expand_dims(train_y_mean, axis=1)
num_outputs = len(train_y_mean)
mask_zero = False
emb_dim = {{choice([50, 100, 200, 300])}}
rnn_dim = {{uniform(50, 300)}}
rnn_dim = int(rnn_dim)
model = Sequential()
model.add(Embedding(args.vocab_size, emb_dim, mask_zero=mask_zero))
model.add(RWA(rnn_dim))
model.add(Dense(num_outputs))
if not args.skip_init_bias:
bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype(
K.floatx())
model.layers[-1].bias.set_value(bias_value)
model.add(Activation('tanh'))
model.emb_index = 0
emb_path = 'embed/glove.6B.{}d.txt'.format(emb_dim)
abs_emb_path = args.abs_root + emb_path
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(abs_emb_path, emb_dim=emb_dim)
emb_reader.load_embeddings(vocab)
emb_wts = emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()[0])
wts = model.layers[model.emb_index].get_weights()
wts[0] = emb_wts
model.layers[model.emb_index].set_weights(wts)
model.compile(loss=loss, optimizer=opt, metrics=[metric])
model_yaml = model.to_yaml()
print('model_id: %s' % (model_id))
print(model_yaml)
print('optimizer: lr= %.4f, rho= %.4f, clipnorm= %.4f, epsilon= %.4f' %
(lr, rho, clipnorm, eps))
print('PARAMS\t\
%s\t\
lr= %.4f\t\
rho= %.4f\t\
clip= %.4f\t\
emb= %.4f\t\
rnn= %.4f' % (model_id, lr, rho, clipnorm, emb_dim, rnn_dim))
for i in range(args.epochs):
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
evl.evaluate(model, i)
evl.output_info()
if i > 5 and evl.dev_metric < 0.4:
break
if i > 10 and evl.dev_metric < 0.5:
break
if i > 15 and evl.dev_metric < 0.6:
break
best_dev_kappa = evl.best_dev
best_test_kappa = evl.best_test
print('Test kappa:', best_dev_kappa)
return {
'loss': 1 - best_dev_kappa,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pk.dumps(model.get_weights())
}
def create_model(X_train, y_train, X_valid, y_valid, X_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
dp = {{uniform(0, 0.5)}}
N_CLASSES = y_train.shape[1]
max_len = X_train.shape[1]
nb_curves = X_train.shape[2]
sequence_input = tf.keras.layers.Input(shape=(max_len, nb_curves),
dtype='float32')
# A 1D convolution with 128 output channels: Extract features from the curves
x = tf.keras.layers.Conv1D(64, 5, activation='relu')(sequence_input)
x = tf.keras.layers.Conv1D(32, 5, activation='relu')(x)
x = tf.keras.layers.Conv1D(16, 5, activation='relu')(x)
# Average those features
average = tf.keras.layers.GlobalAveragePooling1D()(x)
dense2 = tf.keras.layers.Dense(32, activation='relu')(
average) # Does using 2*32 layers make sense ?
drop2 = tf.keras.layers.Dropout(dp)(dense2)
dense3 = tf.keras.layers.Dense(32, activation='relu')(drop2)
drop3 = tf.keras.layers.Dropout(dp)(dense3)
dense4 = tf.keras.layers.Dense(16, activation='relu')(drop3)
drop4 = tf.keras.layers.Dropout(dp)(dense4)
predictions = tf.keras.layers.Dense(N_CLASSES, activation='softmax')(drop4)
model = tf.keras.Model(sequence_input, predictions)
#==================================================
# Specifying the optimizer
#==================================================
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)
check = ModelCheckpoint(filepath='w_categ_hyperopt.hdf5',\
verbose = 1, save_best_only=True)
optim_ch = {{choice(['adam', 'ranger'])}}
lr = {{uniform(1e-3, 1e-2)}}
if optim_ch == 'adam':
optim = tf.keras.optimizers.Adam(lr=lr)
else:
sync_period = {{choice([2, 6, 10])}}
slow_step_size = {{normal(0.5, 0.1)}}
rad = RectifiedAdam(lr=lr)
optim = Lookahead(rad,
sync_period=sync_period,
slow_step_size=slow_step_size)
# Defining the weights: Take the average over SSLAMM data
weights = {{choice(['regular', 'sqrt'])}}
if weights == 'regular':
w = 1 / np.sum(y_train, axis=0)
w = w / w.sum()
else:
w = 1 / np.sqrt(np.sum(y_train, axis=0))
w = w / w.sum()
w = dict(zip(range(N_CLASSES), w))
batch_size = {{choice([64 * 4, 64 * 8])}}
STEP_SIZE_TRAIN = (len(X_train) // batch_size) + 1
STEP_SIZE_VALID = 1
model.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=optim)
result = model.fit(X_train, y_train, validation_data=(X_valid, y_valid), \
steps_per_epoch = STEP_SIZE_TRAIN, validation_steps = STEP_SIZE_VALID,\
epochs = 60, class_weight = w, shuffle=True, verbose=2, callbacks = [check, es])
#Get the highest validation accuracy of the training epochs
loss_acc = np.amin(result.history['val_loss'])
print('Min loss of epoch:', loss_acc)
model.load_weights('w_categ_hyperopt.hdf5')
return {'loss': loss_acc, 'status': STATUS_OK, 'model': model}
def model(train_x, train_y, dev_x, dev_y, test_x, test_y, overal_maxlen, qwks):
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.initializers import Constant
from keras import optimizers
import keras.backend as K
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
import pickle as pk
import numpy as np
import string
import random
import os
from deepats.optimizers import get_optimizer
from deepats.ets_evaluator import Evaluator
import deepats.ets_reader as dataset
from deepats.ets_config import get_args
import GPUtil
def random_id(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def kappa_loss(t, x):
u = K.sum(K.square(x - t))
v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1))
return u / (2 * v + u)
import time
ms = int(round(time.time() * 1000))
rand_seed = ms % (2**32 - 1)
random.seed(rand_seed)
args = get_args()
model_id = random_id()
abs_vocab_file = os.path.join(args.abs_out, 'vocab.pkl')
with open(abs_vocab_file, 'rb') as vocab_file:
vocab = pk.load(vocab_file)
vocab_size = len(vocab)
acts = ['tanh', 'relu', 'hard_sigmoid']
emb_dim = {{choice([50, 100, 200, 300])}}
rnn_dim = {{uniform(50, 500)}}
rnn_dim = int(rnn_dim)
rec_act = {{choice([0, 1, 2])}}
rec_act = acts[rec_act]
dropout = {{uniform(0.2, 0.95)}}
epochs = args.epochs
n_emb = vocab_size * emb_dim
n_rwa = (903 + 2 * rnn_dim) * rnn_dim
n_tot = n_emb + n_rwa + rnn_dim + 1
lr = {{lognormal(-3 * 2.3, .8)}}
lr = 1.5 * lr
rho = {{normal(.875, .04)}}
clipnorm = {{uniform(1, 15)}}
eps = {{loguniform(-8 * 2.3, -5 * 2.3)}}
opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps)
loss = kappa_loss
metric = kappa_metric
evl = Evaluator(dataset,
args.prompt_id,
args.abs_out,
dev_x,
test_x,
dev_df,
test_df,
model_id=model_id)
train_y_mean = train_y.mean(axis=0)
if train_y_mean.ndim == 0:
train_y_mean = np.expand_dims(train_y_mean, axis=1)
num_outputs = len(train_y_mean)
mask_zero = False
model = Sequential()
model.add(Embedding(vocab_size, emb_dim, mask_zero=mask_zero))
model.add(RWA(rnn_dim, recurrent_activation=rec_act))
model.add(Dropout(dropout))
bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype(
K.floatx())
model.add(Dense(num_outputs, bias_initializer=Constant(value=bias_value)))
model.add(Activation('tanh'))
model.emb_index = 0
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim)
emb_reader.load_embeddings(vocab)
emb_wts = emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()[0])
wts = model.layers[model.emb_index].get_weights()
wts[0] = emb_wts
model.layers[model.emb_index].set_weights(wts)
model.compile(loss=loss, optimizer=opt, metrics=[metric])
model_yaml = model.to_yaml()
import GPUtil
if GPUtil.avail_mem() < 0.1:
return {'loss': 1, 'status': STATUS_OK, 'model': '', 'weights': None}
print('model_id: %s' % (model_id))
print(model_yaml)
print('PARAMS\t\
%s\t\
lr= %.4f\t\
rho= %.4f\t\
clip= %.4f\t\
eps= %.4f\t\
embDim= %.4f\t\
rnnDim= %.4f\t\
drop= %.4f\t\
recAct= %s' % (model_id, lr, rho, clipnorm, np.log(eps) / 2.3, emb_dim,
rnn_dim, dropout, rec_act))
for i in range(epochs):
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
evl.evaluate(model, i)
evl.output_info()
p = evl.stats[3] / qwks[0]
if i > 10 and p < 0.9:
break
i = evl.comp_idx
j = i + 2
best_dev_kappa = evl.best_dev[i]
best_test_kappa = evl.best_dev[j]
print('Test kappa:', best_dev_kappa)
return {
'loss': 1 - best_dev_kappa,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pk.dumps(model.get_weights())
}
def split_model_1(x_train, x_test, y_train, y_test, param_list):
np.random.seed(234)
l2_val = l2({{normal(param_list.L2_VAL['mu'], param_list.L2_VAL['std'])}})
image_input = Input(shape=(param_list.input_shape[0],
param_list.input_shape[1] - 2, 1),
dtype='float32')
x = Reshape(target_shape=(param_list.input_shape[0],
param_list.input_shape[1] - 2))(image_input)
x = Conv1D({{choice(param_list.IMG_CONV_FILTERS)}},
{{choice(param_list.IMG_CONV_SIZE)}},
padding='same',
activation='relu',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}},
kernel_regularizer=l2_val)(x)
x = MaxPooling1D(2)(x)
x = Conv1D({{choice(param_list.IMG_CONV_FILTERS)}},
{{choice(param_list.IMG_CONV_SIZE)}},
padding='same',
activation='relu',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}},
kernel_regularizer=l2_val)(x)
image_x = Flatten()(MaxPooling1D(2)(x))
ir_input = Input(shape=(param_list.input_shape[0], 2, 1), dtype='float32')
x = Reshape(target_shape=(param_list.input_shape[0], 2))(ir_input)
x = Conv1D(2,
3,
padding='same',
activation='relu',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}},
kernel_regularizer=l2_val)(x)
x = MaxPooling1D(2)(x)
x = Conv1D(2,
3,
padding='same',
activation='relu',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}},
kernel_regularizer=l2_val)(x)
ir_x = Flatten()(MaxPooling1D(2)(x))
x = concatenate([image_x, ir_x])
x = Dense({{choice(param_list.HIDDEN_UNITS)}},
activation='relu',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}},
kernel_regularizer=l2_val)(x)
preds = Dense(param_list.num_classes,
activation='softmax',
kernel_initializer={{choice(param_list.KERNEL_INITIALIZER)}
})(x)
model = Model([image_input, ir_input], preds)
rmsprop = optimizers.rmsprop(lr={{choice(param_list.LR_VAL)}})
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop,
metrics=['acc'])
model.fit_generator(
create_generator([x_train[:, :, 0:-2, :], x_train[:, :, -2:, :]],
y_train,
batch_size=param_list.BATCH_SIZE),
steps_per_epoch=int(len(x_train) / param_list.BATCH_SIZE),
epochs={{choice(param_list.NB_EPOCHS)}},
verbose=0)
score, acc = model.evaluate_generator(create_generator(
[x_test[:, :, 0:-2, :], x_test[:, :, -2:, :]],
y_test,
batch_size=param_list.BATCH_SIZE),
steps=len(x_test))
print("Test Accuracy: ", acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}