def model(train_x, train_y, dev_x, dev_y, test_x, test_y, dev_y_org,
test_y_org, overal_maxlen):
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras import optimizers
import keras.backend as K
import pickle as pk
import numpy as np
from deepats.optimizers import get_optimizer
from deepats.asap_evaluator import Evaluator
import deepats.asap_reader as dataset
from deepats.config import get_args
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
import string
import random
def random_id(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
import time
ms = int(round(time.time() * 1000))
rand_seed = ms % (2**32 - 1)
random.seed(rand_seed)
args = get_args()
model_id = random_id()
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def kappa_loss(t, x):
u = K.sum(K.square(x - t))
v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1))
return u / (2 * v + u)
lr = {{lognormal(-3 * 2.3, .8)}}
lr = lr * 2
rho = {{normal(.875, .04)}}
clipnorm = {{uniform(1, 15)}}
eps = 1e-6
opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps)
loss = kappa_loss
metric = kappa_metric
dataset.set_score_range(args.data_set)
evl = Evaluator(dataset,
args.prompt_id,
args.abs_out_path,
dev_x,
test_x,
dev_y,
test_y,
dev_y_org,
test_y_org,
model_id=model_id)
abs_vocab_file = args.abs_out_path + '/vocab.pkl'
with open(abs_vocab_file, 'rb') as vocab_file:
vocab = pk.load(vocab_file)
train_y_mean = train_y.mean(axis=0)
if train_y_mean.ndim == 0:
train_y_mean = np.expand_dims(train_y_mean, axis=1)
num_outputs = len(train_y_mean)
mask_zero = False
emb_dim = {{choice([50, 100, 200, 300])}}
rnn_dim = {{uniform(50, 300)}}
rnn_dim = int(rnn_dim)
model = Sequential()
model.add(Embedding(args.vocab_size, emb_dim, mask_zero=mask_zero))
model.add(RWA(rnn_dim))
model.add(Dense(num_outputs))
if not args.skip_init_bias:
bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype(
K.floatx())
model.layers[-1].bias.set_value(bias_value)
model.add(Activation('tanh'))
model.emb_index = 0
emb_path = 'embed/glove.6B.{}d.txt'.format(emb_dim)
abs_emb_path = args.abs_root + emb_path
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(abs_emb_path, emb_dim=emb_dim)
emb_reader.load_embeddings(vocab)
emb_wts = emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()[0])
wts = model.layers[model.emb_index].get_weights()
wts[0] = emb_wts
model.layers[model.emb_index].set_weights(wts)
model.compile(loss=loss, optimizer=opt, metrics=[metric])
model_yaml = model.to_yaml()
print('model_id: %s' % (model_id))
print(model_yaml)
print('optimizer: lr= %.4f, rho= %.4f, clipnorm= %.4f, epsilon= %.4f' %
(lr, rho, clipnorm, eps))
print('PARAMS\t\
%s\t\
lr= %.4f\t\
rho= %.4f\t\
clip= %.4f\t\
emb= %.4f\t\
rnn= %.4f' % (model_id, lr, rho, clipnorm, emb_dim, rnn_dim))
for i in range(args.epochs):
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
evl.evaluate(model, i)
evl.output_info()
if i > 5 and evl.dev_metric < 0.4:
break
if i > 10 and evl.dev_metric < 0.5:
break
if i > 15 and evl.dev_metric < 0.6:
break
best_dev_kappa = evl.best_dev
best_test_kappa = evl.best_test
print('Test kappa:', best_dev_kappa)
return {
'loss': 1 - best_dev_kappa,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pk.dumps(model.get_weights())
}
def model(train_x, train_y, dev_x, dev_y, test_x, test_y, overal_maxlen, qwks):
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, GlobalAveragePooling1D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.initializers import Constant
from keras import optimizers
import keras.backend as K
from deepats.my_layers import MeanOverTime
from deepats.rwa import RWA
import pickle as pk
import numpy as np
import string
import random
import os
from deepats.optimizers import get_optimizer
from deepats.ets_evaluator import Evaluator
import deepats.ets_reader as dataset
from deepats.ets_config import get_args
import GPUtil
def random_id(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def kappa_loss(t, x):
u = K.sum(K.square(x - t))
v = K.dot(K.squeeze(x, 1), K.squeeze(t - K.mean(t), 1))
return u / (2 * v + u)
import time
ms = int(round(time.time() * 1000))
rand_seed = ms % (2**32 - 1)
random.seed(rand_seed)
args = get_args()
model_id = random_id()
abs_vocab_file = os.path.join(args.abs_out, 'vocab.pkl')
with open(abs_vocab_file, 'rb') as vocab_file:
vocab = pk.load(vocab_file)
vocab_size = len(vocab)
acts = ['tanh', 'relu', 'hard_sigmoid']
emb_dim = {{choice([50, 100, 200, 300])}}
rnn_dim = {{uniform(50, 500)}}
rnn_dim = int(rnn_dim)
rec_act = {{choice([0, 1, 2])}}
rec_act = acts[rec_act]
dropout = {{uniform(0.2, 0.95)}}
epochs = args.epochs
n_emb = vocab_size * emb_dim
n_rwa = (903 + 2 * rnn_dim) * rnn_dim
n_tot = n_emb + n_rwa + rnn_dim + 1
lr = {{lognormal(-3 * 2.3, .8)}}
lr = 1.5 * lr
rho = {{normal(.875, .04)}}
clipnorm = {{uniform(1, 15)}}
eps = {{loguniform(-8 * 2.3, -5 * 2.3)}}
opt = optimizers.RMSprop(lr=lr, rho=rho, clipnorm=clipnorm, epsilon=eps)
loss = kappa_loss
metric = kappa_metric
evl = Evaluator(dataset,
args.prompt_id,
args.abs_out,
dev_x,
test_x,
dev_df,
test_df,
model_id=model_id)
train_y_mean = train_y.mean(axis=0)
if train_y_mean.ndim == 0:
train_y_mean = np.expand_dims(train_y_mean, axis=1)
num_outputs = len(train_y_mean)
mask_zero = False
model = Sequential()
model.add(Embedding(vocab_size, emb_dim, mask_zero=mask_zero))
model.add(RWA(rnn_dim, recurrent_activation=rec_act))
model.add(Dropout(dropout))
bias_value = (np.log(train_y_mean) - np.log(1 - train_y_mean)).astype(
K.floatx())
model.add(Dense(num_outputs, bias_initializer=Constant(value=bias_value)))
model.add(Activation('tanh'))
model.emb_index = 0
from deepats.w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim)
emb_reader.load_embeddings(vocab)
emb_wts = emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()[0])
wts = model.layers[model.emb_index].get_weights()
wts[0] = emb_wts
model.layers[model.emb_index].set_weights(wts)
model.compile(loss=loss, optimizer=opt, metrics=[metric])
model_yaml = model.to_yaml()
import GPUtil
if GPUtil.avail_mem() < 0.1:
return {'loss': 1, 'status': STATUS_OK, 'model': '', 'weights': None}
print('model_id: %s' % (model_id))
print(model_yaml)
print('PARAMS\t\
%s\t\
lr= %.4f\t\
rho= %.4f\t\
clip= %.4f\t\
eps= %.4f\t\
embDim= %.4f\t\
rnnDim= %.4f\t\
drop= %.4f\t\
recAct= %s' % (model_id, lr, rho, clipnorm, np.log(eps) / 2.3, emb_dim,
rnn_dim, dropout, rec_act))
for i in range(epochs):
train_history = model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=1,
verbose=0)
evl.evaluate(model, i)
evl.output_info()
p = evl.stats[3] / qwks[0]
if i > 10 and p < 0.9:
break
i = evl.comp_idx
j = i + 2
best_dev_kappa = evl.best_dev[i]
best_test_kappa = evl.best_dev[j]
print('Test kappa:', best_dev_kappa)
return {
'loss': 1 - best_dev_kappa,
'status': STATUS_OK,
'model': model.to_yaml(),
'weights': pk.dumps(model.get_weights())
}