def __init__(self):
self.config = json.loads(
open(
os.path.join(experiment_path, str(experiment_id),
'config.json'), 'rt').read())
self.i2w = pickle.load(open(os.path.join(data_path, 'i2w.pkl'), 'rb'))
self.w2i = pickle.load(open(os.path.join(data_path, 'w2i.pkl'), 'rb'))
# full lexicon and reading dictionary covers all the vocab
# that includes oov words to the model
self.full_lexicon = pickle.load(
open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb'))
self.full_reading_dict = pickle.load(
open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb'))
self.model = LSTM_Model()
def __init__(self, experiment_id=0, comp=0):
self.config = json.loads(
open(
os.path.join(experiment_path, str(experiment_id),
'config.json'), 'rt').read())
self._load_vocab()
# full lexicon and reading dictionary covers all the vocab
# that includes oov words to the model
self.full_lexicon = pickle.load(
open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb'))
self.full_reading_dict = pickle.load(
open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb'))
self.model = LSTM_Model(experiment_id, comp)
self.lattice_vocab = None
self.perf_sen = 0
self.perf_log_lstm = []
self.perf_log_softmax = []
import greengrasssdk
from pandas import read_csv
import json
from numpy import concatenate
from joblib import load
from keras.models import model_from_json
from pandas import DataFrame
from pandas import concat
import time
from model import LSTM_Model
lstm = LSTM_Model()
# Creating a greengrass core sdk client
client = greengrasssdk.client('iot-data')
data_dir = '/src/'
model_dir = '/dest'
def read_test_data():
dataset = read_csv(data_dir + 'pollution.csv', header=0)
test_dataset.dropna(inplace=True)
return dataset
def publishToIotCloud(curr_row, predicted_y, actual_y):
data = curr_row.to_dict(orient='records')[0]
payload = json.dumps(data)
client.publish(
topic='pollution/data',
payload=payload
)
from model import LSTM_Model
mod = LSTM_Model(vocab_size=122630,
sent_max_len=50,
valid_sent_max_len=50,
embedding_size=100)
model = mod.encoder_decoder()
print(model.summary())
def unimodal(mode):
print(('starting unimodal ', mode))
# with open('./mosei/text_glove_average.pickle', 'rb') as handle:
with open('./mosei/2way/2-way-' + mode + '.pickle', 'rb') as handle:
u = pickle._Unpickler(handle)
u.encoding = 'latin1'
# (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load()
(train_data, train_label, _, _, test_data, test_label, _, train_length,
_, test_length, _, _, _) = u.load()
# with open('./input/' + mode + '.pickle', 'rb') as handle:
# (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = pickle.load(handle)
train_label = train_label.astype('int')
test_label = test_label.astype('int')
train_mask = np.zeros((train_data.shape[0], train_data.shape[1]),
dtype='float')
for i in range(len(train_length)):
train_mask[i, :train_length[i]] = 1.0
test_mask = np.zeros((test_data.shape[0], test_data.shape[1]),
dtype='float')
for i in range(len(test_length)):
test_mask[i, :test_length[i]] = 1.0
train_label, test_label = createOneHotMosei3way(train_label, test_label)
attn_fusion = False
print('train_mask', train_mask.shape)
# print(train_mask_bool)
seqlen_train = np.sum(train_mask, axis=-1)
print('seqlen_train', seqlen_train.shape)
seqlen_test = np.sum(test_mask, axis=-1)
print('seqlen_test', seqlen_test.shape)
allow_soft_placement = True
log_device_placement = False
# Multimodal model
session_conf = tf.ConfigProto(
# device_count={'GPU': gpu_count},
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement,
gpu_options=tf.GPUOptions(allow_growth=True))
gpu_device = 0
best_acc = 0
best_epoch = 0
best_loss = 1000000.0
best_epoch_loss = 0
is_unimodal = True
with tf.device('/device:GPU:%d' % gpu_device):
print('Using GPU - ', '/device:GPU:%d' % gpu_device)
with tf.Graph().as_default():
tf.set_random_seed(seed)
sess = tf.Session(config=session_conf)
with sess.as_default():
model = LSTM_Model(train_data.shape[1:],
0.001,
emotions=emotions,
attn_fusion=attn_fusion,
unimodal=is_unimodal,
seed=seed)
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
test_feed_dict = {
model.input: test_data,
model.y: test_label,
model.seq_len: seqlen_test,
model.mask: test_mask,
model.lstm_dropout: 0.0,
model.dropout: 0.0
}
train_feed_dict = {
model.input: train_data,
model.y: train_label,
model.seq_len: seqlen_train,
model.mask: train_mask,
model.lstm_dropout: 0.0,
model.dropout: 0.0
}
# print('\n\nDataset: %s' % (data))
print("\nEvaluation before training:")
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
test_feed_dict)
print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(
0, step, loss, accuracy))
for epoch in range(epochs):
epoch += 1
batches = batch_iter(
list(
zip(train_data, train_mask, seqlen_train,
train_label)), batch_size)
# Training loop. For each batch...
print('\nTraining epoch {}'.format(epoch))
l = []
a = []
for i, batch in tqdm(enumerate(batches)):
b_train_data, b_train_mask, b_seqlen_train, b_train_label = zip(
*batch)
# print('batch_hist_v', len(batch_utt_v))
feed_dict = {
model.input: b_train_data,
model.y: b_train_label,
model.seq_len: b_seqlen_train,
model.mask: b_train_mask,
model.lstm_dropout: 0.5,
model.dropout: 0.3,
}
_, step, loss, accuracy = sess.run([
model.train_op, model.global_step, model.loss,
model.accuracy
], feed_dict)
l.append(loss)
a.append(accuracy)
print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format(
epoch, np.average(l), np.average(a)))
# Evaluation after epoch
step, loss, accuracy, test_activations = sess.run([
model.global_step, model.loss, model.accuracy,
model.inter1
], test_feed_dict)
loss = loss / test_label.shape[0]
print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}".
format(epoch, step, loss, accuracy))
if accuracy > best_acc:
best_epoch = epoch
best_acc = accuracy
step, loss, accuracy, train_activations = sess.run([
model.global_step, model.loss, model.accuracy,
model.inter1
], train_feed_dict)
unimodal_activations[mode +
'_train'] = train_activations
unimodal_activations[mode + '_test'] = test_activations
unimodal_activations['train_mask'] = train_mask
unimodal_activations['test_mask'] = test_mask
unimodal_activations['train_label'] = train_label
unimodal_activations['test_label'] = test_label
if loss < best_loss:
best_epoch_loss = epoch
best_loss = loss
# step, loss, accuracy, train_activations = sess.run(
# [model.global_step, model.loss, model.accuracy, model.inter1],
# train_feed_dict)
# unimodal_activations[mode + '_train'] = train_activations
# unimodal_activations[mode + '_test'] = test_activations
# unimodal_activations['train_mask'] = train_mask
# unimodal_activations['test_mask'] = test_mask
# unimodal_activations['train_label'] = train_label
# unimodal_activations['test_label'] = test_label
print("\n\nBest epoch: {}\nBest test accuracy: {}".format(
best_epoch, best_acc))
print("\n\nBest epoch: {}\nBest test loss: {}".format(
best_epoch_loss, best_loss))
def multimodal(unimodal_activations, attn_fusion=True, enable_attn_2=False):
if attn_fusion:
print('With attention fusion')
print("starting multimodal")
# Fusion (appending) of features
text_train = unimodal_activations['text_train']
audio_train = unimodal_activations['audio_train']
video_train = unimodal_activations['video_train']
text_test = unimodal_activations['text_test']
audio_test = unimodal_activations['audio_test']
video_test = unimodal_activations['video_test']
train_mask = unimodal_activations['train_mask']
test_mask = unimodal_activations['test_mask']
print('train_mask', train_mask.shape)
train_label = unimodal_activations['train_label']
print('train_label', train_label.shape)
test_label = unimodal_activations['test_label']
print('test_label', test_label.shape)
# print(train_mask_bool)
seqlen_train = np.sum(train_mask, axis=-1)
print('seqlen_train', seqlen_train.shape)
seqlen_test = np.sum(test_mask, axis=-1)
print('seqlen_test', seqlen_test.shape)
allow_soft_placement = True
log_device_placement = False
# Multimodal model
session_conf = tf.ConfigProto(
# device_count={'GPU': gpu_count},
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement,
gpu_options=tf.GPUOptions(allow_growth=True))
gpu_device = 0
best_acc = 0
best_loss_accuracy = 0
best_loss = 10000000.0
best_epoch = 0
best_epoch_loss = 0
with tf.device('/device:GPU:%d' % gpu_device):
print('Using GPU - ', '/device:GPU:%d' % gpu_device)
with tf.Graph().as_default():
tf.set_random_seed(seed)
sess = tf.Session(config=session_conf)
with sess.as_default():
model = LSTM_Model(text_train.shape[1:],
0.0001,
emotions=emotions,
attn_fusion=attn_fusion,
unimodal=False,
enable_attn_2=enable_attn_2,
seed=seed)
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
test_feed_dict = {
model.t_input: text_test,
model.a_input: audio_test,
model.v_input: video_test,
model.y: test_label,
model.seq_len: seqlen_test,
model.mask: test_mask,
model.lstm_dropout: 0.0,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.0,
model.dropout_lstm_out: 0.0
}
# print('\n\nDataset: %s' % (data))
print("\nEvaluation before training:")
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
test_feed_dict)
print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(
0, step, loss, accuracy))
for epoch in range(epochs):
epoch += 1
batches = batch_iter(
list(
zip(text_train, audio_train, video_train,
train_mask, seqlen_train, train_label)),
batch_size)
# Training loop. For each batch...
print('\nTraining epoch {}'.format(epoch))
l = []
a = []
for i, batch in tqdm(enumerate(batches)):
b_text_train, b_audio_train, b_video_train, b_train_mask, b_seqlen_train, b_train_label = zip(
*batch)
# print('batch_hist_v', len(batch_utt_v))
feed_dict = {
model.t_input: b_text_train,
model.a_input: b_audio_train,
model.v_input: b_video_train,
model.y: b_train_label,
model.seq_len: b_seqlen_train,
model.mask: b_train_mask,
model.lstm_dropout: 0.4,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.2,
model.dropout_lstm_out: 0.2
}
_, step, loss, accuracy = sess.run([
model.train_op, model.global_step, model.loss,
model.accuracy
], feed_dict)
l.append(loss)
a.append(accuracy)
print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format(
epoch, np.average(l), np.average(a)))
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
test_feed_dict)
print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}".
format(epoch, step, loss / test_label.shape[0],
accuracy))
if accuracy > best_acc:
best_epoch = epoch
best_acc = accuracy
if loss < best_loss:
best_loss = loss
best_loss_accuracy = accuracy
best_epoch_loss = epoch
print(
"\n\nBest epoch: {}\nBest test accuracy: {}\nBest epoch loss: {}\nBest test accuracy when loss is least: {}"
.format(best_epoch, best_acc, best_epoch_loss,
best_loss_accuracy))
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
from model import LSTM_Model
import tensorflow as tf
import keras.backend.tensorflow_backend as ktf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
ktf.set_session(session)
model = LSTM_Model(gpus=4)
for i in range(0, model.rolling_count, 3):
model.rolling(i)
model.save_model()
class Decoder():
def __init__(self):
self.config = json.loads(
open(
os.path.join(experiment_path, str(experiment_id),
'config.json'), 'rt').read())
vocab = Vocab(self.config['vocab_size'])
self.i2w = vocab.i2w
self.w2i = vocab.w2i
# full lexicon and reading dictionary covers all the vocab
# that includes oov words to the model
self.full_lexicon = pickle.load(
open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb'))
self.full_reading_dict = pickle.load(
open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb'))
self.model = LSTM_Model()
def _check_oov(self, word):
return word not in self.w2i
def _build_lattice(self, input, use_oov=False):
def add_node_to_lattice(i, sub_token, id, word, prob):
node = Node(i, len(sub_token), id, word,
prob) # share the node in both lookup table
backward_lookup[i + len(sub_token)].append(node)
# backward_lookup keeps the words that ends at a frame
# e.g. the input A/BC/D, 0 is preserved for <eos>
# 0 1 2 3 4
# <eos> A BC D
backward_lookup = defaultdict(lambda: [])
eos_node = [Node(-1, 1, self.w2i['<eos>'], '<eos>')]
backward_lookup[0] = eos_node
for i, token in enumerate(input):
for j in range(len(input) - i):
sub_token = input[i:i + j + 1]
if sub_token in self.full_reading_dict.keys():
for lexicon_id in self.full_reading_dict[sub_token]:
word = self.full_lexicon[lexicon_id][0]
oov = self._check_oov(word)
if oov:
# skip oov in this experiment,
# note that oov affects conversion quality
continue
prob = 0.0
id = self.w2i[word]
add_node_to_lattice(i, sub_token, id, word, prob)
# put symbol directly if no match at all in reading dictionary
if len(backward_lookup[i + 1]) == 0:
backward_lookup[i + 1].append(
Node(i, 1, self.w2i['<unk>'], input[i]))
return backward_lookup
def _build_current_frame(self, nodes, frame, idx):
# frame 0 contains one path that has eos_node
if idx == 0:
frame[0] = [Path(nodes[0], self.model.hidden_size)]
return
# connect each nodes to its previous best paths, also calculate the new path probability
frame[idx] = []
for node in nodes:
for prev_path in frame[node.start_idx]:
cur_paths = copy(
prev_path) # shallow copy to avoid create dup objects
cur_paths.nodes = copy(prev_path.nodes)
node_prob_idx = node.word_idx
cur_paths.append_node(node, node_prob_idx)
frame[idx].append(cur_paths)
def _batch_predict(self, paths):
pred, state, cell = self.model.predict_with_context(
[path.nodes[-1].word_idx for path in paths],
np.concatenate([path.state for path in paths], axis=0),
np.concatenate([path.cell for path in paths], axis=0))
for i, path in enumerate(paths):
path.state = np.expand_dims(state[i], axis=0)
path.cell = np.expand_dims(cell[i], axis=0)
path.transition_probs = [pred[i]]
def decode(self, input, topN=10, beam_width=10, use_oov=False):
backward_lookup = self._build_lattice(input, use_oov)
frame = {}
for i in range(len(input) + 1):
b_nodes = backward_lookup[i]
self._build_current_frame(b_nodes, frame, i)
if beam_width is not None:
frame[i].sort(key=lambda x: x.neg_log_prob)
frame[i] = frame[i][:beam_width]
self._batch_predict(frame[i])
output = [(x.neg_log_prob,
[n.word for n in x.nodes if n.word != "<eos>"])
for x in frame[len(input)]]
return output[:topN]
def main(args):
lr = 0.001
attn_fusion = args.attention_2
train_u1_data, train_u2_data, train_u3_data, val_u1_data, val_u2_data, val_u3_data, Y_train, Y_val, \
tr_sq_u1, tr_sq_u2, tr_sq_u3, val_sq_u1, val_sq_u2, val_sq_u3, embeddingMatrix = get_data(args)
vocab_size, embedding_dim = embeddingMatrix.shape
print('embeddingMatrix.shape', embeddingMatrix.shape)
print('vocab_size', vocab_size)
print('embedding_dim', embedding_dim)
classes = Y_train.shape[-1]
allow_soft_placement = True
log_device_placement = False
# Multimodal model
session_conf = tf.ConfigProto(
# device_count={'GPU': gpu_count},
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement,
gpu_options=tf.GPUOptions(allow_growth=True))
gpu_device = 0
best_acc = 0
best_epoch = 0
best_loss = 1000000.0
best_epoch_loss = 0
print("Building model...")
with tf.device('/device:GPU:%d' % gpu_device):
print('Using GPU - ', '/device:GPU:%d' % gpu_device)
with tf.Graph().as_default():
tf.set_random_seed(seed)
sess = tf.Session(config=session_conf)
with sess.as_default():
model = LSTM_Model(train_u1_data.shape[1],
lr,
vocab_size,
embedding_dim,
emotions=classes,
seed=seed,
enable_attn_2=attn_fusion)
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
# init embeddings
sess.run(
model.embedding_init,
feed_dict={model.embedding_placeholder: embeddingMatrix})
test_feed_dict = {
model.input1: val_u1_data,
model.input2: val_u2_data,
model.input3: val_u3_data,
model.y: Y_val,
model.seq_len1: val_sq_u1,
model.seq_len2: val_sq_u2,
model.seq_len3: val_sq_u3,
model.lstm_dropout: 0.0,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.0,
model.dropout_lstm_out: 0.0
}
train_feed_dict = {
model.input1: train_u1_data,
model.input2: train_u2_data,
model.input3: train_u3_data,
model.y: Y_train,
model.seq_len1: tr_sq_u1,
model.seq_len2: tr_sq_u2,
model.seq_len3: tr_sq_u3,
model.lstm_dropout: 0.0,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.0,
model.dropout_lstm_out: 0.0
}
# print('\n\nDataset: %s' % (data))
print("\nEvaluation before training:")
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
test_feed_dict)
print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(
0, step, loss, accuracy))
for epoch in range(epochs):
epoch += 1
batches = batch_iter(
list(
zip(train_u1_data, train_u2_data, train_u3_data,
tr_sq_u1, tr_sq_u2, tr_sq_u3, Y_train)),
batch_size)
# Training loop. For each batch...
print('\nTraining epoch {}'.format(epoch))
l = []
a = []
for i, batch in tqdm(enumerate(batches)):
b_train_u1_data, b_train_u2_data, b_train_u3_data, b_tr_sq_u1, b_tr_sq_u2, b_tr_sq_u3, b_Y_train = zip(
*batch)
feed_dict = {
model.input1: b_train_u1_data,
model.input2: b_train_u2_data,
model.input3: b_train_u3_data,
model.y: b_Y_train,
model.seq_len1: b_tr_sq_u1,
model.seq_len2: b_tr_sq_u2,
model.seq_len3: b_tr_sq_u3,
model.lstm_dropout: 0.5,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.2,
model.dropout_lstm_out: 0.2
}
_, step, loss, accuracy = sess.run([
model.train_op, model.global_step, model.loss,
model.accuracy
], feed_dict)
l.append(loss)
a.append(accuracy)
print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format(
epoch, np.average(l), np.average(a)))
# Evaluation after epoch
step, loss, accuracy, test_activations = sess.run([
model.global_step, model.loss, model.accuracy,
model.inter1
], test_feed_dict)
loss = loss / Y_val.shape[0]
print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}".
format(epoch, step, loss, accuracy))
if accuracy > best_acc:
best_epoch = epoch
best_acc = accuracy
# if epoch == 30:
# step, loss, accuracy, train_activations = sess.run(
# [model.global_step, model.loss, model.accuracy, model.inter1],
# train_feed_dict)
if loss < best_loss:
best_epoch_loss = epoch
best_loss = loss
# step, loss, accuracy, train_activations = sess.run(
# [model.global_step, model.loss, model.accuracy, model.inter1],
# train_feed_dict)
print("\n\nBest epoch: {}\nBest test accuracy: {}".format(
best_epoch, best_acc))
print("\n\nBest epoch: {}\nBest test loss: {}".format(
best_epoch_loss, best_loss))
def train():
df_ge = pd.read_csv("ge.txt", engine='python')
df_ge.tail()
# visualize(df_ge)
train_cols = ["Open", "High", "Low", "Close", "Volume"]
df_train, df_test = train_test_split(df_ge,
train_size=0.8,
test_size=0.2,
shuffle=False)
print("Train and Test size", len(df_train), len(df_test), flush=True)
# scale the feature MinMax, build array
x = df_train.loc[:, train_cols].values
min_max_scaler = MinMaxScaler()
x_train = min_max_scaler.fit_transform(x)
x_test = min_max_scaler.transform(df_test.loc[:, train_cols])
x_t, y_t = build_timeseries(x_train, 3)
x_t = trim_dataset(x_t, BATCH_SIZE)
y_t = trim_dataset(y_t, BATCH_SIZE)
x_temp, y_temp = build_timeseries(x_test, 3)
x_val, x_test_t = np.split(trim_dataset(x_temp, BATCH_SIZE), 2)
y_val, y_test_t = np.split(trim_dataset(y_temp, BATCH_SIZE), 2)
model = LSTM_Model()
optimizer = torch.optim.RMSprop(model.parameters(), lr=LR)
model.train()
loss_epo_list = list()
loss_iter_list = list()
for i in range(EPOCHS):
total_iter = int(x_t.shape[0] / BATCH_SIZE)
loss_avg = AverageMeter()
for j in range(total_iter):
input = x_t[j * BATCH_SIZE:(j + 1) * BATCH_SIZE, :, :]
gt = y_t[j * BATCH_SIZE:(j + 1) * BATCH_SIZE]
gt = torch.Tensor(gt)
input = torch.Tensor(input)
input = input.permute(1, 0, 2)
output, loss = model(input, gt)
loss_avg.update(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_iter_list.append(loss.item())
if j % 30 == 0:
print("Epoch{}:{}/{} loss:{}".format(i, j, total_iter,
loss_avg.avg),
flush=True)
loss_epo_list.append(loss_avg.avg)
with open("loss_epo.json", 'w') as f:
json.dump(loss_epo_list, f, indent=4)
with open("loss_iter.json", 'w') as f:
json.dump(loss_iter_list, f, indent=4)
torch.save({
'loss': loss,
'state_dict': model.state_dict(),
}, '{}.pth'.format('checkpoint'))
def evaluate():
model = LSTM_Model()
state_dict = torch.load("checkpoint.pth")
model.load_state_dict(state_dict['state_dict'], strict=True)
df_ge = pd.read_csv("ge.txt", engine='python')
df_ge.tail()
# visualize(df_ge)
train_cols = ["Open", "High", "Low", "Close", "Volume"]
df_train, df_test = train_test_split(df_ge,
train_size=0.8,
test_size=0.2,
shuffle=False)
print("Train and Test size", len(df_train), len(df_test), flush=True)
# scale the feature MinMax, build array
x = df_train.loc[:, train_cols].values
min_max_scaler = MinMaxScaler()
x_train = min_max_scaler.fit_transform(x)
x_test = min_max_scaler.transform(df_test.loc[:, train_cols])
x_t, y_t = build_timeseries(x_train, 3)
x_t = trim_dataset(x_t, BATCH_SIZE)
y_t = trim_dataset(y_t, BATCH_SIZE)
# x_temp, y_temp = build_timeseries(x_test, 3)
# x_val, x_test_t = np.split(trim_dataset(x_temp, BATCH_SIZE), 2)
# y_val, y_test_t = np.split(trim_dataset(y_temp, BATCH_SIZE), 2)
model.eval()
y_gt = list()
y_pred = list()
x = np.array(list(range(11246 + 60, 11246 + 2812)))
for i in range(x_test.shape[0] - TIME_STEPS):
input = x_test[i:i + TIME_STEPS, :]
gt = x_test[i + TIME_STEPS, 3]
y_gt.append(gt)
input = torch.Tensor(input)
input = input.unsqueeze(dim=1)
with torch.no_grad():
output = model(input)
output = np.array(output)
y_pred.append(output)
y_pred = np.array(y_pred)
y_gt = np.array(y_gt)
mse = np.sum(np.power(y_pred - y_gt, 2)) / y_pred.shape[0]
# mse: 0.00023299293330091727
print("mse: ", mse)
y_pred_org = (y_pred *
min_max_scaler.data_range_[3]) + min_max_scaler.data_min_[3]
y_gt_org = (y_gt *
min_max_scaler.data_range_[3]) + min_max_scaler.data_min_[3]
plt.figure()
plt.ylabel('Price (USD)')
plt.xlabel('Days')
plt.title('Prediction vs Real Stock Price')
plt.plot(x, y_gt_org, linewidth=0.8)
plt.plot(x, y_pred_org, linewidth=0.8)
plt.legend(['Real', 'Prediction'], loc='upper left')
plt.show()
def multimodal(unimodal_activations,
data,
classes,
attn_fusion=True,
enable_attn_2=False,
use_raw=True):
if use_raw:
if attn_fusion:
attn_fusion = False
train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask = get_raw_data(
data, classes)
a_dim = audio_train.shape[-1]
v_dim = video_train.shape[-1]
t_dim = text_train.shape[-1]
else:
print("starting multimodal")
# Fusion (appending) of features
train_mask = unimodal_activations['train_mask']
test_mask = unimodal_activations['test_mask']
print('train_mask', train_mask.shape)
array_default = np.empty(
(train_mask.shape[0], train_mask.shape[1], 100))
t_dim = a_dim = v_dim = None #initializing the dims
if ('text_train' in unimodal_activations):
text_train = unimodal_activations['text_train']
text_test = unimodal_activations['text_test']
t_dim = text_train.shape[-1]
else:
text_train = array_default
text_test = array_default
if ('audio_train' in unimodal_activations):
audio_train = unimodal_activations['audio_train']
audio_test = unimodal_activations['audio_test']
a_dim = audio_train.shape[-1]
else:
audio_train = array_default
audio_test = array_default
if ('video_train' in unimodal_activations):
video_train = unimodal_activations['video_train']
video_test = unimodal_activations['video_test']
v_dim = video_train.shape[-1]
else:
video_train = array_default
video_test = array_default
train_label = unimodal_activations['train_label']
print('train_label', train_label.shape)
test_label = unimodal_activations['test_label']
print('test_label', test_label.shape)
# print(train_mask_bool)
seqlen_train = np.sum(train_mask, axis=-1)
print('seqlen_train', seqlen_train.shape)
seqlen_test = np.sum(test_mask, axis=-1)
print('seqlen_test', seqlen_test.shape)
if attn_fusion:
print('With attention fusion')
allow_soft_placement = True
log_device_placement = False
# Multimodal model
session_conf = tf.ConfigProto(
# device_count={'GPU': gpu_count},
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement,
gpu_options=tf.GPUOptions(allow_growth=True))
gpu_device = 0
best_acc = 0
best_loss_accuracy = 0
best_loss = 10000000.0
best_epoch = 0
best_epoch_loss = 0
with tf.device('/device:GPU:%d' % gpu_device):
print('Using GPU - ', '/device:GPU:%d' % gpu_device)
with tf.Graph().as_default():
tf.set_random_seed(seed)
sess = tf.Session(config=session_conf)
with sess.as_default():
model = LSTM_Model(text_train.shape[1:],
0.0001,
a_dim=a_dim,
v_dim=v_dim,
t_dim=t_dim,
emotions=classes,
attn_fusion=attn_fusion,
unimodal=False,
enable_attn_2=enable_attn_2,
seed=seed)
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
test_feed_dict = {
model.y: test_label,
model.seq_len: seqlen_test,
model.mask: test_mask,
model.lstm_dropout: 0.0,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.0,
model.dropout_lstm_out: 0.0
}
if t_dim: test_feed_dict.update({model.t_input: text_test})
if a_dim: test_feed_dict.update({model.a_input: audio_test})
if v_dim: test_feed_dict.update({model.v_input: video_test})
# print('\n\nDataset: %s' % (data))
print("\nEvaluation before training:")
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
#[model.global_step, model.loss, model.f1],
test_feed_dict)
print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(
0, step, loss, accuracy))
for epoch in range(epochs):
epoch += 1
batches = batch_iter(
list(
zip(text_train, audio_train, video_train,
train_mask, seqlen_train, train_label)),
batch_size)
# Training loop. For each batch...
print('\nTraining epoch {}'.format(epoch))
l = []
a = []
for i, batch in tqdm(enumerate(batches)):
b_text_train, b_audio_train, b_video_train, b_train_mask, b_seqlen_train, b_train_label = zip(
*batch)
# print('batch_hist_v', len(batch_utt_v))
feed_dict = {}
if t_dim:
feed_dict.update({model.t_input: b_text_train})
if a_dim:
feed_dict.update({model.a_input: b_audio_train})
if v_dim:
feed_dict.update({model.v_input: b_video_train})
feed_dict.update({
model.y: b_train_label,
model.seq_len: b_seqlen_train,
model.mask: b_train_mask,
model.lstm_dropout: 0.4,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.2,
model.dropout_lstm_out: 0.2
})
_, step, loss, accuracy = sess.run([
model.train_op, model.global_step, model.loss,
model.accuracy
], feed_dict)
l.append(loss)
a.append(accuracy)
print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format(
epoch, np.average(l), np.average(a)))
# Evaluation after epoch
step, loss, accuracy, preds, y, mask = sess.run([
model.global_step, model.loss, model.accuracy,
model.preds, model.y, model.mask
], test_feed_dict)
f1 = f1_score(np.ndarray.flatten(
tf.argmax(y, -1, output_type=tf.int32).eval()),
np.ndarray.flatten(
tf.argmax(preds,
-1,
output_type=tf.int32).eval()),
sample_weight=np.ndarray.flatten(
tf.cast(mask, tf.int32).eval()),
average="weighted")
print(
"EVAL: After epoch {}: step {}, loss {:g}, acc {:g}, f1 {:g}}"
.format(epoch, step, loss / test_label.shape[0],
accuracy, f1))
if accuracy > best_acc:
best_epoch = epoch
best_acc = accuracy
if loss < best_loss:
best_loss = loss
best_loss_accuracy = accuracy
best_epoch_loss = epoch
print(
"\n\nBest epoch: {}\nBest test accuracy: {}\nBest epoch loss: {}\nBest test accuracy when loss is least: {}"
.format(best_epoch, best_acc, best_epoch_loss,
best_loss_accuracy))
def unimodal(mode, data, classes):
print(('starting unimodal ', mode))
# with open('./mosei/text_glove_average.pickle', 'rb') as handle:
if data == 'mosei' or data == 'mosi':
with open(
'./dataset/{0}/raw/{1}_{2}way.pickle'.format(
data, mode, classes), 'rb') as handle:
u = pickle._Unpickler(handle)
u.encoding = 'latin1'
# (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load()
if data == 'mosei':
(train_data, train_label, _, _, test_data, test_label, _,
train_length, _, test_length, _, _, _) = u.load()
if classes == '2':
train_label, test_label = createOneHotMosei2way(
train_label, test_label)
elif data == 'mosi':
(train_data, train_label, test_data, test_label, maxlen,
train_length, test_length) = u.load()
train_label = train_label.astype('int')
test_label = test_label.astype('int')
train_label, test_label = createOneHot(train_label, test_label)
else:
raise NotImplementedError('Unknown dataset...')
train_label = train_label.astype('int')
test_label = test_label.astype('int')
train_mask = np.zeros((train_data.shape[0], train_data.shape[1]),
dtype='float')
for i in range(len(train_length)):
train_mask[i, :train_length[i]] = 1.0
test_mask = np.zeros((test_data.shape[0], test_data.shape[1]),
dtype='float')
for i in range(len(test_length)):
test_mask[i, :test_length[i]] = 1.0
elif data == 'iemocap':
train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask = get_raw_data(
data, classes)
if mode == 'text':
train_data = text_train
test_data = text_test
elif mode == 'audio':
train_data = audio_train
test_data = audio_test
elif mode == 'video':
train_data = video_train
test_data = video_test
elif data == 'cognimuse':
maxlen = 368
train_data = pickle.load(open("X_train_np.pkl", "rb")).numpy()
train_label = pickle.load(open("Y_train_np.pkl", "rb")).numpy()
test_data = pickle.load(open("X_test_np.pkl", "rb")).numpy()
test_label = pickle.load(open("Y_test_np.pkl", "rb")).numpy()
train_length = pickle.load(open("train_length.pkl", "rb"))
test_length = pickle.load(open("test_length.pkl", "rb"))
#train_label = train_label.astype('int')
#test_label = test_label.astype('int')
train_mask = np.zeros((train_data.shape[0], train_data.shape[1]),
dtype='float')
for i in range(len(train_length)):
train_mask[i, :train_length[i]] = 1.0
test_mask = np.zeros((test_data.shape[0], test_data.shape[1]),
dtype='float')
for i in range(len(test_length)):
test_mask[i, :test_length[i]] = 1.0
# train_label, test_label = createOneHotMosei3way(train_label, test_label)
attn_fusion = False
print('train_mask', train_mask.shape)
# print(train_mask_bool)
seqlen_train = np.sum(train_mask, axis=-1)
print('seqlen_train', seqlen_train.shape)
seqlen_test = np.sum(test_mask, axis=-1)
print('seqlen_test', seqlen_test.shape)
allow_soft_placement = True
log_device_placement = False
# Multimodal model
session_conf = tf.ConfigProto(
# device_count={'GPU': gpu_count},
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement,
gpu_options=tf.GPUOptions(allow_growth=True))
gpu_device = 0
best_acc = 0
best_f1 = 0
best_epoch = 0
best_loss = 1000000.0
best_epoch_loss = 0
is_unimodal = True
with tf.device('/device:GPU:%d' % gpu_device):
print('Using GPU - ', '/device:GPU:%d' % gpu_device)
with tf.Graph().as_default():
tf.set_random_seed(seed)
sess = tf.Session(config=session_conf)
with sess.as_default():
model = LSTM_Model(train_data.shape[1:],
0.003,
a_dim=0,
v_dim=0,
t_dim=0,
emotions=classes,
attn_fusion=attn_fusion,
unimodal=is_unimodal,
seed=seed)
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
test_feed_dict = {
model.input: test_data,
model.y: test_label,
model.seq_len: seqlen_test,
model.mask: test_mask,
model.lstm_dropout: 0.0,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.0,
model.dropout_lstm_out: 0.0
}
train_feed_dict = {
model.input: train_data,
model.y: train_label,
model.seq_len: seqlen_train,
model.mask: train_mask,
model.lstm_dropout: 0.4,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.2,
model.dropout_lstm_out: 0.2
}
# print('\n\nDataset: %s' % (data))
print("\nEvaluation before training:")
# Evaluation after epoch
step, loss, accuracy = sess.run(
[model.global_step, model.loss, model.accuracy],
#model.global_step, model.loss, model.f1],
test_feed_dict)
print(accuracy)
#accuracy=accuracy[0]
#print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(0, step, loss, accuracy))
for epoch in range(epochs):
epoch += 1
print(len(train_data))
batches = batch_iter(
list(
zip(train_data, train_mask, seqlen_train,
train_label)), batch_size)
# Training loop. For each batch...
print('\nTraining epoch {}'.format(epoch))
l = []
a = []
for i, batch in tqdm(enumerate(batches)):
b_train_data, b_train_mask, b_seqlen_train, b_train_label = zip(
*batch)
feed_dict = {
model.input: b_train_data,
model.y: b_train_label,
model.seq_len: b_seqlen_train,
model.mask: b_train_mask,
model.lstm_dropout: 0.4,
model.lstm_inp_dropout: 0.0,
model.dropout: 0.2,
model.dropout_lstm_out: 0.2
}
_, step, loss, accuracy = sess.run(
[
model.train_op, model.global_step, model.loss,
model.accuracy
],
#[model.train_op, model.global_step, model.loss, model.f1],
feed_dict)
#accuracy=accuracy[0]
l.append(loss)
a.append(accuracy)
print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format(
epoch, np.average(l), np.average(a)))
# Evaluation after epoch
#step, loss, accuracy, preds, y, mask = sess.run(
#[model.global_step, model.loss, model.accuracy, model.preds, model.y, model.mask],
#test_feed_dict)
step, loss, accuracy, test_activations, preds, y, mask = sess.run(
[
model.global_step, model.loss, model.accuracy,
model.inter1, model.preds, model.y, model.mask
],
#[model.global_step, model.loss, model.f1, model.inter1],
test_feed_dict)
#accuracy=accuracy[0]
loss = loss / test_label.shape[0]
f1 = f1_score(np.ndarray.flatten(
tf.argmax(y, -1, output_type=tf.int32).eval()),
np.ndarray.flatten(
tf.argmax(preds,
-1,
output_type=tf.int32).eval()),
sample_weight=np.ndarray.flatten(
tf.cast(mask, tf.int32).eval()),
average="weighted")
auc = roc_auc_score(np.ndarray.flatten(
tf.argmax(y, -1, output_type=tf.int32).eval()),
np.ndarray.flatten(
tf.argmax(
preds,
-1,
output_type=tf.int32).eval()),
sample_weight=np.ndarray.flatten(
tf.cast(mask, tf.int32).eval()),
average="weighted")
print(
"EVAL: After epoch {}: step {}, loss {:g}, acc {:g},f1 {:g},auc {:g}"
.format(epoch, step, loss, accuracy, f1, auc))
if accuracy > best_acc:
#best_epoch = epoch
best_acc = accuracy
if f1 > best_f1:
best_epoch = epoch
best_f1 = f1
if epoch == 30:
step, loss, accuracy, train_activations = sess.run(
[
model.global_step, model.loss, model.accuracy,
model.inter1
],
#[model.global_step, model.loss, model.f1, model.inter1],
train_feed_dict)
#accuracy=accuracy[0]
unimodal_activations[mode +
'_train'] = train_activations
unimodal_activations[mode + '_test'] = test_activations
unimodal_activations['train_mask'] = train_mask
unimodal_activations['test_mask'] = test_mask
unimodal_activations['train_label'] = train_label
unimodal_activations['test_label'] = test_label
if loss < best_loss:
best_epoch_loss = epoch
best_loss = loss
# step, loss, accuracy, train_activations = sess.run(
# [model.global_step, model.loss, model.accuracy, model.inter1],
# train_feed_dict)
# unimodal_activations[mode + '_train'] = train_activations
# unimodal_activations[mode + '_test'] = test_activations
# unimodal_activations['train_mask'] = train_mask
# unimodal_activations['test_mask'] = test_mask
# unimodal_activations['train_label'] = train_label
# unimodal_activations['test_label'] = test_label
print("\n\nBest epoch: {}\nBest test accuracy: {}".format(
best_epoch, best_acc))
print("\n\nBest epoch: {}\nBest test loss: {}".format(
best_epoch_loss, best_loss))
print("\n\nBest epoch: {}\nBest test f1: {}".format(
best_epoch, best_f1))
class Decoder():
def __init__(self, experiment_id=0, comp=0):
self.config = json.loads(
open(
os.path.join(experiment_path, str(experiment_id),
'config.json'), 'rt').read())
self._load_vocab()
# full lexicon and reading dictionary covers all the vocab
# that includes oov words to the model
self.full_lexicon = pickle.load(
open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb'))
self.full_reading_dict = pickle.load(
open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb'))
self.model = LSTM_Model(experiment_id, comp)
self.lattice_vocab = None
self.perf_sen = 0
self.perf_log_lstm = []
self.perf_log_softmax = []
def _load_vocab(self):
self.vocab = Vocab(self.config['vocab_size'])
self.i2w = self.vocab.i2w
self.w2i = self.vocab.w2i
def _check_oov(self, word):
return word not in self.w2i
def _build_lattice(self,
input,
vocab_select=False,
samples=0,
top_sampling=False,
random_sampling=False):
def add_node_to_lattice(i, sub_token, id, word, prob):
node = Node(i, len(sub_token), id, word,
prob) # share the node in both lookup table
backward_lookup[i + len(sub_token)].append(node)
# backward_lookup keeps the words that ends at a frame
# e.g. the input A/BC/D, 0 is preserved for <eos>
# 0 1 2 3 4
# <eos> A BC D
backward_lookup = defaultdict(lambda: [])
eos_node = [Node(-1, 1, self.w2i['<eos>'], '<eos>')]
backward_lookup[0] = eos_node
for i, token in enumerate(input):
for j in range(len(input) - i):
sub_token = input[i:i + j + 1]
if sub_token in self.full_reading_dict.keys():
word_set = set()
for lexicon_id in sorted(
self.full_reading_dict[sub_token]):
word = self.full_lexicon[lexicon_id][0]
oov = self._check_oov(word)
if oov:
# skip oov in this experiment,
# note that oov affects conversion quality
continue
prob = 0.0
if self.config['char_rnn']:
oov = self._char_check_oov(word)
if oov:
# skip oov in this experiment,
# note that oov affects conversion quality
continue
# char rnn case, we still build lattice use word, keep the first char as index of the node
word = word.split('/')[0]
if word in word_set:
continue
if len(word_set) > 200:
continue
word_set.add(word)
id = self.w2i[word[0]]
else:
id = self.w2i[word]
add_node_to_lattice(i, sub_token, id, word, prob)
# put symbol directly if no match at all in reading dictionary
if len(backward_lookup[i + 1]) == 0:
backward_lookup[i + 1].append(
Node(i, 1, self.w2i['<unk>'], input[i]))
if vocab_select:
self._build_lattice_vocab(backward_lookup, samples, top_sampling,
random_sampling)
return backward_lookup
def _build_lattice_vocab(self,
backward_lookup,
samples=0,
top_sampling=False,
random_sampling=False):
# vocab selection is an advanced pruning algorithm to limit the vocab space in a specified
# conversion task. It will avoid calculating softmax for the full vocab.
def lattice_vocab(backward_lookup):
return sorted(
list(
set(
sum([[node.word_idx for node in nodes]
for nodes in backward_lookup.values()], []))))
self.lattice_vocab = lattice_vocab(backward_lookup)
if samples:
if random_sampling:
samples = np.random.randint(len(self.w2i), size=samples)
self.lattice_vocab += [x for x in samples]
elif top_sampling:
self.lattice_vocab += [x for x in range(samples)]
self.lattice_vocab = sorted(list(set(self.lattice_vocab)))
#if self.lattice_vocab:
# print('{} vocab selected'.format(len(self.lattice_vocab)))
def _frame_vocab(self, frame):
if type(self.lattice_vocab) is list:
return self.lattice_vocab
elif type(self.lattice_vocab) is dict:
return self.lattice_vocab[frame]
else:
return None
def _build_current_frame(self, frame, idx):
# frame 0 contains one path that has eos_node
if idx == 0:
frame[0] = [
Path(self.backward_lookup[0][0], self.model.hidden_size)
]
return
# connect each nodes to its previous best paths, also calculate the new path probability
frame[idx] = []
for node in self.backward_lookup[idx]:
for prev_path in frame[node.start_idx]:
# shallow copy to avoid create dup objects
# note that numpy arrays like state and cell are copied also
cur_path = copy(prev_path)
cur_path.nodes = copy(prev_path.nodes)
node_prob_idx = node.word_idx
if self.lattice_vocab:
node_prob_idx = self._frame_vocab(idx).index(node_prob_idx)
cur_path.append_node(node, node_prob_idx)
frame[idx].append(cur_path)
def _predict(self, paths, vocab=None):
log_lstm = []
log_softmax = []
for i, path in enumerate(paths):
(p, logits, log_lstm_time,
log_softmax_time), s, c = self.model.predict_with_context(
[path.nodes[-1].word_idx], path.state, path.cell, vocab)
log_lstm.append(log_lstm_time)
log_softmax.append(log_softmax_time)
path.state = s
path.cell = c
path.transition_probs = p
self.perf_log_lstm.append(sum(log_lstm))
self.perf_log_softmax.append(sum(log_softmax))
def _batch_predict(self, paths, vocab=None):
# print('batch predict paths {}'.format(len(paths)))
start_time = time.time()
(pred, logits, log_lstm_time,
log_softmax_time), state, cell = self.model.predict_with_context(
[path.nodes[-1].word_idx for path in paths],
np.concatenate([path.state for path in paths], axis=0),
np.concatenate([path.cell for path in paths], axis=0), vocab)
self.perf_log_lstm.append(log_lstm_time)
self.perf_log_softmax.append(log_softmax_time)
for i, path in enumerate(paths):
path.state = np.expand_dims(state[i], axis=0)
path.cell = np.expand_dims(cell[i], axis=0)
path.transition_probs = [pred[i]]
path.logits = logits[i]
def decode(self,
input,
topN=10,
beam_width=10,
vocab_select=False,
samples=0,
top_sampling=False,
random_sampling=False):
self.backward_lookup = self._build_lattice(
input,
vocab_select=vocab_select,
samples=samples,
top_sampling=top_sampling,
random_sampling=random_sampling)
frame = {}
for i in range(len(input) + 1):
self._build_current_frame(frame, i)
if beam_width is not None:
frame[i].sort(key=lambda x: x.neg_log_prob)
frame[i] = frame[i][:beam_width]
batch_predict = True
if batch_predict:
self._batch_predict(frame[i], self._frame_vocab(i))
else:
self._predict(frame[i])
output = [(x.neg_log_prob,
[n.word for n in x.nodes if n.word != "<eos>"])
for x in frame[len(input)]]
self.perf_sen += 1
return output[:topN]
