def __init__(self): self.config = json.loads( open( os.path.join(experiment_path, str(experiment_id), 'config.json'), 'rt').read()) self.i2w = pickle.load(open(os.path.join(data_path, 'i2w.pkl'), 'rb')) self.w2i = pickle.load(open(os.path.join(data_path, 'w2i.pkl'), 'rb')) # full lexicon and reading dictionary covers all the vocab # that includes oov words to the model self.full_lexicon = pickle.load( open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb')) self.full_reading_dict = pickle.load( open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb')) self.model = LSTM_Model()
def __init__(self, experiment_id=0, comp=0): self.config = json.loads( open( os.path.join(experiment_path, str(experiment_id), 'config.json'), 'rt').read()) self._load_vocab() # full lexicon and reading dictionary covers all the vocab # that includes oov words to the model self.full_lexicon = pickle.load( open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb')) self.full_reading_dict = pickle.load( open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb')) self.model = LSTM_Model(experiment_id, comp) self.lattice_vocab = None self.perf_sen = 0 self.perf_log_lstm = [] self.perf_log_softmax = []
import greengrasssdk from pandas import read_csv import json from numpy import concatenate from joblib import load from keras.models import model_from_json from pandas import DataFrame from pandas import concat import time from model import LSTM_Model lstm = LSTM_Model() # Creating a greengrass core sdk client client = greengrasssdk.client('iot-data') data_dir = '/src/' model_dir = '/dest' def read_test_data(): dataset = read_csv(data_dir + 'pollution.csv', header=0) test_dataset.dropna(inplace=True) return dataset def publishToIotCloud(curr_row, predicted_y, actual_y): data = curr_row.to_dict(orient='records')[0] payload = json.dumps(data) client.publish( topic='pollution/data', payload=payload )
from model import LSTM_Model mod = LSTM_Model(vocab_size=122630, sent_max_len=50, valid_sent_max_len=50, embedding_size=100) model = mod.encoder_decoder() print(model.summary())
def unimodal(mode): print(('starting unimodal ', mode)) # with open('./mosei/text_glove_average.pickle', 'rb') as handle: with open('./mosei/2way/2-way-' + mode + '.pickle', 'rb') as handle: u = pickle._Unpickler(handle) u.encoding = 'latin1' # (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load() (train_data, train_label, _, _, test_data, test_label, _, train_length, _, test_length, _, _, _) = u.load() # with open('./input/' + mode + '.pickle', 'rb') as handle: # (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = pickle.load(handle) train_label = train_label.astype('int') test_label = test_label.astype('int') train_mask = np.zeros((train_data.shape[0], train_data.shape[1]), dtype='float') for i in range(len(train_length)): train_mask[i, :train_length[i]] = 1.0 test_mask = np.zeros((test_data.shape[0], test_data.shape[1]), dtype='float') for i in range(len(test_length)): test_mask[i, :test_length[i]] = 1.0 train_label, test_label = createOneHotMosei3way(train_label, test_label) attn_fusion = False print('train_mask', train_mask.shape) # print(train_mask_bool) seqlen_train = np.sum(train_mask, axis=-1) print('seqlen_train', seqlen_train.shape) seqlen_test = np.sum(test_mask, axis=-1) print('seqlen_test', seqlen_test.shape) allow_soft_placement = True log_device_placement = False # Multimodal model session_conf = tf.ConfigProto( # device_count={'GPU': gpu_count}, allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True)) gpu_device = 0 best_acc = 0 best_epoch = 0 best_loss = 1000000.0 best_epoch_loss = 0 is_unimodal = True with tf.device('/device:GPU:%d' % gpu_device): print('Using GPU - ', '/device:GPU:%d' % gpu_device) with tf.Graph().as_default(): tf.set_random_seed(seed) sess = tf.Session(config=session_conf) with sess.as_default(): model = LSTM_Model(train_data.shape[1:], 0.001, emotions=emotions, attn_fusion=attn_fusion, unimodal=is_unimodal, seed=seed) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) test_feed_dict = { model.input: test_data, model.y: test_label, model.seq_len: seqlen_test, model.mask: test_mask, model.lstm_dropout: 0.0, model.dropout: 0.0 } train_feed_dict = { model.input: train_data, model.y: train_label, model.seq_len: seqlen_train, model.mask: train_mask, model.lstm_dropout: 0.0, model.dropout: 0.0 } # print('\n\nDataset: %s' % (data)) print("\nEvaluation before training:") # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], test_feed_dict) print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format( 0, step, loss, accuracy)) for epoch in range(epochs): epoch += 1 batches = batch_iter( list( zip(train_data, train_mask, seqlen_train, train_label)), batch_size) # Training loop. For each batch... print('\nTraining epoch {}'.format(epoch)) l = [] a = [] for i, batch in tqdm(enumerate(batches)): b_train_data, b_train_mask, b_seqlen_train, b_train_label = zip( *batch) # print('batch_hist_v', len(batch_utt_v)) feed_dict = { model.input: b_train_data, model.y: b_train_label, model.seq_len: b_seqlen_train, model.mask: b_train_mask, model.lstm_dropout: 0.5, model.dropout: 0.3, } _, step, loss, accuracy = sess.run([ model.train_op, model.global_step, model.loss, model.accuracy ], feed_dict) l.append(loss) a.append(accuracy) print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format( epoch, np.average(l), np.average(a))) # Evaluation after epoch step, loss, accuracy, test_activations = sess.run([ model.global_step, model.loss, model.accuracy, model.inter1 ], test_feed_dict) loss = loss / test_label.shape[0] print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}". format(epoch, step, loss, accuracy)) if accuracy > best_acc: best_epoch = epoch best_acc = accuracy step, loss, accuracy, train_activations = sess.run([ model.global_step, model.loss, model.accuracy, model.inter1 ], train_feed_dict) unimodal_activations[mode + '_train'] = train_activations unimodal_activations[mode + '_test'] = test_activations unimodal_activations['train_mask'] = train_mask unimodal_activations['test_mask'] = test_mask unimodal_activations['train_label'] = train_label unimodal_activations['test_label'] = test_label if loss < best_loss: best_epoch_loss = epoch best_loss = loss # step, loss, accuracy, train_activations = sess.run( # [model.global_step, model.loss, model.accuracy, model.inter1], # train_feed_dict) # unimodal_activations[mode + '_train'] = train_activations # unimodal_activations[mode + '_test'] = test_activations # unimodal_activations['train_mask'] = train_mask # unimodal_activations['test_mask'] = test_mask # unimodal_activations['train_label'] = train_label # unimodal_activations['test_label'] = test_label print("\n\nBest epoch: {}\nBest test accuracy: {}".format( best_epoch, best_acc)) print("\n\nBest epoch: {}\nBest test loss: {}".format( best_epoch_loss, best_loss))
def multimodal(unimodal_activations, attn_fusion=True, enable_attn_2=False): if attn_fusion: print('With attention fusion') print("starting multimodal") # Fusion (appending) of features text_train = unimodal_activations['text_train'] audio_train = unimodal_activations['audio_train'] video_train = unimodal_activations['video_train'] text_test = unimodal_activations['text_test'] audio_test = unimodal_activations['audio_test'] video_test = unimodal_activations['video_test'] train_mask = unimodal_activations['train_mask'] test_mask = unimodal_activations['test_mask'] print('train_mask', train_mask.shape) train_label = unimodal_activations['train_label'] print('train_label', train_label.shape) test_label = unimodal_activations['test_label'] print('test_label', test_label.shape) # print(train_mask_bool) seqlen_train = np.sum(train_mask, axis=-1) print('seqlen_train', seqlen_train.shape) seqlen_test = np.sum(test_mask, axis=-1) print('seqlen_test', seqlen_test.shape) allow_soft_placement = True log_device_placement = False # Multimodal model session_conf = tf.ConfigProto( # device_count={'GPU': gpu_count}, allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True)) gpu_device = 0 best_acc = 0 best_loss_accuracy = 0 best_loss = 10000000.0 best_epoch = 0 best_epoch_loss = 0 with tf.device('/device:GPU:%d' % gpu_device): print('Using GPU - ', '/device:GPU:%d' % gpu_device) with tf.Graph().as_default(): tf.set_random_seed(seed) sess = tf.Session(config=session_conf) with sess.as_default(): model = LSTM_Model(text_train.shape[1:], 0.0001, emotions=emotions, attn_fusion=attn_fusion, unimodal=False, enable_attn_2=enable_attn_2, seed=seed) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) test_feed_dict = { model.t_input: text_test, model.a_input: audio_test, model.v_input: video_test, model.y: test_label, model.seq_len: seqlen_test, model.mask: test_mask, model.lstm_dropout: 0.0, model.lstm_inp_dropout: 0.0, model.dropout: 0.0, model.dropout_lstm_out: 0.0 } # print('\n\nDataset: %s' % (data)) print("\nEvaluation before training:") # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], test_feed_dict) print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format( 0, step, loss, accuracy)) for epoch in range(epochs): epoch += 1 batches = batch_iter( list( zip(text_train, audio_train, video_train, train_mask, seqlen_train, train_label)), batch_size) # Training loop. For each batch... print('\nTraining epoch {}'.format(epoch)) l = [] a = [] for i, batch in tqdm(enumerate(batches)): b_text_train, b_audio_train, b_video_train, b_train_mask, b_seqlen_train, b_train_label = zip( *batch) # print('batch_hist_v', len(batch_utt_v)) feed_dict = { model.t_input: b_text_train, model.a_input: b_audio_train, model.v_input: b_video_train, model.y: b_train_label, model.seq_len: b_seqlen_train, model.mask: b_train_mask, model.lstm_dropout: 0.4, model.lstm_inp_dropout: 0.0, model.dropout: 0.2, model.dropout_lstm_out: 0.2 } _, step, loss, accuracy = sess.run([ model.train_op, model.global_step, model.loss, model.accuracy ], feed_dict) l.append(loss) a.append(accuracy) print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format( epoch, np.average(l), np.average(a))) # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], test_feed_dict) print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}". format(epoch, step, loss / test_label.shape[0], accuracy)) if accuracy > best_acc: best_epoch = epoch best_acc = accuracy if loss < best_loss: best_loss = loss best_loss_accuracy = accuracy best_epoch_loss = epoch print( "\n\nBest epoch: {}\nBest test accuracy: {}\nBest epoch loss: {}\nBest test accuracy when loss is least: {}" .format(best_epoch, best_acc, best_epoch_loss, best_loss_accuracy))
import os os.environ["CUDA_VISIBLE_DEVICES"] = "0" from model import LSTM_Model import tensorflow as tf import keras.backend.tensorflow_backend as ktf config = tf.ConfigProto() config.gpu_options.allow_growth = True session = tf.Session(config=config) ktf.set_session(session) model = LSTM_Model(gpus=4) for i in range(0, model.rolling_count, 3): model.rolling(i) model.save_model()
class Decoder(): def __init__(self): self.config = json.loads( open( os.path.join(experiment_path, str(experiment_id), 'config.json'), 'rt').read()) vocab = Vocab(self.config['vocab_size']) self.i2w = vocab.i2w self.w2i = vocab.w2i # full lexicon and reading dictionary covers all the vocab # that includes oov words to the model self.full_lexicon = pickle.load( open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb')) self.full_reading_dict = pickle.load( open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb')) self.model = LSTM_Model() def _check_oov(self, word): return word not in self.w2i def _build_lattice(self, input, use_oov=False): def add_node_to_lattice(i, sub_token, id, word, prob): node = Node(i, len(sub_token), id, word, prob) # share the node in both lookup table backward_lookup[i + len(sub_token)].append(node) # backward_lookup keeps the words that ends at a frame # e.g. the input A/BC/D, 0 is preserved for <eos> # 0 1 2 3 4 # <eos> A BC D backward_lookup = defaultdict(lambda: []) eos_node = [Node(-1, 1, self.w2i['<eos>'], '<eos>')] backward_lookup[0] = eos_node for i, token in enumerate(input): for j in range(len(input) - i): sub_token = input[i:i + j + 1] if sub_token in self.full_reading_dict.keys(): for lexicon_id in self.full_reading_dict[sub_token]: word = self.full_lexicon[lexicon_id][0] oov = self._check_oov(word) if oov: # skip oov in this experiment, # note that oov affects conversion quality continue prob = 0.0 id = self.w2i[word] add_node_to_lattice(i, sub_token, id, word, prob) # put symbol directly if no match at all in reading dictionary if len(backward_lookup[i + 1]) == 0: backward_lookup[i + 1].append( Node(i, 1, self.w2i['<unk>'], input[i])) return backward_lookup def _build_current_frame(self, nodes, frame, idx): # frame 0 contains one path that has eos_node if idx == 0: frame[0] = [Path(nodes[0], self.model.hidden_size)] return # connect each nodes to its previous best paths, also calculate the new path probability frame[idx] = [] for node in nodes: for prev_path in frame[node.start_idx]: cur_paths = copy( prev_path) # shallow copy to avoid create dup objects cur_paths.nodes = copy(prev_path.nodes) node_prob_idx = node.word_idx cur_paths.append_node(node, node_prob_idx) frame[idx].append(cur_paths) def _batch_predict(self, paths): pred, state, cell = self.model.predict_with_context( [path.nodes[-1].word_idx for path in paths], np.concatenate([path.state for path in paths], axis=0), np.concatenate([path.cell for path in paths], axis=0)) for i, path in enumerate(paths): path.state = np.expand_dims(state[i], axis=0) path.cell = np.expand_dims(cell[i], axis=0) path.transition_probs = [pred[i]] def decode(self, input, topN=10, beam_width=10, use_oov=False): backward_lookup = self._build_lattice(input, use_oov) frame = {} for i in range(len(input) + 1): b_nodes = backward_lookup[i] self._build_current_frame(b_nodes, frame, i) if beam_width is not None: frame[i].sort(key=lambda x: x.neg_log_prob) frame[i] = frame[i][:beam_width] self._batch_predict(frame[i]) output = [(x.neg_log_prob, [n.word for n in x.nodes if n.word != "<eos>"]) for x in frame[len(input)]] return output[:topN]
def main(args): lr = 0.001 attn_fusion = args.attention_2 train_u1_data, train_u2_data, train_u3_data, val_u1_data, val_u2_data, val_u3_data, Y_train, Y_val, \ tr_sq_u1, tr_sq_u2, tr_sq_u3, val_sq_u1, val_sq_u2, val_sq_u3, embeddingMatrix = get_data(args) vocab_size, embedding_dim = embeddingMatrix.shape print('embeddingMatrix.shape', embeddingMatrix.shape) print('vocab_size', vocab_size) print('embedding_dim', embedding_dim) classes = Y_train.shape[-1] allow_soft_placement = True log_device_placement = False # Multimodal model session_conf = tf.ConfigProto( # device_count={'GPU': gpu_count}, allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True)) gpu_device = 0 best_acc = 0 best_epoch = 0 best_loss = 1000000.0 best_epoch_loss = 0 print("Building model...") with tf.device('/device:GPU:%d' % gpu_device): print('Using GPU - ', '/device:GPU:%d' % gpu_device) with tf.Graph().as_default(): tf.set_random_seed(seed) sess = tf.Session(config=session_conf) with sess.as_default(): model = LSTM_Model(train_u1_data.shape[1], lr, vocab_size, embedding_dim, emotions=classes, seed=seed, enable_attn_2=attn_fusion) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # init embeddings sess.run( model.embedding_init, feed_dict={model.embedding_placeholder: embeddingMatrix}) test_feed_dict = { model.input1: val_u1_data, model.input2: val_u2_data, model.input3: val_u3_data, model.y: Y_val, model.seq_len1: val_sq_u1, model.seq_len2: val_sq_u2, model.seq_len3: val_sq_u3, model.lstm_dropout: 0.0, model.lstm_inp_dropout: 0.0, model.dropout: 0.0, model.dropout_lstm_out: 0.0 } train_feed_dict = { model.input1: train_u1_data, model.input2: train_u2_data, model.input3: train_u3_data, model.y: Y_train, model.seq_len1: tr_sq_u1, model.seq_len2: tr_sq_u2, model.seq_len3: tr_sq_u3, model.lstm_dropout: 0.0, model.lstm_inp_dropout: 0.0, model.dropout: 0.0, model.dropout_lstm_out: 0.0 } # print('\n\nDataset: %s' % (data)) print("\nEvaluation before training:") # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], test_feed_dict) print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format( 0, step, loss, accuracy)) for epoch in range(epochs): epoch += 1 batches = batch_iter( list( zip(train_u1_data, train_u2_data, train_u3_data, tr_sq_u1, tr_sq_u2, tr_sq_u3, Y_train)), batch_size) # Training loop. For each batch... print('\nTraining epoch {}'.format(epoch)) l = [] a = [] for i, batch in tqdm(enumerate(batches)): b_train_u1_data, b_train_u2_data, b_train_u3_data, b_tr_sq_u1, b_tr_sq_u2, b_tr_sq_u3, b_Y_train = zip( *batch) feed_dict = { model.input1: b_train_u1_data, model.input2: b_train_u2_data, model.input3: b_train_u3_data, model.y: b_Y_train, model.seq_len1: b_tr_sq_u1, model.seq_len2: b_tr_sq_u2, model.seq_len3: b_tr_sq_u3, model.lstm_dropout: 0.5, model.lstm_inp_dropout: 0.0, model.dropout: 0.2, model.dropout_lstm_out: 0.2 } _, step, loss, accuracy = sess.run([ model.train_op, model.global_step, model.loss, model.accuracy ], feed_dict) l.append(loss) a.append(accuracy) print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format( epoch, np.average(l), np.average(a))) # Evaluation after epoch step, loss, accuracy, test_activations = sess.run([ model.global_step, model.loss, model.accuracy, model.inter1 ], test_feed_dict) loss = loss / Y_val.shape[0] print("EVAL: After epoch {}: step {}, loss {:g}, acc {:g}". format(epoch, step, loss, accuracy)) if accuracy > best_acc: best_epoch = epoch best_acc = accuracy # if epoch == 30: # step, loss, accuracy, train_activations = sess.run( # [model.global_step, model.loss, model.accuracy, model.inter1], # train_feed_dict) if loss < best_loss: best_epoch_loss = epoch best_loss = loss # step, loss, accuracy, train_activations = sess.run( # [model.global_step, model.loss, model.accuracy, model.inter1], # train_feed_dict) print("\n\nBest epoch: {}\nBest test accuracy: {}".format( best_epoch, best_acc)) print("\n\nBest epoch: {}\nBest test loss: {}".format( best_epoch_loss, best_loss))
def train(): df_ge = pd.read_csv("ge.txt", engine='python') df_ge.tail() # visualize(df_ge) train_cols = ["Open", "High", "Low", "Close", "Volume"] df_train, df_test = train_test_split(df_ge, train_size=0.8, test_size=0.2, shuffle=False) print("Train and Test size", len(df_train), len(df_test), flush=True) # scale the feature MinMax, build array x = df_train.loc[:, train_cols].values min_max_scaler = MinMaxScaler() x_train = min_max_scaler.fit_transform(x) x_test = min_max_scaler.transform(df_test.loc[:, train_cols]) x_t, y_t = build_timeseries(x_train, 3) x_t = trim_dataset(x_t, BATCH_SIZE) y_t = trim_dataset(y_t, BATCH_SIZE) x_temp, y_temp = build_timeseries(x_test, 3) x_val, x_test_t = np.split(trim_dataset(x_temp, BATCH_SIZE), 2) y_val, y_test_t = np.split(trim_dataset(y_temp, BATCH_SIZE), 2) model = LSTM_Model() optimizer = torch.optim.RMSprop(model.parameters(), lr=LR) model.train() loss_epo_list = list() loss_iter_list = list() for i in range(EPOCHS): total_iter = int(x_t.shape[0] / BATCH_SIZE) loss_avg = AverageMeter() for j in range(total_iter): input = x_t[j * BATCH_SIZE:(j + 1) * BATCH_SIZE, :, :] gt = y_t[j * BATCH_SIZE:(j + 1) * BATCH_SIZE] gt = torch.Tensor(gt) input = torch.Tensor(input) input = input.permute(1, 0, 2) output, loss = model(input, gt) loss_avg.update(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() loss_iter_list.append(loss.item()) if j % 30 == 0: print("Epoch{}:{}/{} loss:{}".format(i, j, total_iter, loss_avg.avg), flush=True) loss_epo_list.append(loss_avg.avg) with open("loss_epo.json", 'w') as f: json.dump(loss_epo_list, f, indent=4) with open("loss_iter.json", 'w') as f: json.dump(loss_iter_list, f, indent=4) torch.save({ 'loss': loss, 'state_dict': model.state_dict(), }, '{}.pth'.format('checkpoint'))
def evaluate(): model = LSTM_Model() state_dict = torch.load("checkpoint.pth") model.load_state_dict(state_dict['state_dict'], strict=True) df_ge = pd.read_csv("ge.txt", engine='python') df_ge.tail() # visualize(df_ge) train_cols = ["Open", "High", "Low", "Close", "Volume"] df_train, df_test = train_test_split(df_ge, train_size=0.8, test_size=0.2, shuffle=False) print("Train and Test size", len(df_train), len(df_test), flush=True) # scale the feature MinMax, build array x = df_train.loc[:, train_cols].values min_max_scaler = MinMaxScaler() x_train = min_max_scaler.fit_transform(x) x_test = min_max_scaler.transform(df_test.loc[:, train_cols]) x_t, y_t = build_timeseries(x_train, 3) x_t = trim_dataset(x_t, BATCH_SIZE) y_t = trim_dataset(y_t, BATCH_SIZE) # x_temp, y_temp = build_timeseries(x_test, 3) # x_val, x_test_t = np.split(trim_dataset(x_temp, BATCH_SIZE), 2) # y_val, y_test_t = np.split(trim_dataset(y_temp, BATCH_SIZE), 2) model.eval() y_gt = list() y_pred = list() x = np.array(list(range(11246 + 60, 11246 + 2812))) for i in range(x_test.shape[0] - TIME_STEPS): input = x_test[i:i + TIME_STEPS, :] gt = x_test[i + TIME_STEPS, 3] y_gt.append(gt) input = torch.Tensor(input) input = input.unsqueeze(dim=1) with torch.no_grad(): output = model(input) output = np.array(output) y_pred.append(output) y_pred = np.array(y_pred) y_gt = np.array(y_gt) mse = np.sum(np.power(y_pred - y_gt, 2)) / y_pred.shape[0] # mse: 0.00023299293330091727 print("mse: ", mse) y_pred_org = (y_pred * min_max_scaler.data_range_[3]) + min_max_scaler.data_min_[3] y_gt_org = (y_gt * min_max_scaler.data_range_[3]) + min_max_scaler.data_min_[3] plt.figure() plt.ylabel('Price (USD)') plt.xlabel('Days') plt.title('Prediction vs Real Stock Price') plt.plot(x, y_gt_org, linewidth=0.8) plt.plot(x, y_pred_org, linewidth=0.8) plt.legend(['Real', 'Prediction'], loc='upper left') plt.show()
def multimodal(unimodal_activations, data, classes, attn_fusion=True, enable_attn_2=False, use_raw=True): if use_raw: if attn_fusion: attn_fusion = False train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask = get_raw_data( data, classes) a_dim = audio_train.shape[-1] v_dim = video_train.shape[-1] t_dim = text_train.shape[-1] else: print("starting multimodal") # Fusion (appending) of features train_mask = unimodal_activations['train_mask'] test_mask = unimodal_activations['test_mask'] print('train_mask', train_mask.shape) array_default = np.empty( (train_mask.shape[0], train_mask.shape[1], 100)) t_dim = a_dim = v_dim = None #initializing the dims if ('text_train' in unimodal_activations): text_train = unimodal_activations['text_train'] text_test = unimodal_activations['text_test'] t_dim = text_train.shape[-1] else: text_train = array_default text_test = array_default if ('audio_train' in unimodal_activations): audio_train = unimodal_activations['audio_train'] audio_test = unimodal_activations['audio_test'] a_dim = audio_train.shape[-1] else: audio_train = array_default audio_test = array_default if ('video_train' in unimodal_activations): video_train = unimodal_activations['video_train'] video_test = unimodal_activations['video_test'] v_dim = video_train.shape[-1] else: video_train = array_default video_test = array_default train_label = unimodal_activations['train_label'] print('train_label', train_label.shape) test_label = unimodal_activations['test_label'] print('test_label', test_label.shape) # print(train_mask_bool) seqlen_train = np.sum(train_mask, axis=-1) print('seqlen_train', seqlen_train.shape) seqlen_test = np.sum(test_mask, axis=-1) print('seqlen_test', seqlen_test.shape) if attn_fusion: print('With attention fusion') allow_soft_placement = True log_device_placement = False # Multimodal model session_conf = tf.ConfigProto( # device_count={'GPU': gpu_count}, allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True)) gpu_device = 0 best_acc = 0 best_loss_accuracy = 0 best_loss = 10000000.0 best_epoch = 0 best_epoch_loss = 0 with tf.device('/device:GPU:%d' % gpu_device): print('Using GPU - ', '/device:GPU:%d' % gpu_device) with tf.Graph().as_default(): tf.set_random_seed(seed) sess = tf.Session(config=session_conf) with sess.as_default(): model = LSTM_Model(text_train.shape[1:], 0.0001, a_dim=a_dim, v_dim=v_dim, t_dim=t_dim, emotions=classes, attn_fusion=attn_fusion, unimodal=False, enable_attn_2=enable_attn_2, seed=seed) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) test_feed_dict = { model.y: test_label, model.seq_len: seqlen_test, model.mask: test_mask, model.lstm_dropout: 0.0, model.lstm_inp_dropout: 0.0, model.dropout: 0.0, model.dropout_lstm_out: 0.0 } if t_dim: test_feed_dict.update({model.t_input: text_test}) if a_dim: test_feed_dict.update({model.a_input: audio_test}) if v_dim: test_feed_dict.update({model.v_input: video_test}) # print('\n\nDataset: %s' % (data)) print("\nEvaluation before training:") # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], #[model.global_step, model.loss, model.f1], test_feed_dict) print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format( 0, step, loss, accuracy)) for epoch in range(epochs): epoch += 1 batches = batch_iter( list( zip(text_train, audio_train, video_train, train_mask, seqlen_train, train_label)), batch_size) # Training loop. For each batch... print('\nTraining epoch {}'.format(epoch)) l = [] a = [] for i, batch in tqdm(enumerate(batches)): b_text_train, b_audio_train, b_video_train, b_train_mask, b_seqlen_train, b_train_label = zip( *batch) # print('batch_hist_v', len(batch_utt_v)) feed_dict = {} if t_dim: feed_dict.update({model.t_input: b_text_train}) if a_dim: feed_dict.update({model.a_input: b_audio_train}) if v_dim: feed_dict.update({model.v_input: b_video_train}) feed_dict.update({ model.y: b_train_label, model.seq_len: b_seqlen_train, model.mask: b_train_mask, model.lstm_dropout: 0.4, model.lstm_inp_dropout: 0.0, model.dropout: 0.2, model.dropout_lstm_out: 0.2 }) _, step, loss, accuracy = sess.run([ model.train_op, model.global_step, model.loss, model.accuracy ], feed_dict) l.append(loss) a.append(accuracy) print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format( epoch, np.average(l), np.average(a))) # Evaluation after epoch step, loss, accuracy, preds, y, mask = sess.run([ model.global_step, model.loss, model.accuracy, model.preds, model.y, model.mask ], test_feed_dict) f1 = f1_score(np.ndarray.flatten( tf.argmax(y, -1, output_type=tf.int32).eval()), np.ndarray.flatten( tf.argmax(preds, -1, output_type=tf.int32).eval()), sample_weight=np.ndarray.flatten( tf.cast(mask, tf.int32).eval()), average="weighted") print( "EVAL: After epoch {}: step {}, loss {:g}, acc {:g}, f1 {:g}}" .format(epoch, step, loss / test_label.shape[0], accuracy, f1)) if accuracy > best_acc: best_epoch = epoch best_acc = accuracy if loss < best_loss: best_loss = loss best_loss_accuracy = accuracy best_epoch_loss = epoch print( "\n\nBest epoch: {}\nBest test accuracy: {}\nBest epoch loss: {}\nBest test accuracy when loss is least: {}" .format(best_epoch, best_acc, best_epoch_loss, best_loss_accuracy))
def unimodal(mode, data, classes): print(('starting unimodal ', mode)) # with open('./mosei/text_glove_average.pickle', 'rb') as handle: if data == 'mosei' or data == 'mosi': with open( './dataset/{0}/raw/{1}_{2}way.pickle'.format( data, mode, classes), 'rb') as handle: u = pickle._Unpickler(handle) u.encoding = 'latin1' # (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load() if data == 'mosei': (train_data, train_label, _, _, test_data, test_label, _, train_length, _, test_length, _, _, _) = u.load() if classes == '2': train_label, test_label = createOneHotMosei2way( train_label, test_label) elif data == 'mosi': (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load() train_label = train_label.astype('int') test_label = test_label.astype('int') train_label, test_label = createOneHot(train_label, test_label) else: raise NotImplementedError('Unknown dataset...') train_label = train_label.astype('int') test_label = test_label.astype('int') train_mask = np.zeros((train_data.shape[0], train_data.shape[1]), dtype='float') for i in range(len(train_length)): train_mask[i, :train_length[i]] = 1.0 test_mask = np.zeros((test_data.shape[0], test_data.shape[1]), dtype='float') for i in range(len(test_length)): test_mask[i, :test_length[i]] = 1.0 elif data == 'iemocap': train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask = get_raw_data( data, classes) if mode == 'text': train_data = text_train test_data = text_test elif mode == 'audio': train_data = audio_train test_data = audio_test elif mode == 'video': train_data = video_train test_data = video_test elif data == 'cognimuse': maxlen = 368 train_data = pickle.load(open("X_train_np.pkl", "rb")).numpy() train_label = pickle.load(open("Y_train_np.pkl", "rb")).numpy() test_data = pickle.load(open("X_test_np.pkl", "rb")).numpy() test_label = pickle.load(open("Y_test_np.pkl", "rb")).numpy() train_length = pickle.load(open("train_length.pkl", "rb")) test_length = pickle.load(open("test_length.pkl", "rb")) #train_label = train_label.astype('int') #test_label = test_label.astype('int') train_mask = np.zeros((train_data.shape[0], train_data.shape[1]), dtype='float') for i in range(len(train_length)): train_mask[i, :train_length[i]] = 1.0 test_mask = np.zeros((test_data.shape[0], test_data.shape[1]), dtype='float') for i in range(len(test_length)): test_mask[i, :test_length[i]] = 1.0 # train_label, test_label = createOneHotMosei3way(train_label, test_label) attn_fusion = False print('train_mask', train_mask.shape) # print(train_mask_bool) seqlen_train = np.sum(train_mask, axis=-1) print('seqlen_train', seqlen_train.shape) seqlen_test = np.sum(test_mask, axis=-1) print('seqlen_test', seqlen_test.shape) allow_soft_placement = True log_device_placement = False # Multimodal model session_conf = tf.ConfigProto( # device_count={'GPU': gpu_count}, allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement, gpu_options=tf.GPUOptions(allow_growth=True)) gpu_device = 0 best_acc = 0 best_f1 = 0 best_epoch = 0 best_loss = 1000000.0 best_epoch_loss = 0 is_unimodal = True with tf.device('/device:GPU:%d' % gpu_device): print('Using GPU - ', '/device:GPU:%d' % gpu_device) with tf.Graph().as_default(): tf.set_random_seed(seed) sess = tf.Session(config=session_conf) with sess.as_default(): model = LSTM_Model(train_data.shape[1:], 0.003, a_dim=0, v_dim=0, t_dim=0, emotions=classes, attn_fusion=attn_fusion, unimodal=is_unimodal, seed=seed) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) test_feed_dict = { model.input: test_data, model.y: test_label, model.seq_len: seqlen_test, model.mask: test_mask, model.lstm_dropout: 0.0, model.lstm_inp_dropout: 0.0, model.dropout: 0.0, model.dropout_lstm_out: 0.0 } train_feed_dict = { model.input: train_data, model.y: train_label, model.seq_len: seqlen_train, model.mask: train_mask, model.lstm_dropout: 0.4, model.lstm_inp_dropout: 0.0, model.dropout: 0.2, model.dropout_lstm_out: 0.2 } # print('\n\nDataset: %s' % (data)) print("\nEvaluation before training:") # Evaluation after epoch step, loss, accuracy = sess.run( [model.global_step, model.loss, model.accuracy], #model.global_step, model.loss, model.f1], test_feed_dict) print(accuracy) #accuracy=accuracy[0] #print("EVAL: epoch {}: step {}, loss {:g}, acc {:g}".format(0, step, loss, accuracy)) for epoch in range(epochs): epoch += 1 print(len(train_data)) batches = batch_iter( list( zip(train_data, train_mask, seqlen_train, train_label)), batch_size) # Training loop. For each batch... print('\nTraining epoch {}'.format(epoch)) l = [] a = [] for i, batch in tqdm(enumerate(batches)): b_train_data, b_train_mask, b_seqlen_train, b_train_label = zip( *batch) feed_dict = { model.input: b_train_data, model.y: b_train_label, model.seq_len: b_seqlen_train, model.mask: b_train_mask, model.lstm_dropout: 0.4, model.lstm_inp_dropout: 0.0, model.dropout: 0.2, model.dropout_lstm_out: 0.2 } _, step, loss, accuracy = sess.run( [ model.train_op, model.global_step, model.loss, model.accuracy ], #[model.train_op, model.global_step, model.loss, model.f1], feed_dict) #accuracy=accuracy[0] l.append(loss) a.append(accuracy) print("\t \tEpoch {}:, loss {:g}, accuracy {:g}".format( epoch, np.average(l), np.average(a))) # Evaluation after epoch #step, loss, accuracy, preds, y, mask = sess.run( #[model.global_step, model.loss, model.accuracy, model.preds, model.y, model.mask], #test_feed_dict) step, loss, accuracy, test_activations, preds, y, mask = sess.run( [ model.global_step, model.loss, model.accuracy, model.inter1, model.preds, model.y, model.mask ], #[model.global_step, model.loss, model.f1, model.inter1], test_feed_dict) #accuracy=accuracy[0] loss = loss / test_label.shape[0] f1 = f1_score(np.ndarray.flatten( tf.argmax(y, -1, output_type=tf.int32).eval()), np.ndarray.flatten( tf.argmax(preds, -1, output_type=tf.int32).eval()), sample_weight=np.ndarray.flatten( tf.cast(mask, tf.int32).eval()), average="weighted") auc = roc_auc_score(np.ndarray.flatten( tf.argmax(y, -1, output_type=tf.int32).eval()), np.ndarray.flatten( tf.argmax( preds, -1, output_type=tf.int32).eval()), sample_weight=np.ndarray.flatten( tf.cast(mask, tf.int32).eval()), average="weighted") print( "EVAL: After epoch {}: step {}, loss {:g}, acc {:g},f1 {:g},auc {:g}" .format(epoch, step, loss, accuracy, f1, auc)) if accuracy > best_acc: #best_epoch = epoch best_acc = accuracy if f1 > best_f1: best_epoch = epoch best_f1 = f1 if epoch == 30: step, loss, accuracy, train_activations = sess.run( [ model.global_step, model.loss, model.accuracy, model.inter1 ], #[model.global_step, model.loss, model.f1, model.inter1], train_feed_dict) #accuracy=accuracy[0] unimodal_activations[mode + '_train'] = train_activations unimodal_activations[mode + '_test'] = test_activations unimodal_activations['train_mask'] = train_mask unimodal_activations['test_mask'] = test_mask unimodal_activations['train_label'] = train_label unimodal_activations['test_label'] = test_label if loss < best_loss: best_epoch_loss = epoch best_loss = loss # step, loss, accuracy, train_activations = sess.run( # [model.global_step, model.loss, model.accuracy, model.inter1], # train_feed_dict) # unimodal_activations[mode + '_train'] = train_activations # unimodal_activations[mode + '_test'] = test_activations # unimodal_activations['train_mask'] = train_mask # unimodal_activations['test_mask'] = test_mask # unimodal_activations['train_label'] = train_label # unimodal_activations['test_label'] = test_label print("\n\nBest epoch: {}\nBest test accuracy: {}".format( best_epoch, best_acc)) print("\n\nBest epoch: {}\nBest test loss: {}".format( best_epoch_loss, best_loss)) print("\n\nBest epoch: {}\nBest test f1: {}".format( best_epoch, best_f1))
class Decoder(): def __init__(self, experiment_id=0, comp=0): self.config = json.loads( open( os.path.join(experiment_path, str(experiment_id), 'config.json'), 'rt').read()) self._load_vocab() # full lexicon and reading dictionary covers all the vocab # that includes oov words to the model self.full_lexicon = pickle.load( open(os.path.join(root_path, 'data', 'lexicon.pkl'), 'rb')) self.full_reading_dict = pickle.load( open(os.path.join(root_path, 'data', 'reading_dict.pkl'), 'rb')) self.model = LSTM_Model(experiment_id, comp) self.lattice_vocab = None self.perf_sen = 0 self.perf_log_lstm = [] self.perf_log_softmax = [] def _load_vocab(self): self.vocab = Vocab(self.config['vocab_size']) self.i2w = self.vocab.i2w self.w2i = self.vocab.w2i def _check_oov(self, word): return word not in self.w2i def _build_lattice(self, input, vocab_select=False, samples=0, top_sampling=False, random_sampling=False): def add_node_to_lattice(i, sub_token, id, word, prob): node = Node(i, len(sub_token), id, word, prob) # share the node in both lookup table backward_lookup[i + len(sub_token)].append(node) # backward_lookup keeps the words that ends at a frame # e.g. the input A/BC/D, 0 is preserved for <eos> # 0 1 2 3 4 # <eos> A BC D backward_lookup = defaultdict(lambda: []) eos_node = [Node(-1, 1, self.w2i['<eos>'], '<eos>')] backward_lookup[0] = eos_node for i, token in enumerate(input): for j in range(len(input) - i): sub_token = input[i:i + j + 1] if sub_token in self.full_reading_dict.keys(): word_set = set() for lexicon_id in sorted( self.full_reading_dict[sub_token]): word = self.full_lexicon[lexicon_id][0] oov = self._check_oov(word) if oov: # skip oov in this experiment, # note that oov affects conversion quality continue prob = 0.0 if self.config['char_rnn']: oov = self._char_check_oov(word) if oov: # skip oov in this experiment, # note that oov affects conversion quality continue # char rnn case, we still build lattice use word, keep the first char as index of the node word = word.split('/')[0] if word in word_set: continue if len(word_set) > 200: continue word_set.add(word) id = self.w2i[word[0]] else: id = self.w2i[word] add_node_to_lattice(i, sub_token, id, word, prob) # put symbol directly if no match at all in reading dictionary if len(backward_lookup[i + 1]) == 0: backward_lookup[i + 1].append( Node(i, 1, self.w2i['<unk>'], input[i])) if vocab_select: self._build_lattice_vocab(backward_lookup, samples, top_sampling, random_sampling) return backward_lookup def _build_lattice_vocab(self, backward_lookup, samples=0, top_sampling=False, random_sampling=False): # vocab selection is an advanced pruning algorithm to limit the vocab space in a specified # conversion task. It will avoid calculating softmax for the full vocab. def lattice_vocab(backward_lookup): return sorted( list( set( sum([[node.word_idx for node in nodes] for nodes in backward_lookup.values()], [])))) self.lattice_vocab = lattice_vocab(backward_lookup) if samples: if random_sampling: samples = np.random.randint(len(self.w2i), size=samples) self.lattice_vocab += [x for x in samples] elif top_sampling: self.lattice_vocab += [x for x in range(samples)] self.lattice_vocab = sorted(list(set(self.lattice_vocab))) #if self.lattice_vocab: # print('{} vocab selected'.format(len(self.lattice_vocab))) def _frame_vocab(self, frame): if type(self.lattice_vocab) is list: return self.lattice_vocab elif type(self.lattice_vocab) is dict: return self.lattice_vocab[frame] else: return None def _build_current_frame(self, frame, idx): # frame 0 contains one path that has eos_node if idx == 0: frame[0] = [ Path(self.backward_lookup[0][0], self.model.hidden_size) ] return # connect each nodes to its previous best paths, also calculate the new path probability frame[idx] = [] for node in self.backward_lookup[idx]: for prev_path in frame[node.start_idx]: # shallow copy to avoid create dup objects # note that numpy arrays like state and cell are copied also cur_path = copy(prev_path) cur_path.nodes = copy(prev_path.nodes) node_prob_idx = node.word_idx if self.lattice_vocab: node_prob_idx = self._frame_vocab(idx).index(node_prob_idx) cur_path.append_node(node, node_prob_idx) frame[idx].append(cur_path) def _predict(self, paths, vocab=None): log_lstm = [] log_softmax = [] for i, path in enumerate(paths): (p, logits, log_lstm_time, log_softmax_time), s, c = self.model.predict_with_context( [path.nodes[-1].word_idx], path.state, path.cell, vocab) log_lstm.append(log_lstm_time) log_softmax.append(log_softmax_time) path.state = s path.cell = c path.transition_probs = p self.perf_log_lstm.append(sum(log_lstm)) self.perf_log_softmax.append(sum(log_softmax)) def _batch_predict(self, paths, vocab=None): # print('batch predict paths {}'.format(len(paths))) start_time = time.time() (pred, logits, log_lstm_time, log_softmax_time), state, cell = self.model.predict_with_context( [path.nodes[-1].word_idx for path in paths], np.concatenate([path.state for path in paths], axis=0), np.concatenate([path.cell for path in paths], axis=0), vocab) self.perf_log_lstm.append(log_lstm_time) self.perf_log_softmax.append(log_softmax_time) for i, path in enumerate(paths): path.state = np.expand_dims(state[i], axis=0) path.cell = np.expand_dims(cell[i], axis=0) path.transition_probs = [pred[i]] path.logits = logits[i] def decode(self, input, topN=10, beam_width=10, vocab_select=False, samples=0, top_sampling=False, random_sampling=False): self.backward_lookup = self._build_lattice( input, vocab_select=vocab_select, samples=samples, top_sampling=top_sampling, random_sampling=random_sampling) frame = {} for i in range(len(input) + 1): self._build_current_frame(frame, i) if beam_width is not None: frame[i].sort(key=lambda x: x.neg_log_prob) frame[i] = frame[i][:beam_width] batch_predict = True if batch_predict: self._batch_predict(frame[i], self._frame_vocab(i)) else: self._predict(frame[i]) output = [(x.neg_log_prob, [n.word for n in x.nodes if n.word != "<eos>"]) for x in frame[len(input)]] self.perf_sen += 1 return output[:topN]