def __init__(self, dataset=None):
if dataset == None:
self.dataset = MOSI()
else:
self.dataset = dataset
self.train_ids = self.dataset.train()
self.valid_ids = self.dataset.valid()
self.test_ids = self.dataset.test()
self.sentiments = self.dataset.sentiments()
np.random.seed(seed)
import os
os.environ['PYTHONHASHSEED'] = '0'
import tensorflow as tf
tf.set_random_seed(seed)
from joblib import Parallel, delayed
import multiprocessing
num_cores = multiprocessing.cpu_count()
mode = sys.argv[1]
task = sys.argv[2]
# Download the data if not present
mosei = MOSI()
embeddings = mosei.embeddings()
if mode == "all" or mode == "AV" or mode == "VT" or mode == "V":
facet = mosei.facet()
if mode == "all" or mode == "AT" or mode == "AV" or mode == "A":
covarep = mosei.covarep()
sentiments = mosei.sentiments()
emotions = mosei.emotions()
train_ids = mosei.train()
valid_ids = mosei.valid()
test_ids = mosei.test()
# Merge different features and do word level feature alignment (align according to timestamps of embeddings)
if mode == "all" or mode == "AV":
bimodal = Dataset.merge(embeddings, facet)
trimodal = Dataset.merge(bimodal, covarep)
return K.sum(K.mean(fsp*fst,axis=0)/(devP*devT))
# meta parameters
maxlen = 15 # Each utterance will be truncated/padded to 15 words
batch_size = 128
nb_epoch = 1000 # number of total epochs to train the model
# if the validation loss isn't decreasing for a number of epochs, stop training to prevent over-fitting
early_stopping = EarlyStopping(monitor='val_loss', patience=5)
opt_func = Adamax(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # optimization function
loss_func = 'mae' # loss function
metr = 'mae' # evaluation metric
# Download the data if not present
mosi = MOSI()
embeddings = mosi.embeddings() # features
sentiments = mosi.sentiments() # Valence labels
train_ids = mosi.train()
valid_ids = mosi.valid()
test_ids = mosi.test()
# Some data preprocessing
x_train = []
y_train = []
x_valid = []
y_valid = []
x_test = []
y_test = []
print("Preparing train and test data...")
def get_data(max_len_audio=20, max_len_text=15, max_len_visual=20):
mosi = MOSI()
embeddings = mosi.embeddings()
facet = mosi.facet()
covarep = mosi.covarep()
sentiments = mosi.sentiments(
) # sentiment labels, real-valued. for this tutorial we'll binarize them
train_ids = mosi.train()
valid_ids = mosi.valid()
test_ids = mosi.test()
# sort through all the video ID, segment ID pairs
train_set_ids = []
for vid in train_ids:
for sid in embeddings['embeddings'][vid].keys():
if embeddings['embeddings'][vid][sid] and facet['facet'][vid][
sid] and covarep['covarep'][vid][sid]:
train_set_ids.append((vid, sid))
valid_set_ids = []
for vid in valid_ids:
for sid in embeddings['embeddings'][vid].keys():
if embeddings['embeddings'][vid][sid] and facet['facet'][vid][
sid] and covarep['covarep'][vid][sid]:
valid_set_ids.append((vid, sid))
test_set_ids = []
for vid in test_ids:
for sid in embeddings['embeddings'][vid].keys():
if embeddings['embeddings'][vid][sid] and facet['facet'][vid][
sid] and covarep['covarep'][vid][sid]:
test_set_ids.append((vid, sid))
# partition the training, valid and tesembeddingsall sequences will be padded/truncated to 15 steps
# data will have shape (dataset_size, max_len, feature_dim)
max_len = max_len_audio
train_set_audio = np.stack([
pad(covarep['covarep'][vid][sid], max_len)
for (vid, sid) in train_set_ids if covarep['covarep'][vid][sid]
],
axis=0)
valid_set_audio = np.stack([
pad(covarep['covarep'][vid][sid], max_len)
for (vid, sid) in valid_set_ids if covarep['covarep'][vid][sid]
],
axis=0)
test_set_audio = np.stack([
pad(covarep['covarep'][vid][sid], max_len)
for (vid, sid) in test_set_ids if covarep['covarep'][vid][sid]
],
axis=0)
max_len = max_len_visual
train_set_visual = np.stack([
pad(facet['facet'][vid][sid], max_len) for (vid, sid) in train_set_ids
],
axis=0)
valid_set_visual = np.stack([
pad(facet['facet'][vid][sid], max_len) for (vid, sid) in valid_set_ids
],
axis=0)
test_set_visual = np.stack([
pad(facet['facet'][vid][sid], max_len) for (vid, sid) in test_set_ids
],
axis=0)
max_len = max_len_text
train_set_text = np.stack([
pad(embeddings['embeddings'][vid][sid], max_len)
for (vid, sid) in train_set_ids
],
axis=0)
valid_set_text = np.stack([
pad(embeddings['embeddings'][vid][sid], max_len)
for (vid, sid) in valid_set_ids
],
axis=0)
test_set_text = np.stack([
pad(embeddings['embeddings'][vid][sid], max_len)
for (vid, sid) in test_set_ids
],
axis=0)
# binarize the sentiment scores for binary classification task
y_train = np.array([sentiments[vid][sid]
for (vid, sid) in train_set_ids]) > 0
y_valid = np.array([sentiments[vid][sid]
for (vid, sid) in valid_set_ids]) > 0
y_test = np.array([sentiments[vid][sid]
for (vid, sid) in test_set_ids]) > 0
# train_set_audio = train_set_audio[:,:,1:35]
# valid_set_audio = valid_set_audio[:,:,1:35]
# test_set_audio = test_set_audio[:,:,1:35]
visual_max = np.max(np.max(np.abs(train_set_visual), axis=0), axis=0)
visual_max[visual_max ==
0] = 1 # if the maximum is 0 we don't normalize this dimension
train_set_visual = train_set_visual / visual_max
valid_set_visual = valid_set_visual / visual_max
test_set_visual = test_set_visual / visual_max
train_set_visual[train_set_visual != train_set_visual] = 0
valid_set_visual[valid_set_visual != valid_set_visual] = 0
test_set_visual[test_set_visual != test_set_visual] = 0
audio_max = np.max(np.max(np.abs(train_set_audio), axis=0), axis=0)
audio_max[audio_max == 0] = 1
train_set_audio = train_set_audio / audio_max
valid_set_audio = valid_set_audio / audio_max
test_set_audio = test_set_audio / audio_max
train_set_audio[train_set_audio != train_set_audio] = 0
valid_set_audio[valid_set_audio != valid_set_audio] = 0
test_set_audio[test_set_audio != test_set_audio] = 0
return train_set_audio, valid_set_audio, test_set_audio, train_set_text, valid_set_text, test_set_text, train_set_visual, valid_set_visual, test_set_visual, y_train, y_valid, y_test