def __init__(self, opt):
super(LSTHM, self).__init__()
self.input_dims = opt.input_dims
self.output_dim = opt.output_dim
if type(opt.hidden_dims) == int:
self.hidden_dims = [opt.hidden_dims]
else:
self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')]
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table,
dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(embedding_matrix,
freeze=False)
self.num_modalities = len(self.input_dims)
self.device = opt.device
self.hybrid_in_size = opt.hybrid_in_size
self.hybrid_cell_size = opt.hybrid_cell_size
self.hybrid_dropout_rate = opt.hybrid_dropout_rate
self.output_cell_dim = opt.output_cell_dim
self.output_dropout_rate = opt.output_dropout_rate
self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim, input_dim,self.hybrid_in_size, \
device = self.device) for hidden_dim, input_dim \
in zip(self.hidden_dims, self.input_dims)])
self.fc_hybrid = SimpleNet(sum(self.hidden_dims),
self.hybrid_cell_size,
self.hybrid_dropout_rate,
self.hybrid_in_size, nn.Softmax(dim=1))
self.fc_out = SimpleNet(
sum(self.hidden_dims) + self.hybrid_in_size, self.output_cell_dim,
self.output_dropout_rate, self.output_dim)
def __init__(self, opt):
super(QMultiTask, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
self.features = opt.features
# MELD data
# The one-hot vectors are not the global user ID
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes_emo = opt.output_dim_emo
self.n_classes_act = opt.output_dim_act
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([
PositionEmbedding(
self.embed_dim, input_dim=self.speaker_num, device=self.device)
] * len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.measurement_emotion = QMeasurement(self.embed_dim)
self.measurement_act = QMeasurement(self.embed_dim)
self.fc_out_emo = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes_emo,
output_activation=nn.Tanh())
self.fc_out_act = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes_act,
output_activation=nn.Tanh())
self.num_layers = opt.num_layers
#self.rnn=nn.ModuleList([QRNNCell(self.embed_dim, device = self.device)]*self.num_layers)
self.RNNs = nn.ModuleList([
QRNN(self.embed_dim, self.device, self.num_layers)
for i in range(len(opt.features))
])
self.rnn_outer = QOuter()
self.action_qrnn = QRNN(self.embed_dim, self.device, self.num_layers)
def __init__(self, opt):
super(QMPN, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
if type(opt.embed_dims) == int:
self.embed_dims = [opt.embed_dims]
self.embed_dims = [int(s) for s in opt.embed_dims.split(',')]
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
# MELD data
# The one-hot vectors are not the global user ID
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes = opt.output_dim
self.projections = nn.ModuleList([
nn.Linear(dim, embed_dim)
for dim, embed_dim in zip(self.input_dims, self.embed_dims)
])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([
PositionEmbedding(embed_dim,
input_dim=self.speaker_num,
device=self.device)
for embed_dim in self.embed_dims
])
self.out_dropout_rate = opt.out_dropout_rate
self.num_layers = opt.num_layers
self.state_product = QProduct()
self.recurrent_dim = 1
for embed_dim in self.embed_dims:
self.recurrent_dim = self.recurrent_dim * embed_dim
self.recurrent_cells = nn.ModuleList(
[QRNNCell(self.recurrent_dim, device=self.device)] *
self.num_layers)
self.out_dropout = QDropout(p=self.out_dropout_rate)
self.activation = QActivation(scale_factor=1, beta=0.8)
self.measurement = QMeasurement(self.recurrent_dim)
self.fc_out = SimpleNet(self.recurrent_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
def __init__(self, opt):
super(RMFN, self).__init__()
self.input_dims = opt.input_dims
self.output_dim = opt.output_dim
if type(opt.hidden_dims) == int:
self.hidden_dims = [opt.hidden_dims]
else:
self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')]
self.num_modalities = len(self.input_dims)
self.steps = opt.steps
self.device = opt.device
self.compression_cell_dim = opt.compression_cell_dim
self.compression_dropout_rate = opt.compression_dropout_rate
self.hlt_memory_init_cell_dim = opt.hlt_memory_init_cell_dim
self.hlt_memory_init_dropout_rate = opt.hlt_memory_init_dropout_rate
self.output_cell_dim = opt.output_cell_dim
self.output_dropout_rate = opt.output_dropout_rate
# initialize the LSTHM for each modality
self.compressed_dim = opt.compressed_dim
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table,
dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(
embedding_matrix, freeze=not opt.embedding_trainable)
self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim,input_dim,self.compressed_dim,self.device) \
for input_dim,hidden_dim in zip(self.input_dims,self.hidden_dims)])
self.hlt_memory_init_net = SimpleNet(sum(self.hidden_dims),
self.hlt_memory_init_cell_dim,
self.hlt_memory_init_dropout_rate,
sum(self.hidden_dims),
nn.Sigmoid())
self.compression_net = SimpleNet(self.steps * sum(self.hidden_dims),
self.compression_cell_dim,
self.compression_dropout_rate,
self.compressed_dim, nn.Sigmoid())
self.recurrent_fusion = RecurrentFusion(self.hidden_dims,
self.hlt_memory_init_net,
self.compression_net,
self.steps, self.device)
self.fc_out = SimpleNet(self.compressed_dim + sum(self.hidden_dims),
self.output_cell_dim, self.output_dropout_rate,
self.output_dim)
def __init__(self, opt):
super(QAttN, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.n_classes = opt.output_dim
self.output_cell_dim = opt.output_cell_dim
self.dataset_name = opt.dataset_name
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.phase_embeddings = nn.ModuleList([
PositionEmbedding(self.embed_dim, input_dim=1, device=self.device)
] * len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.attention = QAttention()
self.num_modalities = len(self.input_dims)
#self.out_dropout = QDropout(p=self.out_dropout_rate)
#self.dense = QDense(self.embed_dim, self.n_classes)
self.measurement_type = opt.measurement_type
if opt.measurement_type == 'quantum':
self.measurement = QMeasurement(self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim * self.num_modalities,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
else:
self.measurement = ComplexMeasurement(self.embed_dim *
self.num_modalities,
units=20)
self.fc_out = SimpleNet(20,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
def train_models(trainX, testX, trainY, testY):
print("[INFO] initialize models")
models = []
models.append(SimpleNet(width, height, depth, classes))
models.append(LeNet5(width, height, depth, classes))
print("[INFO] the number of models :", len(models))
Hs = []
# 遍历模型训练个数
for i in np.arange(0, len(models)):
# 初始化优化器和模型
print("[INFO] training model {}/{}".format(i + 1, len(models)))
# opt = Adam(lr=lr, decay=1e-4 / epochs)
# opt = SGD(lr=0.001,decay=0.01/ 40,momentum=0.9,
# nesterov=True)
# 训练网络
H = models[i].fit_generator(aug.flow(trainX, trainY, batch_size=bs),
validation_data=(testX, testY),
epochs=epochs,
steps_per_epoch=len(trainX) // bs,
verbose=1)
# 将模型保存到磁盘中
p = ['save_models', "model_{}.model".format(i)]
models[i].save(os.path.sep.join(p))
Hs.append(models[i])
# plot the training loss and accuracy
N = epochs
p = ['model_{}.png'.format(i)]
plt.style.use('ggplot')
plt.figure()
plt.plot(np.arange(0, N), H.history['loss'], label='train_loss')
plt.plot(np.arange(0, N), H.history['val_loss'], label='val_loss')
plt.plot(np.arange(0, N), H.history['acc'], label='train-acc')
plt.plot(np.arange(0, N), H.history['val_acc'], label='val-acc')
plt.title("Training Loss and Accuracy for model {}".format(i))
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.savefig(os.path.sep.join(p))
plt.close()
def __init__(self, opt):
super(EFLSTM, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.hidden_dim = opt.hidden_dim
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table,
dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(
embedding_matrix, freeze=not opt.embedding_trainable)
if self.hidden_dim == -1:
self.hidden_dim = self.total_input_dim
self.output_dim = opt.output_dim
self.output_cell_dim = opt.output_cell_dim
self.output_dropout_rate = opt.output_dropout_rate
self.lstm = nn.LSTMCell(self.total_input_dim, self.hidden_dim)
self.fc_out = SimpleNet(self.hidden_dim, self.output_cell_dim,
self.output_dropout_rate, self.output_dim)
def __init__(self, opt): #, n_layers, bidirectional, dropout):
super(LFLSTM, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
if type(opt.hidden_dims) == int:
self.hidden_dims = [opt.hidden_dims]
else:
self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')]
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(embedding_matrix, freeze=False)
self.num_modalities = len(self.input_dims)
self.output_dim = opt.output_dim
self.output_cell_dim = opt.output_cell_dim
self.output_dropout_rate = opt.output_dropout_rate
self.lstms = nn.ModuleList([nn.LSTMCell(input_dim, hidden_dim) \
for input_dim, hidden_dim in \
zip(self.input_dims, self.hidden_dims)])
self.fc_out = SimpleNet(sum(self.hidden_dims),self.output_cell_dim,
self.output_dropout_rate,self.output_dim)
class QAttN(nn.Module):
def __init__(self, opt):
super(QAttN, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.n_classes = opt.output_dim
self.output_cell_dim = opt.output_cell_dim
self.dataset_name = opt.dataset_name
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.phase_embeddings = nn.ModuleList([
PositionEmbedding(self.embed_dim, input_dim=1, device=self.device)
] * len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.attention = QAttention()
self.num_modalities = len(self.input_dims)
#self.out_dropout = QDropout(p=self.out_dropout_rate)
#self.dense = QDense(self.embed_dim, self.n_classes)
self.measurement_type = opt.measurement_type
if opt.measurement_type == 'quantum':
self.measurement = QMeasurement(self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim * self.num_modalities,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
else:
self.measurement = ComplexMeasurement(self.embed_dim *
self.num_modalities,
units=20)
self.fc_out = SimpleNet(20,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
def get_params(self):
unitary_params = []
remaining_params = []
remaining_params.extend(list(self.projections.parameters()))
remaining_params.extend(list(self.phase_embeddings.parameters()))
if self.measurement_type == 'quantum':
unitary_params.extend(list(self.measurement.parameters()))
else:
remaining_params.extend(list(self.measurement.parameters()))
remaining_params.extend(list(self.fc_out.parameters()))
return unitary_params, remaining_params
def forward(self, in_modalities):
smask = in_modalities[-2] # Speaker ids
in_modalities = in_modalities[:-2]
batch_size = in_modalities[0].shape[0]
time_stamps = in_modalities[0].shape[1]
# Project All modalities of each utterance to the same space
#utterance_reps = [nn.Tanh()(projection(x)) for x, projection in zip(in_modalities,self.projections)]
utterance_reps = [
nn.ReLU()(projection(x))
for x, projection in zip(in_modalities, self.projections)
]
# Take the amplitudes
# multiply with modality specific vectors to construct weights
weights = [self.norm(rep) for rep in utterance_reps]
amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps]
phases = [
phase_embed(smask.argmax(dim=-1))
for phase_embed in self.phase_embeddings
]
unimodal_pure = [
self.multiply([phase, amplitude])
for phase, amplitude in zip(phases, amplitudes)
]
unimodal_matrices = [self.outer(s) for s in unimodal_pure]
probs = []
# For each modality
# we mix the remaining modalities as queries (to_be_measured systems)
# and treat the modality features as keys (measurement operators)
for ind in range(self.num_modalities):
# Obtain mixed states for the rest modalities
other_weights = [
weights[i] for i in range(self.num_modalities) if not i == ind
]
mixture_weights = F.softmax(torch.cat(other_weights, dim=-1),
dim=-1)
other_states = [
unimodal_matrices[i] for i in range(self.num_modalities)
if not i == ind
]
q_states = self.mixture([other_states, mixture_weights])
# Obtain pure states and weights for the modality of interest
k_weights = weights[ind]
k_states = unimodal_pure[ind]
# Compute cross-modal interactions, output being a list of post-measurement states
in_states = self.attention(q_states, k_states, k_weights)
# Apply measurement to each output state
output = []
for _h in in_states:
measurement_probs = self.measurement(_h)
output.append(measurement_probs)
probs.append(output)
# Concatenate the measurement probabilities per-time-stamp
concat_probs = [
self.fc_out(torch.cat(output_t, dim=-1))
for output_t in zip(*probs)
]
concat_probs = torch.stack(concat_probs, dim=-2)
log_prob = F.log_softmax(concat_probs, 2) # batch, seq_len, n_classes
return log_prob
def __init__(self, opt):
super(MFN, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.num_modalities = len(self.input_dims)
if type(opt.hidden_dims) == int:
self.hidden_dims = [opt.hidden_dims]
else:
self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')]
self.total_cell_dim = sum(self.hidden_dims)
self.memory_dim = opt.memory_dim
self.window_dim = opt.window_dim
self.output_dim = opt.output_dim
self.attn_in_dim = self.total_cell_dim * self.window_dim
self.gamma_in_dim = self.attn_in_dim + self.memory_dim
self.final_fc_in_dim = self.total_cell_dim + self.memory_dim
if type(opt.attn_cell_dims) == int:
self.attn_cell_dims = [opt.attn_cell_dims]
else:
self.attn_cell_dims = [
int(s) for s in opt.attn_cell_dims.split(',')
]
if type(opt.gamma_cell_dims) == int:
self.gamma_cell_dims = [opt.gamma_cell_dims]
else:
self.gamma_cell_dims = [
int(s) for s in opt.gamma_cell_dims.split(',')
]
self.output_cell_dim = opt.output_cell_dim
if type(opt.attn_dropout_rates) == float:
self.attn_dropout_rates = [opt.attn_dropout_rates]
else:
self.attn_dropout_rates = [
float(s) for s in opt.attn_dropout_rates.split(',')
]
if type(opt.gamma_dropout_rates) == float:
self.gamma_dropout_rates = [opt.gamma_dropout_rates]
else:
self.gamma_dropout_rates = [
float(s) for s in opt.gamma_dropout_rates.split(',')
]
self.out_dropout_rate = opt.out_dropout_rate
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table,
dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(
embedding_matrix, freeze=not opt.embedding_trainable)
self.lstms = nn.ModuleList([nn.LSTMCell(input_dim, cell_dim) \
for input_dim, cell_dim in zip(self.input_dims, self.hidden_dims)])
self.fc_attn_1 = SimpleNet(self.attn_in_dim, self.attn_cell_dims[0],
self.attn_dropout_rates[0],
self.attn_in_dim, nn.Softmax(dim=1))
self.fc_attn_2 = SimpleNet(self.attn_in_dim, self.attn_cell_dims[1],
self.attn_dropout_rates[1], self.memory_dim,
nn.Tanh())
self.fc_gamma = nn.ModuleList([
SimpleNet(self.gamma_in_dim, cell_dim, dropout, self.memory_dim,
nn.Sigmoid()) for cell_dim, dropout in zip(
self.gamma_cell_dims, self.gamma_dropout_rates)
])
self.fc_out = SimpleNet(self.final_fc_in_dim, self.output_cell_dim,
self.out_dropout_rate, self.output_dim)
def main(params, config):
data_pedestal = PedestalDataset(config)
train_loader, validation_loader = split_dataset(data_pedestal,
params['batch_size'])
if config['experiment']['load_model'] != None:
PATH = config['experiment']['load_model']
checkpoint = torch.load(PATH)
# Load Model
net = SimpleNet(params, config)
net.load_state_dict(checkpoint['model_state_dict'])
# Load Optimizer
optimizer = model_utils.map_optimizer(params['optimizer'],
net.parameters(), 0.0)
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# Assign Loss Function
loss_func = model_utils.map_loss_func(params['loss'])
# Set EPOCH and LOSS for retraining
epoch = checkpoint['epoch']
loss = checkpoint['loss']
else:
net = SimpleNet(params, config)
optimizer = model_utils.map_optimizer(params['optimizer'],
net.parameters(),
params['learning_rate'])
loss_func = model_utils.map_loss_func(params['loss'])
epochs = config['epochs']
last_results = []
metrics = {}
for epoch in range(epochs):
# TRAINING
net.train()
for i, batch in enumerate(train_loader):
optimizer.zero_grad()
inputs, targets = batch['input'], batch['target']
output = net(inputs)
loss = loss_func(output, targets)
loss.backward()
optimizer.step()
# Validation
if epoch % 5 == 4:
net.eval()
max_error = 0.0
for i, batch in enumerate(validation_loader):
inputs, targets = batch['input'], batch['target']
output = net(inputs)
MSE = torch.sum((output - targets)**
2) / (len(output) * params['batch_size'])
max_error = max(MSE, max_error)
score = -math.log10(max_error)
# print(epoch, score)
if epoch > epochs - 5:
last_results.append(score)
final_score = min(last_results)
metrics['default'] = final_score
if config['experiment']['save_model'] is not None:
PATH = config['experiment']['save_model']
# save mode
torch.save(
{
'epoch': epoch,
'model_state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, PATH)
def __init__(self, opt):
super(QMNAblation, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
# MELD data
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes = opt.output_dim
self.input_concat = opt.input_concat
self.zero_phase = opt.zero_phase
self.measurement_type = opt.measurement_type
self.classical_recurrent = opt.classical_recurrent
self.quantum_recurrent = opt.quantum_recurrent
if self.input_concat:
self.projections = nn.ModuleList(
[nn.Linear(self.total_input_dim, self.embed_dim)])
elif self.classical_recurrent:
self.projections = nn.ModuleList(
[nn.GRU(dim, self.embed_dim, 1) for dim in self.input_dims])
else:
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([PositionEmbedding(self.embed_dim, input_dim = self.speaker_num,\
zero_phase = self.zero_phase, device = self.device)]*len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.num_layers = opt.num_layers
self.recurrent_cells = nn.ModuleList(
[QRNNCell(self.embed_dim, device=self.device)] * self.num_layers)
self.out_dropout = QDropout(p=self.out_dropout_rate)
self.activation = QActivation(scale_factor=1, beta=0.8)
self.measurement_type = opt.measurement_type
if self.measurement_type == 'quantum':
self.measurement = QMeasurement(self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
elif self.measurement_type == 'flatten':
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
else:
self.measurement = ComplexMeasurement(self.embed_dim,
units=self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
class QMNAblation(nn.Module):
def __init__(self, opt):
super(QMNAblation, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
# MELD data
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes = opt.output_dim
self.input_concat = opt.input_concat
self.zero_phase = opt.zero_phase
self.measurement_type = opt.measurement_type
self.classical_recurrent = opt.classical_recurrent
self.quantum_recurrent = opt.quantum_recurrent
if self.input_concat:
self.projections = nn.ModuleList(
[nn.Linear(self.total_input_dim, self.embed_dim)])
elif self.classical_recurrent:
self.projections = nn.ModuleList(
[nn.GRU(dim, self.embed_dim, 1) for dim in self.input_dims])
else:
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([PositionEmbedding(self.embed_dim, input_dim = self.speaker_num,\
zero_phase = self.zero_phase, device = self.device)]*len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.num_layers = opt.num_layers
self.recurrent_cells = nn.ModuleList(
[QRNNCell(self.embed_dim, device=self.device)] * self.num_layers)
self.out_dropout = QDropout(p=self.out_dropout_rate)
self.activation = QActivation(scale_factor=1, beta=0.8)
self.measurement_type = opt.measurement_type
if self.measurement_type == 'quantum':
self.measurement = QMeasurement(self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
elif self.measurement_type == 'flatten':
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
else:
self.measurement = ComplexMeasurement(self.embed_dim,
units=self.embed_dim)
self.fc_out = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes,
output_activation=nn.Tanh())
def get_params(self):
unitary_params = []
remaining_params = []
for i in range(self.num_layers):
unitary_params.append(self.recurrent_cells[i].unitary_x)
unitary_params.append(self.recurrent_cells[i].unitary_h)
remaining_params.append(self.recurrent_cells[i].Lambda)
remaining_params.extend(list(self.projections.parameters()))
remaining_params.extend(list(self.phase_embeddings.parameters()))
for i in range(self.num_layers):
remaining_params.append(self.recurrent_cells[i].Lambda)
if self.measurement_type == 'quantum':
unitary_params.extend(list(self.measurement.parameters()))
else:
remaining_params.extend(list(self.measurement.parameters()))
remaining_params.extend(list(self.fc_out.parameters()))
return unitary_params, remaining_params
def forward(self, in_modalities):
smask = in_modalities[-2] # Speaker ids
in_modalities = in_modalities[:-2]
batch_size = in_modalities[0].shape[0]
time_stamps = in_modalities[0].shape[1]
# Project All modalities of each utterance to the same space
#utterance_reps = [nn.Tanh()(projection(x)) for x, projection in zip(in_modalities,self.projections)]
if self.input_concat:
in_modalities = torch.cat(in_modalities, dim=-1)
if self.classical_recurrent:
utterance_reps = [
nn.ReLU()(projection(x)[0])
for x, projection in zip(in_modalities, self.projections)
]
else:
utterance_reps = [
nn.ReLU()(projection(x))
for x, projection in zip(in_modalities, self.projections)
]
# Take the amplitudes
# multiply with modality specific vectors to construct weights
weights = [self.norm(rep) for rep in utterance_reps]
weights = F.softmax(torch.cat(weights, dim=-1), dim=-1)
amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps]
phases = [
phase_embed(smask.argmax(dim=-1))
for phase_embed in self.phase_embeddings
]
unimodal_pure = [
self.multiply([phase, amplitude])
for phase, amplitude in zip(phases, amplitudes)
]
unimodal_matrices = [self.outer(s) for s in unimodal_pure]
in_states = self.mixture([unimodal_matrices, weights])
if self.quantum_recurrent:
for l in range(self.num_layers):
# Initialize the cell h
h_r = torch.stack(batch_size *
[torch.eye(self.embed_dim) / self.embed_dim],
dim=0)
h_i = torch.zeros_like(h_r)
h = [h_r.to(self.device), h_i.to(self.device)]
all_h = []
for t in range(time_stamps):
h = self.recurrent_cells[l](in_states[t], h)
all_h.append(self.activation(h))
in_states = all_h
output = []
for _h in in_states:
# _h = self.out_dropout(_h)
# _h = self.dense(_h)
# measurement_probs = self.measurement(_h)
if self.measurement_type == 'flatten':
_output = self.fc_out(_h[0].reshape(batch_size, -1))
else:
measurement_probs = self.measurement(_h)
_output = self.fc_out(measurement_probs)
output.append(_output)
output = torch.stack(output, dim=-2)
log_prob = F.log_softmax(output, 2) # batch, seq_len, n_classes
return log_prob
class QMultiTask(nn.Module):
def __init__(self, opt):
super(QMultiTask, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
self.embed_dim = opt.embed_dim
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
self.features = opt.features
# MELD data
# The one-hot vectors are not the global user ID
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes_emo = opt.output_dim_emo
self.n_classes_act = opt.output_dim_act
self.projections = nn.ModuleList(
[nn.Linear(dim, self.embed_dim) for dim in self.input_dims])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim=-1)
self.mixture = QMixture(device=self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([
PositionEmbedding(
self.embed_dim, input_dim=self.speaker_num, device=self.device)
] * len(self.input_dims))
self.out_dropout_rate = opt.out_dropout_rate
self.measurement_emotion = QMeasurement(self.embed_dim)
self.measurement_act = QMeasurement(self.embed_dim)
self.fc_out_emo = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes_emo,
output_activation=nn.Tanh())
self.fc_out_act = SimpleNet(self.embed_dim,
self.output_cell_dim,
self.out_dropout_rate,
self.n_classes_act,
output_activation=nn.Tanh())
self.num_layers = opt.num_layers
#self.rnn=nn.ModuleList([QRNNCell(self.embed_dim, device = self.device)]*self.num_layers)
self.RNNs = nn.ModuleList([
QRNN(self.embed_dim, self.device, self.num_layers)
for i in range(len(opt.features))
])
self.rnn_outer = QOuter()
self.action_qrnn = QRNN(self.embed_dim, self.device, self.num_layers)
def get_params(self):
unitary_params = []
remaining_params = []
for i in range(len(self.features)):
qrnn = self.RNNs[i]
for k in range(self.num_layers):
unitary_params.append(qrnn.recurrent_cells[k].unitary_x)
unitary_params.append(qrnn.recurrent_cells[k].unitary_h)
remaining_params.append(qrnn.recurrent_cells[k].Lambda)
for k in range(self.num_layers):
unitary_params.append(
self.action_qrnn.recurrent_cells[k].unitary_x)
unitary_params.append(
self.action_qrnn.recurrent_cells[k].unitary_h)
remaining_params.append(self.action_qrnn.recurrent_cells[k].Lambda)
unitary_params.extend(list(self.measurement_act.parameters()))
unitary_params.extend(list(self.measurement_emotion.parameters()))
remaining_params.extend(list(self.projections.parameters()))
remaining_params.extend(list(self.phase_embeddings.parameters()))
remaining_params.extend(list(self.fc_out_act.parameters()))
remaining_params.extend(list(self.fc_out_emo.parameters()))
return unitary_params, remaining_params
def forward(self, in_modalities):
smask = in_modalities[-2] # Speaker ids
in_modalities = in_modalities[:-2]
batch_size = in_modalities[0].shape[0]
time_stamps = in_modalities[0].shape[1]
# Project All modalities of each utterance to the same space
utterance_reps = [
nn.ReLU()(projection(x))
for x, projection in zip(in_modalities, self.projections)
] ##
# Take the amplitudes
# multiply with modality specific vectors to construct weights
amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps]
phases = [
phase_embed(smask.argmax(dim=-1))
for phase_embed in self.phase_embeddings
]
weights = [self.norm(rep) for rep in utterance_reps]
weights = F.softmax(torch.cat(weights, dim=-1), dim=-1)
unimodal_pure = [
self.multiply([phase, amplitude])
for phase, amplitude in zip(phases, amplitudes)
]
unimodal_matrices = [self.outer(s) for s in unimodal_pure]
rnn_unimodal_data = [
rnn(data) for data, rnn in zip(unimodal_matrices, self.RNNs)
] ##
# weights = [self.norm(rep) for rep in rnn_unimodal_data]
# weights = F.softmax(torch.cat(weights, dim = -1), dim = -1)
emo_states = self.mixture([rnn_unimodal_data, weights])
action_states = self.action_qrnn(emo_states)
###emotion classifier###
output_emo = []
for _h in emo_states:
measurement_probs = self.measurement_emotion(_h)
_output = self.fc_out_emo(measurement_probs)
output_emo.append(_output)
output_e = torch.stack(output_emo, dim=-2)
log_prob_e = F.log_softmax(output_e, 2) # batch, seq_len, n_classes
###action classifier###
output_act = []
for _h in emo_states:
measurement_probs = self.measurement_act(_h)
_output = self.fc_out_act(measurement_probs)
output_act.append(_output)
output_a = torch.stack(output_act, dim=-2)
log_prob_a = F.log_softmax(output_a, 2) # batch, seq_len, n_classes
return log_prob_e, log_prob_a
def __init__(self, opt):
super(MARN, self).__init__()
self.input_dims = opt.input_dims
self.output_dim = opt.output_dim
if type(opt.hidden_dims) == int:
self.hidden_dims = [opt.hidden_dims]
else:
self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')]
self.device = opt.device
self.attn_num = opt.attn_num
self.attn_cell_dim = opt.attn_cell_dim
self.attn_dropout_rate = opt.attn_dropout_rate
if opt.embedding_enabled:
embedding_matrix = torch.tensor(opt.lookup_table,
dtype=torch.float)
self.embed = nn.Embedding.from_pretrained(
embedding_matrix, freeze=not opt.embedding_trainable)
if type(opt.compression_cell_dims) == int:
self.compression_cell_dims = [opt.compression_cell_dims]
else:
self.compression_cell_dims = [
int(s) for s in opt.compression_cell_dims.split(',')
]
if type(opt.compression_dropout_rates) == float:
self.compression_dropout_rates = [opt.compression_dropout_rates]
else:
self.compression_dropout_rates = [
float(s) for s in opt.compression_dropout_rates.split(',')
]
self.total_hidden_dim = sum(self.hidden_dims)
self.output_cell_dim = opt.output_cell_dim
self.output_dropout_rate = opt.output_dropout_rate
# self.batch_size = opt.batch_size
# initialize the LSTHM for each modalitye
if type(opt.compressed_dims) == int:
self.compressed_dims = [opt.compressed_dims]
else:
self.compressed_dims = [
int(s) for s in opt.compressed_dims.split(',')
]
self.num_modalities = len(self.input_dims)
self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim,input_dim,sum(self.compressed_dims),self.device) \
for input_dim,hidden_dim in zip(self.input_dims,self.hidden_dims)])
attention = SimpleNet(self.total_hidden_dim,
self.attn_cell_dim * self.attn_num,
self.attn_dropout_rate,
self.total_hidden_dim * self.attn_num,
nn.Sigmoid())
compression_nets = nn.ModuleList([
SimpleNet(hidden_dim * self.attn_num, compression_cell_dim,
compression_dropout_rate, compressed_dim, nn.Sigmoid())
for compressed_dim, hidden_dim,
compression_cell_dim, compression_dropout_rate in zip(
self.compressed_dims, self.hidden_dims,
self.compression_cell_dims, self.compression_dropout_rates)
])
self.multi_attention = MultipleAttention(attention, compression_nets,
self.attn_num)
self.fc_output_in_dim = sum(
self.compressed_dims) + self.total_hidden_dim
self.fc_out = SimpleNet(self.fc_output_in_dim, self.output_cell_dim,
self.output_dropout_rate, self.output_dim)
def train_model(dataset,
model_output,
model_type,
training_ratio=0.7,
validation_ratio=0.1,
epoch=10,
batch_size=50,
learning_rate=0.01):
if dataset is None:
raise ValueError('No dataset specified')
hdf5_file = h5py.File(dataset, 'r')
train, validation, test, validation_data = data_split(
hdf5_file=hdf5_file,
training_ratio=training_ratio,
validation_ratio=validation_ratio)
train_start, train_end = train
validation_start, validation_end = validation
test_start, test_end = test
shape = input_shape(hdf5_file=hdf5_file)
height, width, channels = shape
if model_type in [
'AlexNet', 'alexnet', 'ALEXNET', 'Alex', 'alex', 'ALEX',
'alex_net', 'Alex_Net', 'ALEX_NET'
]:
model = AlexNet(learning_rate=learning_rate,
width=width,
height=height,
channels=channels)
elif model_type in [
'NvidiaNet', 'nvidianet', 'NVIDIANET', 'Nvidia', 'nvidia',
'NVIDIA', 'nvidia_net', 'Nvidia_Net', 'NVIDIA_NET'
]:
model = NvidiaNet(learning_rate=learning_rate,
width=width,
height=height,
channels=channels)
elif model_type in ['VGG16', 'vgg16', 'Vgg16', 'VGG', 'Vgg', 'vgg']:
model = VGG16Net(learning_rate=learning_rate,
width=width,
height=height,
channels=channels)
elif model_type in [
'SimpleNet', 'simplenet', 'SIMPLENET', 'Simplenet', 'simple',
'SIMPLE'
]:
model = SimpleNet(learning_rate=learning_rate,
width=width,
height=height,
channels=channels)
else:
raise ValueError('Not a valid model type')
sys.exit(0)
train_start_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
if model_output:
model_output_path, _ = os.path.splitext(model_output)
model_filename = model_output_path + '-' + train_start_time
else:
dataset_path, _ = os.path.splitext(dataset)
dataset_filename = dataset_path[dataset_path.find('/') + 1:]
model_filename = 'models/' + dataset_filename + '-' + train_start_time
loss_checkpoint = ModelCheckpoint(model_filename + '-val-loss-lowest.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
accuracy_checkpoint = ModelCheckpoint(model_filename +
'-val-acc-highest.h5',
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
early_stopping = EarlyStopping(monitor='val_loss',
patience=6,
verbose=1,
mode='min')
reduce_learning_rate = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=5)
callbacks = [loss_checkpoint, accuracy_checkpoint]
if validation_data:
train_gen = generate_examples(hdf5_file=hdf5_file,
batch_size=batch_size,
start=train_start,
end=train_end)
validation_gen = generate_examples(hdf5_file=hdf5_file,
batch_size=batch_size,
start=validation_start,
end=validation_end)
model.fit_generator(train_gen,
samples_per_epoch=train_end - train_start,
validation_data=validation_gen,
nb_val_samples=validation_end - validation_start,
nb_epoch=epoch,
verbose=1,
callbacks=callbacks)
else:
train_gen = generate_examples(hdf5_file=hdf5_file,
batch_size=batch_size,
start=train_start,
end=train_end)
model.fit_generator(train_gen,
samples_per_epoch=train_end - train_start,
nb_epoch=epoch,
verbose=1,
callbacks=callbacks)
model.save(model_filename + '.h5')
# Just to be safe
model.save('tmp.h5')
return model
class QMPN(nn.Module):
def __init__(self, opt):
super(QMPN, self).__init__()
self.device = opt.device
self.input_dims = opt.input_dims
self.total_input_dim = sum(self.input_dims)
if type(opt.embed_dims) == int:
self.embed_dims = [opt.embed_dims]
self.embed_dims = [int(s) for s in opt.embed_dims.split(',')]
self.speaker_num = opt.speaker_num
self.dataset_name = opt.dataset_name
# MELD data
# The one-hot vectors are not the global user ID
if self.dataset_name.lower() == 'meld':
self.speaker_num = 1
self.n_classes = opt.output_dim
self.projections = nn.ModuleList([nn.Linear(dim, embed_dim) for dim, embed_dim in zip(self.input_dims,self.embed_dims)])
self.multiply = ComplexMultiply()
self.outer = QOuter()
self.norm = L2Norm(dim = -1)
self.mixture = QMixture(device = self.device)
self.output_cell_dim = opt.output_cell_dim
self.phase_embeddings = nn.ModuleList([PositionEmbedding(embed_dim, input_dim = self.speaker_num, device = self.device) for embed_dim in self.embed_dims])
self.out_dropout_rate = opt.out_dropout_rate
self.num_layers = opt.num_layers
self.state_product = QProduct()
self.recurrent_dim = 1
for embed_dim in self.embed_dims:
self.recurrent_dim = self.recurrent_dim*embed_dim
self.recurrent_cells = nn.ModuleList([QRNNCell(self.recurrent_dim, device = self.device)]*self.num_layers)
self.measurement = QMeasurement(self.recurrent_dim)
self.fc_out = SimpleNet(self.recurrent_dim, self.output_cell_dim,
self.out_dropout_rate,self.n_classes,
output_activation = nn.Tanh())
def get_params(self):
unitary_params = []
remaining_params = []
for i in range(self.num_layers):
unitary_params.append(self.recurrent_cells[i].unitary_x)
unitary_params.append(self.recurrent_cells[i].unitary_h)
remaining_params.append(self.recurrent_cells[i].Lambda)
remaining_params.extend(list(self.projections.parameters()))
remaining_params.extend(list(self.phase_embeddings.parameters()))
for i in range(self.num_layers):
remaining_params.append(self.recurrent_cells[i].Lambda)
unitary_params.extend(list(self.measurement.parameters()))
remaining_params.extend(list(self.fc_out.parameters()))
return unitary_params, remaining_params
def forward(self, in_modalities):
smask = in_modalities[-2] # Speaker ids
in_modalities = in_modalities[:-2]
batch_size = in_modalities[0].shape[0]
time_stamps = in_modalities[0].shape[1]
# Project All modalities of each utterance to the same space
utterance_reps = [nn.ReLU()(projection(x)) for x, projection in zip(in_modalities,self.projections)]
amplitudes = [F.normalize(rep, dim = -1) for rep in utterance_reps]
phases = [phase_embed(smask.argmax(dim = -1)) for phase_embed in self.phase_embeddings]
unimodal_pure = [self.multiply([phase, amplitude]) for phase, amplitude in zip(phases,amplitudes)]
multimodal_pure = self.state_product(unimodal_pure)
in_states = self.outer(multimodal_pure)
# Take the amplitudes
# multiply with modality specific vectors to construct weights
#weights = [self.norm(rep) for rep in utterance_reps]
#weights = F.softmax(torch.cat(weights, dim = -1), dim = -1)
#unimodal_matrices = [self.outer(s) for s in unimodal_pure]
#in_states = self.mixture([unimodal_matrices, weights])
for l in range(self.num_layers):
# Initialize the cell h
h_r = torch.stack(batch_size*[torch.eye(self.recurrent_dim)/self.recurrent_dim],dim =0)
h_i = torch.zeros_like(h_r)
h = [h_r.to(self.device),h_i.to(self.device)]
all_h = []
for t in range(time_stamps):
h = self.recurrent_cells[l](in_states[t],h)
all_h.append(self.activation(h))
in_states = all_h
output = []
for _h in in_states:
measurement_probs = self.measurement(_h)
_output = self.fc_out(measurement_probs)
output.append(_output)
output = torch.stack(output, dim=-2)
log_prob = F.log_softmax(output, 2) # batch, seq_len, n_classes
return log_prob
def main(params, config):
dataset = PedestalDataset(config)
train_loader, validation_loader = split_dataset(dataset,
params['batch_size'])
if config['experiment']['load_model'] != None:
PATH = config['experiment']['load_model']
checkpoint = torch.load(PATH)
# Load Model
net = SimpleNet(params, config)
net.load_state_dict(checkpoint['model_state_dict'])
# Load Optimizer
optimizer = model_utils.map_optimizer(params['optimizer'],
net.parameters(), 0.0)
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# Assign Loss Function
loss_func = model_utils.map_loss_func(params['loss'])
# Set EPOCH and LOSS for retraining
epoch = checkpoint['epoch']
loss = checkpoint['loss']
else:
net = SimpleNet(params, config)
optimizer = model_utils.map_optimizer(params['optimizer'],
net.parameters(),
params['learning_rate'])
loss_func = model_utils.map_loss_func(params['loss'])
metrics = {}
target_norms = dataset.targets_norms
input_norms = dataset.inputs_norms
target_list = dataset.target_params
input_list = dataset.input_params
net.eval()
outputs = []
actual_array = []
scaled_list = []
save_path = config['experiment']['name']
for i, batch in enumerate(validation_loader):
inputs, targets = batch['input'], batch['target']
for val in inputs:
output = net(val).detach().numpy()
output = denormalize(output, target_list, target_norms)
outputs.append(output[0])
normed_vals = denormalize(val.numpy(), input_list, input_norms)
for tar in targets:
denorm_targ = denormalize(tar.numpy(), target_list, target_norms)
actual_array.append(denorm_targ[0])
if config['experiment']['target'] == 'density':
for i, batch in enumerate(validation_loader):
inputs, targets = batch['input'], batch['target']
for val in inputs:
normed_vals = denormalize(val.numpy(), input_list, input_norms)
scaled_vals = fitted_scale(normed_vals)
scaled_list.append(scaled_vals)
plt.scatter(actual_array, scaled_list, label='Scale Law')
plt.scatter(actual_array, actual_array, label='Actual')
plt.scatter(actual_array, outputs, label='NN')
plt.legend()
plt.ylabel('Predicted')
plt.xlabel('Actual Density Height')
plt.ylim(0, 12)
plt.title('Neural Network vs Scaling Law')
plt.savefig('./results/' + save_path)
plt.show()