def __init__(self, opt): super(LSTHM, self).__init__() self.input_dims = opt.input_dims self.output_dim = opt.output_dim if type(opt.hidden_dims) == int: self.hidden_dims = [opt.hidden_dims] else: self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')] if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained(embedding_matrix, freeze=False) self.num_modalities = len(self.input_dims) self.device = opt.device self.hybrid_in_size = opt.hybrid_in_size self.hybrid_cell_size = opt.hybrid_cell_size self.hybrid_dropout_rate = opt.hybrid_dropout_rate self.output_cell_dim = opt.output_cell_dim self.output_dropout_rate = opt.output_dropout_rate self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim, input_dim,self.hybrid_in_size, \ device = self.device) for hidden_dim, input_dim \ in zip(self.hidden_dims, self.input_dims)]) self.fc_hybrid = SimpleNet(sum(self.hidden_dims), self.hybrid_cell_size, self.hybrid_dropout_rate, self.hybrid_in_size, nn.Softmax(dim=1)) self.fc_out = SimpleNet( sum(self.hidden_dims) + self.hybrid_in_size, self.output_cell_dim, self.output_dropout_rate, self.output_dim)
def __init__(self, opt): super(QMultiTask, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name self.features = opt.features # MELD data # The one-hot vectors are not the global user ID if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes_emo = opt.output_dim_emo self.n_classes_act = opt.output_dim_act self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([ PositionEmbedding( self.embed_dim, input_dim=self.speaker_num, device=self.device) ] * len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.measurement_emotion = QMeasurement(self.embed_dim) self.measurement_act = QMeasurement(self.embed_dim) self.fc_out_emo = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes_emo, output_activation=nn.Tanh()) self.fc_out_act = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes_act, output_activation=nn.Tanh()) self.num_layers = opt.num_layers #self.rnn=nn.ModuleList([QRNNCell(self.embed_dim, device = self.device)]*self.num_layers) self.RNNs = nn.ModuleList([ QRNN(self.embed_dim, self.device, self.num_layers) for i in range(len(opt.features)) ]) self.rnn_outer = QOuter() self.action_qrnn = QRNN(self.embed_dim, self.device, self.num_layers)
def __init__(self, opt): super(QMPN, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) if type(opt.embed_dims) == int: self.embed_dims = [opt.embed_dims] self.embed_dims = [int(s) for s in opt.embed_dims.split(',')] self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name # MELD data # The one-hot vectors are not the global user ID if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes = opt.output_dim self.projections = nn.ModuleList([ nn.Linear(dim, embed_dim) for dim, embed_dim in zip(self.input_dims, self.embed_dims) ]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([ PositionEmbedding(embed_dim, input_dim=self.speaker_num, device=self.device) for embed_dim in self.embed_dims ]) self.out_dropout_rate = opt.out_dropout_rate self.num_layers = opt.num_layers self.state_product = QProduct() self.recurrent_dim = 1 for embed_dim in self.embed_dims: self.recurrent_dim = self.recurrent_dim * embed_dim self.recurrent_cells = nn.ModuleList( [QRNNCell(self.recurrent_dim, device=self.device)] * self.num_layers) self.out_dropout = QDropout(p=self.out_dropout_rate) self.activation = QActivation(scale_factor=1, beta=0.8) self.measurement = QMeasurement(self.recurrent_dim) self.fc_out = SimpleNet(self.recurrent_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh())
def __init__(self, opt): super(RMFN, self).__init__() self.input_dims = opt.input_dims self.output_dim = opt.output_dim if type(opt.hidden_dims) == int: self.hidden_dims = [opt.hidden_dims] else: self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')] self.num_modalities = len(self.input_dims) self.steps = opt.steps self.device = opt.device self.compression_cell_dim = opt.compression_cell_dim self.compression_dropout_rate = opt.compression_dropout_rate self.hlt_memory_init_cell_dim = opt.hlt_memory_init_cell_dim self.hlt_memory_init_dropout_rate = opt.hlt_memory_init_dropout_rate self.output_cell_dim = opt.output_cell_dim self.output_dropout_rate = opt.output_dropout_rate # initialize the LSTHM for each modality self.compressed_dim = opt.compressed_dim if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained( embedding_matrix, freeze=not opt.embedding_trainable) self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim,input_dim,self.compressed_dim,self.device) \ for input_dim,hidden_dim in zip(self.input_dims,self.hidden_dims)]) self.hlt_memory_init_net = SimpleNet(sum(self.hidden_dims), self.hlt_memory_init_cell_dim, self.hlt_memory_init_dropout_rate, sum(self.hidden_dims), nn.Sigmoid()) self.compression_net = SimpleNet(self.steps * sum(self.hidden_dims), self.compression_cell_dim, self.compression_dropout_rate, self.compressed_dim, nn.Sigmoid()) self.recurrent_fusion = RecurrentFusion(self.hidden_dims, self.hlt_memory_init_net, self.compression_net, self.steps, self.device) self.fc_out = SimpleNet(self.compressed_dim + sum(self.hidden_dims), self.output_cell_dim, self.output_dropout_rate, self.output_dim)
def __init__(self, opt): super(QAttN, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.n_classes = opt.output_dim self.output_cell_dim = opt.output_cell_dim self.dataset_name = opt.dataset_name self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.phase_embeddings = nn.ModuleList([ PositionEmbedding(self.embed_dim, input_dim=1, device=self.device) ] * len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.attention = QAttention() self.num_modalities = len(self.input_dims) #self.out_dropout = QDropout(p=self.out_dropout_rate) #self.dense = QDense(self.embed_dim, self.n_classes) self.measurement_type = opt.measurement_type if opt.measurement_type == 'quantum': self.measurement = QMeasurement(self.embed_dim) self.fc_out = SimpleNet(self.embed_dim * self.num_modalities, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) else: self.measurement = ComplexMeasurement(self.embed_dim * self.num_modalities, units=20) self.fc_out = SimpleNet(20, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh())
def train_models(trainX, testX, trainY, testY): print("[INFO] initialize models") models = [] models.append(SimpleNet(width, height, depth, classes)) models.append(LeNet5(width, height, depth, classes)) print("[INFO] the number of models :", len(models)) Hs = [] # 遍历模型训练个数 for i in np.arange(0, len(models)): # 初始化优化器和模型 print("[INFO] training model {}/{}".format(i + 1, len(models))) # opt = Adam(lr=lr, decay=1e-4 / epochs) # opt = SGD(lr=0.001,decay=0.01/ 40,momentum=0.9, # nesterov=True) # 训练网络 H = models[i].fit_generator(aug.flow(trainX, trainY, batch_size=bs), validation_data=(testX, testY), epochs=epochs, steps_per_epoch=len(trainX) // bs, verbose=1) # 将模型保存到磁盘中 p = ['save_models', "model_{}.model".format(i)] models[i].save(os.path.sep.join(p)) Hs.append(models[i]) # plot the training loss and accuracy N = epochs p = ['model_{}.png'.format(i)] plt.style.use('ggplot') plt.figure() plt.plot(np.arange(0, N), H.history['loss'], label='train_loss') plt.plot(np.arange(0, N), H.history['val_loss'], label='val_loss') plt.plot(np.arange(0, N), H.history['acc'], label='train-acc') plt.plot(np.arange(0, N), H.history['val_acc'], label='val-acc') plt.title("Training Loss and Accuracy for model {}".format(i)) plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend() plt.savefig(os.path.sep.join(p)) plt.close()
def __init__(self, opt): super(EFLSTM, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.hidden_dim = opt.hidden_dim if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained( embedding_matrix, freeze=not opt.embedding_trainable) if self.hidden_dim == -1: self.hidden_dim = self.total_input_dim self.output_dim = opt.output_dim self.output_cell_dim = opt.output_cell_dim self.output_dropout_rate = opt.output_dropout_rate self.lstm = nn.LSTMCell(self.total_input_dim, self.hidden_dim) self.fc_out = SimpleNet(self.hidden_dim, self.output_cell_dim, self.output_dropout_rate, self.output_dim)
def __init__(self, opt): #, n_layers, bidirectional, dropout): super(LFLSTM, self).__init__() self.device = opt.device self.input_dims = opt.input_dims if type(opt.hidden_dims) == int: self.hidden_dims = [opt.hidden_dims] else: self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')] if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained(embedding_matrix, freeze=False) self.num_modalities = len(self.input_dims) self.output_dim = opt.output_dim self.output_cell_dim = opt.output_cell_dim self.output_dropout_rate = opt.output_dropout_rate self.lstms = nn.ModuleList([nn.LSTMCell(input_dim, hidden_dim) \ for input_dim, hidden_dim in \ zip(self.input_dims, self.hidden_dims)]) self.fc_out = SimpleNet(sum(self.hidden_dims),self.output_cell_dim, self.output_dropout_rate,self.output_dim)
class QAttN(nn.Module): def __init__(self, opt): super(QAttN, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.n_classes = opt.output_dim self.output_cell_dim = opt.output_cell_dim self.dataset_name = opt.dataset_name self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.phase_embeddings = nn.ModuleList([ PositionEmbedding(self.embed_dim, input_dim=1, device=self.device) ] * len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.attention = QAttention() self.num_modalities = len(self.input_dims) #self.out_dropout = QDropout(p=self.out_dropout_rate) #self.dense = QDense(self.embed_dim, self.n_classes) self.measurement_type = opt.measurement_type if opt.measurement_type == 'quantum': self.measurement = QMeasurement(self.embed_dim) self.fc_out = SimpleNet(self.embed_dim * self.num_modalities, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) else: self.measurement = ComplexMeasurement(self.embed_dim * self.num_modalities, units=20) self.fc_out = SimpleNet(20, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) def get_params(self): unitary_params = [] remaining_params = [] remaining_params.extend(list(self.projections.parameters())) remaining_params.extend(list(self.phase_embeddings.parameters())) if self.measurement_type == 'quantum': unitary_params.extend(list(self.measurement.parameters())) else: remaining_params.extend(list(self.measurement.parameters())) remaining_params.extend(list(self.fc_out.parameters())) return unitary_params, remaining_params def forward(self, in_modalities): smask = in_modalities[-2] # Speaker ids in_modalities = in_modalities[:-2] batch_size = in_modalities[0].shape[0] time_stamps = in_modalities[0].shape[1] # Project All modalities of each utterance to the same space #utterance_reps = [nn.Tanh()(projection(x)) for x, projection in zip(in_modalities,self.projections)] utterance_reps = [ nn.ReLU()(projection(x)) for x, projection in zip(in_modalities, self.projections) ] # Take the amplitudes # multiply with modality specific vectors to construct weights weights = [self.norm(rep) for rep in utterance_reps] amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps] phases = [ phase_embed(smask.argmax(dim=-1)) for phase_embed in self.phase_embeddings ] unimodal_pure = [ self.multiply([phase, amplitude]) for phase, amplitude in zip(phases, amplitudes) ] unimodal_matrices = [self.outer(s) for s in unimodal_pure] probs = [] # For each modality # we mix the remaining modalities as queries (to_be_measured systems) # and treat the modality features as keys (measurement operators) for ind in range(self.num_modalities): # Obtain mixed states for the rest modalities other_weights = [ weights[i] for i in range(self.num_modalities) if not i == ind ] mixture_weights = F.softmax(torch.cat(other_weights, dim=-1), dim=-1) other_states = [ unimodal_matrices[i] for i in range(self.num_modalities) if not i == ind ] q_states = self.mixture([other_states, mixture_weights]) # Obtain pure states and weights for the modality of interest k_weights = weights[ind] k_states = unimodal_pure[ind] # Compute cross-modal interactions, output being a list of post-measurement states in_states = self.attention(q_states, k_states, k_weights) # Apply measurement to each output state output = [] for _h in in_states: measurement_probs = self.measurement(_h) output.append(measurement_probs) probs.append(output) # Concatenate the measurement probabilities per-time-stamp concat_probs = [ self.fc_out(torch.cat(output_t, dim=-1)) for output_t in zip(*probs) ] concat_probs = torch.stack(concat_probs, dim=-2) log_prob = F.log_softmax(concat_probs, 2) # batch, seq_len, n_classes return log_prob
def __init__(self, opt): super(MFN, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.num_modalities = len(self.input_dims) if type(opt.hidden_dims) == int: self.hidden_dims = [opt.hidden_dims] else: self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')] self.total_cell_dim = sum(self.hidden_dims) self.memory_dim = opt.memory_dim self.window_dim = opt.window_dim self.output_dim = opt.output_dim self.attn_in_dim = self.total_cell_dim * self.window_dim self.gamma_in_dim = self.attn_in_dim + self.memory_dim self.final_fc_in_dim = self.total_cell_dim + self.memory_dim if type(opt.attn_cell_dims) == int: self.attn_cell_dims = [opt.attn_cell_dims] else: self.attn_cell_dims = [ int(s) for s in opt.attn_cell_dims.split(',') ] if type(opt.gamma_cell_dims) == int: self.gamma_cell_dims = [opt.gamma_cell_dims] else: self.gamma_cell_dims = [ int(s) for s in opt.gamma_cell_dims.split(',') ] self.output_cell_dim = opt.output_cell_dim if type(opt.attn_dropout_rates) == float: self.attn_dropout_rates = [opt.attn_dropout_rates] else: self.attn_dropout_rates = [ float(s) for s in opt.attn_dropout_rates.split(',') ] if type(opt.gamma_dropout_rates) == float: self.gamma_dropout_rates = [opt.gamma_dropout_rates] else: self.gamma_dropout_rates = [ float(s) for s in opt.gamma_dropout_rates.split(',') ] self.out_dropout_rate = opt.out_dropout_rate if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained( embedding_matrix, freeze=not opt.embedding_trainable) self.lstms = nn.ModuleList([nn.LSTMCell(input_dim, cell_dim) \ for input_dim, cell_dim in zip(self.input_dims, self.hidden_dims)]) self.fc_attn_1 = SimpleNet(self.attn_in_dim, self.attn_cell_dims[0], self.attn_dropout_rates[0], self.attn_in_dim, nn.Softmax(dim=1)) self.fc_attn_2 = SimpleNet(self.attn_in_dim, self.attn_cell_dims[1], self.attn_dropout_rates[1], self.memory_dim, nn.Tanh()) self.fc_gamma = nn.ModuleList([ SimpleNet(self.gamma_in_dim, cell_dim, dropout, self.memory_dim, nn.Sigmoid()) for cell_dim, dropout in zip( self.gamma_cell_dims, self.gamma_dropout_rates) ]) self.fc_out = SimpleNet(self.final_fc_in_dim, self.output_cell_dim, self.out_dropout_rate, self.output_dim)
def main(params, config): data_pedestal = PedestalDataset(config) train_loader, validation_loader = split_dataset(data_pedestal, params['batch_size']) if config['experiment']['load_model'] != None: PATH = config['experiment']['load_model'] checkpoint = torch.load(PATH) # Load Model net = SimpleNet(params, config) net.load_state_dict(checkpoint['model_state_dict']) # Load Optimizer optimizer = model_utils.map_optimizer(params['optimizer'], net.parameters(), 0.0) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # Assign Loss Function loss_func = model_utils.map_loss_func(params['loss']) # Set EPOCH and LOSS for retraining epoch = checkpoint['epoch'] loss = checkpoint['loss'] else: net = SimpleNet(params, config) optimizer = model_utils.map_optimizer(params['optimizer'], net.parameters(), params['learning_rate']) loss_func = model_utils.map_loss_func(params['loss']) epochs = config['epochs'] last_results = [] metrics = {} for epoch in range(epochs): # TRAINING net.train() for i, batch in enumerate(train_loader): optimizer.zero_grad() inputs, targets = batch['input'], batch['target'] output = net(inputs) loss = loss_func(output, targets) loss.backward() optimizer.step() # Validation if epoch % 5 == 4: net.eval() max_error = 0.0 for i, batch in enumerate(validation_loader): inputs, targets = batch['input'], batch['target'] output = net(inputs) MSE = torch.sum((output - targets)** 2) / (len(output) * params['batch_size']) max_error = max(MSE, max_error) score = -math.log10(max_error) # print(epoch, score) if epoch > epochs - 5: last_results.append(score) final_score = min(last_results) metrics['default'] = final_score if config['experiment']['save_model'] is not None: PATH = config['experiment']['save_model'] # save mode torch.save( { 'epoch': epoch, 'model_state_dict': net.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss }, PATH)
def __init__(self, opt): super(QMNAblation, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name # MELD data if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes = opt.output_dim self.input_concat = opt.input_concat self.zero_phase = opt.zero_phase self.measurement_type = opt.measurement_type self.classical_recurrent = opt.classical_recurrent self.quantum_recurrent = opt.quantum_recurrent if self.input_concat: self.projections = nn.ModuleList( [nn.Linear(self.total_input_dim, self.embed_dim)]) elif self.classical_recurrent: self.projections = nn.ModuleList( [nn.GRU(dim, self.embed_dim, 1) for dim in self.input_dims]) else: self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([PositionEmbedding(self.embed_dim, input_dim = self.speaker_num,\ zero_phase = self.zero_phase, device = self.device)]*len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.num_layers = opt.num_layers self.recurrent_cells = nn.ModuleList( [QRNNCell(self.embed_dim, device=self.device)] * self.num_layers) self.out_dropout = QDropout(p=self.out_dropout_rate) self.activation = QActivation(scale_factor=1, beta=0.8) self.measurement_type = opt.measurement_type if self.measurement_type == 'quantum': self.measurement = QMeasurement(self.embed_dim) self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) elif self.measurement_type == 'flatten': self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) else: self.measurement = ComplexMeasurement(self.embed_dim, units=self.embed_dim) self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh())
class QMNAblation(nn.Module): def __init__(self, opt): super(QMNAblation, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name # MELD data if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes = opt.output_dim self.input_concat = opt.input_concat self.zero_phase = opt.zero_phase self.measurement_type = opt.measurement_type self.classical_recurrent = opt.classical_recurrent self.quantum_recurrent = opt.quantum_recurrent if self.input_concat: self.projections = nn.ModuleList( [nn.Linear(self.total_input_dim, self.embed_dim)]) elif self.classical_recurrent: self.projections = nn.ModuleList( [nn.GRU(dim, self.embed_dim, 1) for dim in self.input_dims]) else: self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([PositionEmbedding(self.embed_dim, input_dim = self.speaker_num,\ zero_phase = self.zero_phase, device = self.device)]*len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.num_layers = opt.num_layers self.recurrent_cells = nn.ModuleList( [QRNNCell(self.embed_dim, device=self.device)] * self.num_layers) self.out_dropout = QDropout(p=self.out_dropout_rate) self.activation = QActivation(scale_factor=1, beta=0.8) self.measurement_type = opt.measurement_type if self.measurement_type == 'quantum': self.measurement = QMeasurement(self.embed_dim) self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) elif self.measurement_type == 'flatten': self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) else: self.measurement = ComplexMeasurement(self.embed_dim, units=self.embed_dim) self.fc_out = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes, output_activation=nn.Tanh()) def get_params(self): unitary_params = [] remaining_params = [] for i in range(self.num_layers): unitary_params.append(self.recurrent_cells[i].unitary_x) unitary_params.append(self.recurrent_cells[i].unitary_h) remaining_params.append(self.recurrent_cells[i].Lambda) remaining_params.extend(list(self.projections.parameters())) remaining_params.extend(list(self.phase_embeddings.parameters())) for i in range(self.num_layers): remaining_params.append(self.recurrent_cells[i].Lambda) if self.measurement_type == 'quantum': unitary_params.extend(list(self.measurement.parameters())) else: remaining_params.extend(list(self.measurement.parameters())) remaining_params.extend(list(self.fc_out.parameters())) return unitary_params, remaining_params def forward(self, in_modalities): smask = in_modalities[-2] # Speaker ids in_modalities = in_modalities[:-2] batch_size = in_modalities[0].shape[0] time_stamps = in_modalities[0].shape[1] # Project All modalities of each utterance to the same space #utterance_reps = [nn.Tanh()(projection(x)) for x, projection in zip(in_modalities,self.projections)] if self.input_concat: in_modalities = torch.cat(in_modalities, dim=-1) if self.classical_recurrent: utterance_reps = [ nn.ReLU()(projection(x)[0]) for x, projection in zip(in_modalities, self.projections) ] else: utterance_reps = [ nn.ReLU()(projection(x)) for x, projection in zip(in_modalities, self.projections) ] # Take the amplitudes # multiply with modality specific vectors to construct weights weights = [self.norm(rep) for rep in utterance_reps] weights = F.softmax(torch.cat(weights, dim=-1), dim=-1) amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps] phases = [ phase_embed(smask.argmax(dim=-1)) for phase_embed in self.phase_embeddings ] unimodal_pure = [ self.multiply([phase, amplitude]) for phase, amplitude in zip(phases, amplitudes) ] unimodal_matrices = [self.outer(s) for s in unimodal_pure] in_states = self.mixture([unimodal_matrices, weights]) if self.quantum_recurrent: for l in range(self.num_layers): # Initialize the cell h h_r = torch.stack(batch_size * [torch.eye(self.embed_dim) / self.embed_dim], dim=0) h_i = torch.zeros_like(h_r) h = [h_r.to(self.device), h_i.to(self.device)] all_h = [] for t in range(time_stamps): h = self.recurrent_cells[l](in_states[t], h) all_h.append(self.activation(h)) in_states = all_h output = [] for _h in in_states: # _h = self.out_dropout(_h) # _h = self.dense(_h) # measurement_probs = self.measurement(_h) if self.measurement_type == 'flatten': _output = self.fc_out(_h[0].reshape(batch_size, -1)) else: measurement_probs = self.measurement(_h) _output = self.fc_out(measurement_probs) output.append(_output) output = torch.stack(output, dim=-2) log_prob = F.log_softmax(output, 2) # batch, seq_len, n_classes return log_prob
class QMultiTask(nn.Module): def __init__(self, opt): super(QMultiTask, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) self.embed_dim = opt.embed_dim self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name self.features = opt.features # MELD data # The one-hot vectors are not the global user ID if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes_emo = opt.output_dim_emo self.n_classes_act = opt.output_dim_act self.projections = nn.ModuleList( [nn.Linear(dim, self.embed_dim) for dim in self.input_dims]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim=-1) self.mixture = QMixture(device=self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([ PositionEmbedding( self.embed_dim, input_dim=self.speaker_num, device=self.device) ] * len(self.input_dims)) self.out_dropout_rate = opt.out_dropout_rate self.measurement_emotion = QMeasurement(self.embed_dim) self.measurement_act = QMeasurement(self.embed_dim) self.fc_out_emo = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes_emo, output_activation=nn.Tanh()) self.fc_out_act = SimpleNet(self.embed_dim, self.output_cell_dim, self.out_dropout_rate, self.n_classes_act, output_activation=nn.Tanh()) self.num_layers = opt.num_layers #self.rnn=nn.ModuleList([QRNNCell(self.embed_dim, device = self.device)]*self.num_layers) self.RNNs = nn.ModuleList([ QRNN(self.embed_dim, self.device, self.num_layers) for i in range(len(opt.features)) ]) self.rnn_outer = QOuter() self.action_qrnn = QRNN(self.embed_dim, self.device, self.num_layers) def get_params(self): unitary_params = [] remaining_params = [] for i in range(len(self.features)): qrnn = self.RNNs[i] for k in range(self.num_layers): unitary_params.append(qrnn.recurrent_cells[k].unitary_x) unitary_params.append(qrnn.recurrent_cells[k].unitary_h) remaining_params.append(qrnn.recurrent_cells[k].Lambda) for k in range(self.num_layers): unitary_params.append( self.action_qrnn.recurrent_cells[k].unitary_x) unitary_params.append( self.action_qrnn.recurrent_cells[k].unitary_h) remaining_params.append(self.action_qrnn.recurrent_cells[k].Lambda) unitary_params.extend(list(self.measurement_act.parameters())) unitary_params.extend(list(self.measurement_emotion.parameters())) remaining_params.extend(list(self.projections.parameters())) remaining_params.extend(list(self.phase_embeddings.parameters())) remaining_params.extend(list(self.fc_out_act.parameters())) remaining_params.extend(list(self.fc_out_emo.parameters())) return unitary_params, remaining_params def forward(self, in_modalities): smask = in_modalities[-2] # Speaker ids in_modalities = in_modalities[:-2] batch_size = in_modalities[0].shape[0] time_stamps = in_modalities[0].shape[1] # Project All modalities of each utterance to the same space utterance_reps = [ nn.ReLU()(projection(x)) for x, projection in zip(in_modalities, self.projections) ] ## # Take the amplitudes # multiply with modality specific vectors to construct weights amplitudes = [F.normalize(rep, dim=-1) for rep in utterance_reps] phases = [ phase_embed(smask.argmax(dim=-1)) for phase_embed in self.phase_embeddings ] weights = [self.norm(rep) for rep in utterance_reps] weights = F.softmax(torch.cat(weights, dim=-1), dim=-1) unimodal_pure = [ self.multiply([phase, amplitude]) for phase, amplitude in zip(phases, amplitudes) ] unimodal_matrices = [self.outer(s) for s in unimodal_pure] rnn_unimodal_data = [ rnn(data) for data, rnn in zip(unimodal_matrices, self.RNNs) ] ## # weights = [self.norm(rep) for rep in rnn_unimodal_data] # weights = F.softmax(torch.cat(weights, dim = -1), dim = -1) emo_states = self.mixture([rnn_unimodal_data, weights]) action_states = self.action_qrnn(emo_states) ###emotion classifier### output_emo = [] for _h in emo_states: measurement_probs = self.measurement_emotion(_h) _output = self.fc_out_emo(measurement_probs) output_emo.append(_output) output_e = torch.stack(output_emo, dim=-2) log_prob_e = F.log_softmax(output_e, 2) # batch, seq_len, n_classes ###action classifier### output_act = [] for _h in emo_states: measurement_probs = self.measurement_act(_h) _output = self.fc_out_act(measurement_probs) output_act.append(_output) output_a = torch.stack(output_act, dim=-2) log_prob_a = F.log_softmax(output_a, 2) # batch, seq_len, n_classes return log_prob_e, log_prob_a
def __init__(self, opt): super(MARN, self).__init__() self.input_dims = opt.input_dims self.output_dim = opt.output_dim if type(opt.hidden_dims) == int: self.hidden_dims = [opt.hidden_dims] else: self.hidden_dims = [int(s) for s in opt.hidden_dims.split(',')] self.device = opt.device self.attn_num = opt.attn_num self.attn_cell_dim = opt.attn_cell_dim self.attn_dropout_rate = opt.attn_dropout_rate if opt.embedding_enabled: embedding_matrix = torch.tensor(opt.lookup_table, dtype=torch.float) self.embed = nn.Embedding.from_pretrained( embedding_matrix, freeze=not opt.embedding_trainable) if type(opt.compression_cell_dims) == int: self.compression_cell_dims = [opt.compression_cell_dims] else: self.compression_cell_dims = [ int(s) for s in opt.compression_cell_dims.split(',') ] if type(opt.compression_dropout_rates) == float: self.compression_dropout_rates = [opt.compression_dropout_rates] else: self.compression_dropout_rates = [ float(s) for s in opt.compression_dropout_rates.split(',') ] self.total_hidden_dim = sum(self.hidden_dims) self.output_cell_dim = opt.output_cell_dim self.output_dropout_rate = opt.output_dropout_rate # self.batch_size = opt.batch_size # initialize the LSTHM for each modalitye if type(opt.compressed_dims) == int: self.compressed_dims = [opt.compressed_dims] else: self.compressed_dims = [ int(s) for s in opt.compressed_dims.split(',') ] self.num_modalities = len(self.input_dims) self.lsthms = nn.ModuleList([LSTHMCell(hidden_dim,input_dim,sum(self.compressed_dims),self.device) \ for input_dim,hidden_dim in zip(self.input_dims,self.hidden_dims)]) attention = SimpleNet(self.total_hidden_dim, self.attn_cell_dim * self.attn_num, self.attn_dropout_rate, self.total_hidden_dim * self.attn_num, nn.Sigmoid()) compression_nets = nn.ModuleList([ SimpleNet(hidden_dim * self.attn_num, compression_cell_dim, compression_dropout_rate, compressed_dim, nn.Sigmoid()) for compressed_dim, hidden_dim, compression_cell_dim, compression_dropout_rate in zip( self.compressed_dims, self.hidden_dims, self.compression_cell_dims, self.compression_dropout_rates) ]) self.multi_attention = MultipleAttention(attention, compression_nets, self.attn_num) self.fc_output_in_dim = sum( self.compressed_dims) + self.total_hidden_dim self.fc_out = SimpleNet(self.fc_output_in_dim, self.output_cell_dim, self.output_dropout_rate, self.output_dim)
def train_model(dataset, model_output, model_type, training_ratio=0.7, validation_ratio=0.1, epoch=10, batch_size=50, learning_rate=0.01): if dataset is None: raise ValueError('No dataset specified') hdf5_file = h5py.File(dataset, 'r') train, validation, test, validation_data = data_split( hdf5_file=hdf5_file, training_ratio=training_ratio, validation_ratio=validation_ratio) train_start, train_end = train validation_start, validation_end = validation test_start, test_end = test shape = input_shape(hdf5_file=hdf5_file) height, width, channels = shape if model_type in [ 'AlexNet', 'alexnet', 'ALEXNET', 'Alex', 'alex', 'ALEX', 'alex_net', 'Alex_Net', 'ALEX_NET' ]: model = AlexNet(learning_rate=learning_rate, width=width, height=height, channels=channels) elif model_type in [ 'NvidiaNet', 'nvidianet', 'NVIDIANET', 'Nvidia', 'nvidia', 'NVIDIA', 'nvidia_net', 'Nvidia_Net', 'NVIDIA_NET' ]: model = NvidiaNet(learning_rate=learning_rate, width=width, height=height, channels=channels) elif model_type in ['VGG16', 'vgg16', 'Vgg16', 'VGG', 'Vgg', 'vgg']: model = VGG16Net(learning_rate=learning_rate, width=width, height=height, channels=channels) elif model_type in [ 'SimpleNet', 'simplenet', 'SIMPLENET', 'Simplenet', 'simple', 'SIMPLE' ]: model = SimpleNet(learning_rate=learning_rate, width=width, height=height, channels=channels) else: raise ValueError('Not a valid model type') sys.exit(0) train_start_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") if model_output: model_output_path, _ = os.path.splitext(model_output) model_filename = model_output_path + '-' + train_start_time else: dataset_path, _ = os.path.splitext(dataset) dataset_filename = dataset_path[dataset_path.find('/') + 1:] model_filename = 'models/' + dataset_filename + '-' + train_start_time loss_checkpoint = ModelCheckpoint(model_filename + '-val-loss-lowest.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min') accuracy_checkpoint = ModelCheckpoint(model_filename + '-val-acc-highest.h5', monitor='val_acc', verbose=1, save_best_only=True, mode='max') early_stopping = EarlyStopping(monitor='val_loss', patience=6, verbose=1, mode='min') reduce_learning_rate = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5) callbacks = [loss_checkpoint, accuracy_checkpoint] if validation_data: train_gen = generate_examples(hdf5_file=hdf5_file, batch_size=batch_size, start=train_start, end=train_end) validation_gen = generate_examples(hdf5_file=hdf5_file, batch_size=batch_size, start=validation_start, end=validation_end) model.fit_generator(train_gen, samples_per_epoch=train_end - train_start, validation_data=validation_gen, nb_val_samples=validation_end - validation_start, nb_epoch=epoch, verbose=1, callbacks=callbacks) else: train_gen = generate_examples(hdf5_file=hdf5_file, batch_size=batch_size, start=train_start, end=train_end) model.fit_generator(train_gen, samples_per_epoch=train_end - train_start, nb_epoch=epoch, verbose=1, callbacks=callbacks) model.save(model_filename + '.h5') # Just to be safe model.save('tmp.h5') return model
class QMPN(nn.Module): def __init__(self, opt): super(QMPN, self).__init__() self.device = opt.device self.input_dims = opt.input_dims self.total_input_dim = sum(self.input_dims) if type(opt.embed_dims) == int: self.embed_dims = [opt.embed_dims] self.embed_dims = [int(s) for s in opt.embed_dims.split(',')] self.speaker_num = opt.speaker_num self.dataset_name = opt.dataset_name # MELD data # The one-hot vectors are not the global user ID if self.dataset_name.lower() == 'meld': self.speaker_num = 1 self.n_classes = opt.output_dim self.projections = nn.ModuleList([nn.Linear(dim, embed_dim) for dim, embed_dim in zip(self.input_dims,self.embed_dims)]) self.multiply = ComplexMultiply() self.outer = QOuter() self.norm = L2Norm(dim = -1) self.mixture = QMixture(device = self.device) self.output_cell_dim = opt.output_cell_dim self.phase_embeddings = nn.ModuleList([PositionEmbedding(embed_dim, input_dim = self.speaker_num, device = self.device) for embed_dim in self.embed_dims]) self.out_dropout_rate = opt.out_dropout_rate self.num_layers = opt.num_layers self.state_product = QProduct() self.recurrent_dim = 1 for embed_dim in self.embed_dims: self.recurrent_dim = self.recurrent_dim*embed_dim self.recurrent_cells = nn.ModuleList([QRNNCell(self.recurrent_dim, device = self.device)]*self.num_layers) self.measurement = QMeasurement(self.recurrent_dim) self.fc_out = SimpleNet(self.recurrent_dim, self.output_cell_dim, self.out_dropout_rate,self.n_classes, output_activation = nn.Tanh()) def get_params(self): unitary_params = [] remaining_params = [] for i in range(self.num_layers): unitary_params.append(self.recurrent_cells[i].unitary_x) unitary_params.append(self.recurrent_cells[i].unitary_h) remaining_params.append(self.recurrent_cells[i].Lambda) remaining_params.extend(list(self.projections.parameters())) remaining_params.extend(list(self.phase_embeddings.parameters())) for i in range(self.num_layers): remaining_params.append(self.recurrent_cells[i].Lambda) unitary_params.extend(list(self.measurement.parameters())) remaining_params.extend(list(self.fc_out.parameters())) return unitary_params, remaining_params def forward(self, in_modalities): smask = in_modalities[-2] # Speaker ids in_modalities = in_modalities[:-2] batch_size = in_modalities[0].shape[0] time_stamps = in_modalities[0].shape[1] # Project All modalities of each utterance to the same space utterance_reps = [nn.ReLU()(projection(x)) for x, projection in zip(in_modalities,self.projections)] amplitudes = [F.normalize(rep, dim = -1) for rep in utterance_reps] phases = [phase_embed(smask.argmax(dim = -1)) for phase_embed in self.phase_embeddings] unimodal_pure = [self.multiply([phase, amplitude]) for phase, amplitude in zip(phases,amplitudes)] multimodal_pure = self.state_product(unimodal_pure) in_states = self.outer(multimodal_pure) # Take the amplitudes # multiply with modality specific vectors to construct weights #weights = [self.norm(rep) for rep in utterance_reps] #weights = F.softmax(torch.cat(weights, dim = -1), dim = -1) #unimodal_matrices = [self.outer(s) for s in unimodal_pure] #in_states = self.mixture([unimodal_matrices, weights]) for l in range(self.num_layers): # Initialize the cell h h_r = torch.stack(batch_size*[torch.eye(self.recurrent_dim)/self.recurrent_dim],dim =0) h_i = torch.zeros_like(h_r) h = [h_r.to(self.device),h_i.to(self.device)] all_h = [] for t in range(time_stamps): h = self.recurrent_cells[l](in_states[t],h) all_h.append(self.activation(h)) in_states = all_h output = [] for _h in in_states: measurement_probs = self.measurement(_h) _output = self.fc_out(measurement_probs) output.append(_output) output = torch.stack(output, dim=-2) log_prob = F.log_softmax(output, 2) # batch, seq_len, n_classes return log_prob
def main(params, config): dataset = PedestalDataset(config) train_loader, validation_loader = split_dataset(dataset, params['batch_size']) if config['experiment']['load_model'] != None: PATH = config['experiment']['load_model'] checkpoint = torch.load(PATH) # Load Model net = SimpleNet(params, config) net.load_state_dict(checkpoint['model_state_dict']) # Load Optimizer optimizer = model_utils.map_optimizer(params['optimizer'], net.parameters(), 0.0) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # Assign Loss Function loss_func = model_utils.map_loss_func(params['loss']) # Set EPOCH and LOSS for retraining epoch = checkpoint['epoch'] loss = checkpoint['loss'] else: net = SimpleNet(params, config) optimizer = model_utils.map_optimizer(params['optimizer'], net.parameters(), params['learning_rate']) loss_func = model_utils.map_loss_func(params['loss']) metrics = {} target_norms = dataset.targets_norms input_norms = dataset.inputs_norms target_list = dataset.target_params input_list = dataset.input_params net.eval() outputs = [] actual_array = [] scaled_list = [] save_path = config['experiment']['name'] for i, batch in enumerate(validation_loader): inputs, targets = batch['input'], batch['target'] for val in inputs: output = net(val).detach().numpy() output = denormalize(output, target_list, target_norms) outputs.append(output[0]) normed_vals = denormalize(val.numpy(), input_list, input_norms) for tar in targets: denorm_targ = denormalize(tar.numpy(), target_list, target_norms) actual_array.append(denorm_targ[0]) if config['experiment']['target'] == 'density': for i, batch in enumerate(validation_loader): inputs, targets = batch['input'], batch['target'] for val in inputs: normed_vals = denormalize(val.numpy(), input_list, input_norms) scaled_vals = fitted_scale(normed_vals) scaled_list.append(scaled_vals) plt.scatter(actual_array, scaled_list, label='Scale Law') plt.scatter(actual_array, actual_array, label='Actual') plt.scatter(actual_array, outputs, label='NN') plt.legend() plt.ylabel('Predicted') plt.xlabel('Actual Density Height') plt.ylim(0, 12) plt.title('Neural Network vs Scaling Law') plt.savefig('./results/' + save_path) plt.show()