def main(): os.environ["WSJ_PATH"] = "/home/ubuntu/hw1 Speech Processing/pt2_data" loader = WSJ() train = loader.train print('Train') print('X Shape:', train[0].shape) print('Y Shape:', train[1].shape) val = loader.dev print('Val') print('X Shape:', val[0].shape) print('Y Shape:', val[1].shape) test = loader.test print('Test') print('X Shape:', test[0].shape) load_and_test = sys.argv[1] model_name = sys.argv[2] mlp = Flexible_MLP(12, (40, 512, 1024, 1024, 512), (2, 2, 2, 2, 2), 138, load_and_test, model_name) print('Initialized model') if load_and_test == "False": mlp.train(train, val, 14, True, 0.01) out = mlp.test(test, True) print(out)
def get_data(config): num_workers = config['num_workers'] batch_size = config['batch_size'] k = config['k'] wsj = WSJ(config['data_path']) train_set = SpeechDataset(wsj.train[0], wsj.train[1], k) val_set = SpeechDataset(wsj.dev[0], wsj.dev[1], k) test_set = SpeechDataset(wsj.test[0], wsj.test[1], k) train_data = DataHanlder(train_set, True, batch_size, num_workers) val_data = DataHanlder(val_set, False, batch_size, num_workers) test_data = DataHanlder(test_set, False, batch_size, num_workers) return train_data, val_data, test_data
from wsj_loader import WSJ import numpy as np # loading training data loader = WSJ() trainX, trainY = loader.train N = np.shape(trainX)[0] m = sum([np.shape(trainX[i])[0] for i in range(N)]) #print(m) # 15449191 trainX_pad = np.empty((m, 40)) trainY_pad = np.empty(m) train_utter_len = np.empty(m) train_utter_idx = np.empty(m) line_idx = 0 for n in range(N): l = np.shape(trainX[n])[0] trainX_pad[line_idx:(line_idx + l), ] = trainX[n] trainY_pad[line_idx:(line_idx + l)] = trainY[n] train_utter_len[line_idx:(line_idx + l)] = l * np.ones(l) train_utter_idx[line_idx:(line_idx + l)] = np.array(range(l)) line_idx += l np.save('processed/trainX_pad', trainX_pad) np.save('processed/trainY_pad', trainY_pad) np.save('processed/train_utter_len', train_utter_len) np.save('processed/train_utter_idx', train_utter_idx) # loading validation data devX, devY = loader.dev N = np.shape(devX)[0] m = sum([np.shape(devX[i])[0] for i in range(N)])
# pos = 0 # for i in range(out_predicted.size(0)): # for all utterances # pred = "".join([o-1] for o in out_predicted[i, 0, :predicted_seq_len[i, 0]]) # print(pred) # final = np.append(final, pred) # del inputs,sorted_lens, targets_sorted_test, logits,probs,out_predicted, scores, timesteps pd.DataFrame(data=final).to_csv(filename,header=False, index=True) model.train() return if __name__ == "__main__": device = "cuda" if torch.cuda.is_available() else "cpu" device ld = WSJ() trainX, trainY = ld.train valX,valY=ld.dev testX = np.load('../data/test.npy',encoding='bytes') print("trainlens",len(trainY)) print("data_loaded") # small_test=np.array([a for a in testX[1:10]]) # small_train_X=np.array([a for a in trainX[1:100]]) # small_train_Y=np.array([y for y in trainY[1:100]]) # small_dev_X=np.array([a for a in valX[1:10]]) # small_dev_Y=np.array([y for y in valY[1:10]]) # np.save("small_test",small_test) # np.save("small_dev_X",small_dev_X) # np.save("small_dev_Y",small_dev_Y)