def main():
os.environ["WSJ_PATH"] = "/home/ubuntu/hw1 Speech Processing/pt2_data"
loader = WSJ()
train = loader.train
print('Train')
print('X Shape:', train[0].shape)
print('Y Shape:', train[1].shape)
val = loader.dev
print('Val')
print('X Shape:', val[0].shape)
print('Y Shape:', val[1].shape)
test = loader.test
print('Test')
print('X Shape:', test[0].shape)
load_and_test = sys.argv[1]
model_name = sys.argv[2]
mlp = Flexible_MLP(12, (40, 512, 1024, 1024, 512), (2, 2, 2, 2, 2), 138,
load_and_test, model_name)
print('Initialized model')
if load_and_test == "False":
mlp.train(train, val, 14, True, 0.01)
out = mlp.test(test, True)
print(out)
def get_data(config):
num_workers = config['num_workers']
batch_size = config['batch_size']
k = config['k']
wsj = WSJ(config['data_path'])
train_set = SpeechDataset(wsj.train[0], wsj.train[1], k)
val_set = SpeechDataset(wsj.dev[0], wsj.dev[1], k)
test_set = SpeechDataset(wsj.test[0], wsj.test[1], k)
train_data = DataHanlder(train_set, True, batch_size, num_workers)
val_data = DataHanlder(val_set, False, batch_size, num_workers)
test_data = DataHanlder(test_set, False, batch_size, num_workers)
return train_data, val_data, test_data
from wsj_loader import WSJ
import numpy as np
# loading training data
loader = WSJ()
trainX, trainY = loader.train
N = np.shape(trainX)[0]
m = sum([np.shape(trainX[i])[0] for i in range(N)])
#print(m) # 15449191
trainX_pad = np.empty((m, 40))
trainY_pad = np.empty(m)
train_utter_len = np.empty(m)
train_utter_idx = np.empty(m)
line_idx = 0
for n in range(N):
l = np.shape(trainX[n])[0]
trainX_pad[line_idx:(line_idx + l), ] = trainX[n]
trainY_pad[line_idx:(line_idx + l)] = trainY[n]
train_utter_len[line_idx:(line_idx + l)] = l * np.ones(l)
train_utter_idx[line_idx:(line_idx + l)] = np.array(range(l))
line_idx += l
np.save('processed/trainX_pad', trainX_pad)
np.save('processed/trainY_pad', trainY_pad)
np.save('processed/train_utter_len', train_utter_len)
np.save('processed/train_utter_idx', train_utter_idx)
# loading validation data
devX, devY = loader.dev
N = np.shape(devX)[0]
m = sum([np.shape(devX[i])[0] for i in range(N)])
# pos = 0
# for i in range(out_predicted.size(0)): # for all utterances
# pred = "".join([o-1] for o in out_predicted[i, 0, :predicted_seq_len[i, 0]])
# print(pred)
# final = np.append(final, pred)
# del inputs,sorted_lens, targets_sorted_test, logits,probs,out_predicted, scores, timesteps
pd.DataFrame(data=final).to_csv(filename,header=False, index=True)
model.train()
return
if __name__ == "__main__":
device = "cuda" if torch.cuda.is_available() else "cpu"
device
ld = WSJ()
trainX, trainY = ld.train
valX,valY=ld.dev
testX = np.load('../data/test.npy',encoding='bytes')
print("trainlens",len(trainY))
print("data_loaded")
# small_test=np.array([a for a in testX[1:10]])
# small_train_X=np.array([a for a in trainX[1:100]])
# small_train_Y=np.array([y for y in trainY[1:100]])
# small_dev_X=np.array([a for a in valX[1:10]])
# small_dev_Y=np.array([y for y in valY[1:10]])
# np.save("small_test",small_test)
# np.save("small_dev_X",small_dev_X)
# np.save("small_dev_Y",small_dev_Y)