def play_synthesized_features( feats, norm_path='/home/ubuntu/loop/data/vctk/norm_info/norm.dat'): output_dir = './' output_file = 'test.wav' generate_merlin_wav(feats['audio_features'], output_dir, output_file, norm_path) IPython.display.display( IPython.display.Audio(output_file + '.wav', autoplay=True)) rate, wav_data = wavfile.read(output_file + '.wav') return rate, wav_data
def main(): weights = torch.load(args.checkpoint, map_location=lambda storage, loc: storage) opt = torch.load(os.path.dirname(args.checkpoint) + '/args.pth') train_args = opt[0] char2code = {'aa': 0, 'ae': 1, 'ah': 2, 'ao': 3, 'aw': 4, 'ax': 5, 'ay': 6, 'b': 7, 'ch': 8, 'd': 9, 'dh': 10, 'eh': 11, 'er': 12, 'ey': 13, 'f': 14, 'g': 15, 'hh': 16, 'i': 17, 'ih': 18, 'iy': 19, 'jh': 20, 'k': 21, 'l': 22, 'm': 23, 'n': 24, 'ng': 25, 'ow': 26, 'oy': 27, 'p': 28, 'pau': 29, 'r': 30, 's': 31, 'sh': 32, 'ssil': 33, 't': 34, 'th': 35, 'uh': 36, 'uw': 37, 'v': 38, 'w': 39, 'y': 40, 'z': 41} nspkr = train_args.nspk norm_path = None if os.path.exists(train_args.data + '/norm_info/norm.dat'): norm_path = train_args.data + '/norm_info/norm.dat' elif os.path.exists(os.path.dirname(args.checkpoint) + '/norm.dat'): norm_path = os.path.dirname(args.checkpoint) + '/norm.dat' else: print('ERROR: Failed to find norm file.') return train_args.noise = 0 model = Loop(train_args) model.load_state_dict(weights) if args.gpu >= 0: model.cuda() model.eval() if args.spkr not in range(nspkr): print('ERROR: Unknown speaker id: %d.' % args.spkr) return txt, feat, spkr, output_fname = None, None, None, None if args.npz is not '': txt, text, feat = npy_loader_phonemes(args.npz) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(torch.LongTensor([args.spkr]), volatile=True) fname = os.path.basename(args.npz)[:-4] output_fname = fname + '.gen_' + str(args.spkr) words = np.char.split(text).tolist() words = [word.encode('utf-8') for word in words] action = 'none' number = 'none' objectt = 'none' location = 'none' # Remove extra word for special cases if len(words) == 7: words = words[1:] action = words[0] if len(words) == 2: objectt = words[1] elif len(words) > 3: number = words[1] objectt = words[2] location = words[-1] #print(words[0], words[1], words[2], words[-1]) #print(text) # Read dataframe frames = {} if os.path.exists(args.dataset_file): df = pd.read_csv(args.dataset_file) for row in zip(*[df[col].values.tolist() for col in ['path', 'speakerId', 'transcription', 'action', 'number', 'object', 'location']]): frames[row[0]] = {'path': row[0], 'speakerId': row[1], 'transcription': row[2], 'action': row[3], 'number': row[4], 'object': row[5], 'location': row[6]} # Add new data path = os.path.join('wavs/synthetic', output_fname.strip("/") + '.wav') frames[path] = {'path': path, 'speakerId': args.spkr, 'transcription': text, 'action': action, 'number': number, 'object': objectt, 'location': location} paths = [] speakerIds = [] transcriptions = [] actions = [] numbers = [] objects = [] locations = [] for key, frame in frames.items(): paths.append(frame['path']) speakerIds.append(frame['speakerId']) transcriptions.append(frame['transcription']) actions.append(frame['action']) numbers.append(frame['number']) objects.append(frame['object']) locations.append(frame['location']) df = pd.DataFrame(OrderedDict([('path', paths), ('speakerId', speakerIds), ('transcription', transcriptions), ('action', actions), ('number', numbers), ('object', objects), ('location', locations)])) df.to_csv(args.dataset_file) else: print('ERROR: Must supply npz file path or text as source.') return ### key_list = list(char2code.keys()) val_list = list(char2code.values()) phrase = [key_list[val_list.index(letter)] for letter in txt.data.numpy()] #print(phrase) ### if args.gpu >= 0: txt = txt.cuda() feat = feat.cuda() spkr = spkr.cuda() out, attn = model([txt, spkr], feat) out, attn = trim_pred(out, attn) output_dir = os.path.join(os.path.dirname(args.checkpoint), 'results') if not os.path.exists(output_dir): os.makedirs(output_dir) generate_merlin_wav(out.data.cpu().numpy(), output_dir, output_fname, norm_path)
def generate_sample_with_loop( npz='', text='', spkr_id=1, gender=1, checkpoint='models/vctk-16khz-cmu-no-boundaries-all/bestmodel.pth', output_dir='./', npz_path='/home/ubuntu/loop/data/vctk-16khz-cmu-no-boundaries-all/numpy_features', output_file_override=None, ident_override=None): # npz = '' # text = 'Your tickets for the social issues' # text = 'see that girl watch that scene' # npz = '/home/ubuntu/loop/data/vctk/numpy_features/p294_011.npz' # spkr_id = 12 # checkpoint = 'checkpoints/vctk/lastmodel.pth' # checkpoint = 'models/vctk/bestmodel.pth' gender = np.array(gender).reshape(-1) out_dict = dict() if not os.path.exists(output_dir): os.makedirs(output_dir) gpu = 0 # load loop weights & params from checkpoint weights = torch.load(checkpoint, map_location=lambda storage, loc: storage) opt = torch.load(os.path.dirname(checkpoint) + '/args.pth') train_args = opt[0] train_dataset = NpzFolder( '/home/ubuntu/loop/data/vctk-16khz-cmu-no-boundaries-all/numpy_features' ) char2code = train_dataset.dict spkr2code = train_dataset.speakers # print spkr2code.cpu().data norm_path = train_args.data + '/norm_info/norm.dat' norm_path = '/home/ubuntu/loop/data/vctk-16khz-cmu-no-boundaries-all/norm_info/norm.dat' train_args.noise = 0 valid_dataset_path = npz_path + '_valid' # prepare loop model if ident_override: #model = Loop_Ident(train_args) pass else: model = Loop_Base(train_args) model.load_state_dict(weights) if gpu >= 0: model.cuda() model.eval() # check speaker id is valid if spkr_id not in range(len(spkr2code)): print('ERROR: Unknown speaker id: %d.' % spkr_id) # get phone sequence txt, feat, spkr, output_fname = None, None, None, None if npz is not '': # use pre-calculated phonemes etc. txt, feat, pre_calc_feat = npy_loader_phonemes( os.path.join(npz_path, npz)) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(torch.LongTensor([spkr_id]), volatile=True) output_file = os.path.basename(npz)[:-4] + '_' + str(spkr_id) out_dict['pre_calc_feat'] = pre_calc_feat elif text is not '': # use specified text string # extract phonemes from the text txt = text2phone(text, char2code) feat = torch.FloatTensor(txt.size(0) * 20, 63) spkr = torch.LongTensor([spkr_id]) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(spkr, volatile=True) output_file = text.replace(' ', '_') else: print('ERROR: Must supply npz file path or text as source.') raise Exception('Need source') if output_file_override: output_file = output_file_override # use gpu if gpu >= 0: txt = txt.cuda() feat = feat.cuda() spkr = spkr.cuda() # run loop model to generate output features # print(ident_override) if ident_override: loop_feat, attn = model([txt, spkr, gender], feat, ident_override=ident_override) else: loop_feat, attn = model([txt, spkr, gender], feat) loop_feat, attn = trim_pred(loop_feat, attn) # add to output dictionary out_dict['txt'] = txt[:, 0].squeeze().data.tolist() out_dict['spkr'] = spkr out_dict['feat'] = feat.data.cpu().numpy() out_dict['loop_feat'] = loop_feat.data.cpu().numpy() out_dict['attn'] = attn.squeeze().data.cpu().numpy() out_dict['output_file'] = output_file out_dict['valid_dataset_path'] = valid_dataset_path # print output_dir # generate .wav file from loop output features #print(output_dir) #print(output_file) #print(norm_path) generate_merlin_wav(loop_feat.data.cpu().numpy(), output_dir, output_file, norm_path) # generate .wav file from original features for reference if npz is not '': output_orig_fname = os.path.basename(npz)[:-4] + '.orig' generate_merlin_wav(feat[:, 0, :].data.cpu().numpy(), output_dir, output_orig_fname, norm_path) out_dict['output_orig_fname'] = output_orig_fname return out_dict
def main(): weights = torch.load(args.checkpoint, map_location=lambda storage, loc: storage) opt = torch.load(os.path.dirname(args.checkpoint) + '/args.pth') train_args = opt[0] char2code = {'aa': 0, 'ae': 1, 'ah': 2, 'ao': 3, 'aw': 4, 'ax': 5, 'ay': 6, 'b': 7, 'ch': 8, 'd': 9, 'dh': 10, 'eh': 11, 'er': 12, 'ey': 13, 'f': 14, 'g': 15, 'hh': 16, 'i': 17, 'ih': 18, 'iy': 19, 'jh': 20, 'k': 21, 'l': 22, 'm': 23, 'n': 24, 'ng': 25, 'ow': 26, 'oy': 27, 'p': 28, 'pau': 29, 'r': 30, 's': 31, 'sh': 32, 'ssil': 33, 't': 34, 'th': 35, 'uh': 36, 'uw': 37, 'v': 38, 'w': 39, 'y': 40, 'z': 41} nspkr = train_args.nspk norm_path = None if os.path.exists(train_args.data + '/norm_info/norm.dat'): norm_path = train_args.data + '/norm_info/norm.dat' elif os.path.exists(os.path.dirname(args.checkpoint) + '/norm.dat'): norm_path = os.path.dirname(args.checkpoint) + '/norm.dat' else: print('ERROR: Failed to find norm file.') return train_args.noise = 0 model = Loop(train_args) model.load_state_dict(weights) if args.gpu >= 0: model.cuda() model.eval() if args.spkr not in range(nspkr): print('ERROR: Unknown speaker id: %d.' % args.spkr) return txt, feat, spkr, output_fname = None, None, None, None if args.npz is not '': txt, feat = npy_loader_phonemes(args.npz) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(torch.LongTensor([args.spkr]), volatile=True) fname = os.path.basename(args.npz)[:-4] output_fname = fname + '.gen_' + str(args.spkr) elif args.text is not '': txt = text2phone(args.text, char2code) feat = torch.FloatTensor(txt.size(0)*20, 63) spkr = torch.LongTensor([args.spkr]) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(spkr, volatile=True) # slugify input string to file name fname = args.text.replace(' ', '_') valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) fname = ''.join(c for c in fname if c in valid_chars) output_fname = fname + '.gen_' + str(args.spkr) else: print('ERROR: Must supply npz file path or text as source.') return if args.gpu >= 0: txt = txt.cuda() feat = feat.cuda() spkr = spkr.cuda() out, attn = model([txt, spkr], feat) out, attn = trim_pred(out, attn) output_dir = os.path.join(os.path.dirname(args.checkpoint), 'results') if not os.path.exists(output_dir): os.makedirs(output_dir) generate_merlin_wav(out.data.cpu().numpy(), output_dir, output_fname, norm_path) if args.npz is not '': output_orig_fname = os.path.basename(args.npz)[:-4] + '.orig' generate_merlin_wav(feat[:, 0, :].data.cpu().numpy(), output_dir, output_orig_fname, norm_path)
def main(): weights = torch.load(args.checkpoint, map_location=lambda storage, loc: storage) opt = torch.load(os.path.dirname(args.checkpoint) + '/args.pth') train_args = opt[0] train_dataset = NpzFolder(train_args.data + '/numpy_features') char2code = train_dataset.dict spkr2code = train_dataset.speakers norm_path = train_args.data + '/norm_info/norm.dat.npy' train_args.noise = 0 model = Loop(train_args) model.load_state_dict(weights) if args.gpu >= 0: model.cuda() model.eval() if args.spkr not in range(len(spkr2code)): print('ERROR: Unknown speaker id: %d.' % args.spkr) return txt, feat, spkr, output_fname = None, None, None, None if args.npz is not '': txt, feat = npy_loader_phonemes(args.npz) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(torch.LongTensor([args.spkr]), volatile=True) fname = os.path.basename(args.npz)[:-4] output_fname = fname + '.gen_' + str(args.spkr) elif args.text is not '': txt = text2phone(args.text, char2code) #feat = torch.FloatTensor(500, 67) feat = torch.FloatTensor(1500, 67) spkr = torch.LongTensor([args.spkr]) txt = Variable(txt.unsqueeze(1), volatile=True) feat = Variable(feat.unsqueeze(1), volatile=True) spkr = Variable(spkr, volatile=True) fname = args.text.replace(' ', '_') output_fname = fname + '.gen_' + str(args.spkr) else: print('ERROR: Must supply npz file path or text as source.') return if args.gpu >= 0: txt = txt.cuda() feat = feat.cuda() spkr = spkr.cuda() out, attn = model([txt, spkr], feat) out, attn = trim_pred(out, attn) output_dir = os.path.join(os.path.dirname(args.checkpoint), 'results') if not os.path.exists(output_dir): os.makedirs(output_dir) #''' generate_merlin_wav(out.data.cpu().numpy(), output_dir, output_fname, norm_path) #''' #out.data.cpu().numpy().tofile(output_fname) if args.npz is not '': output_orig_fname = os.path.basename(args.npz)[:-4] + '.orig' generate_merlin_wav(feat[:, 0, :].data.cpu().numpy(), output_dir, output_orig_fname, norm_path)