async def create_upload_file(text: str): texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Created the mel spectrogram") # Generating the waveform print("Synthesizing the waveform:") # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. generated_wav = vocoder.infer_waveform(spec) # Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") # Trim excess silences to compensate for gaps in spectrograms (issue #53) generated_wav = encoder.preprocess_wav(generated_wav) # Save it on the disk output = BytesIO() wavfile.write(output, synthesizer.sample_rate, generated_wav.astype(np.float32)) return StreamingResponse(output, media_type="audio/x-wav")
def vocode(self): speaker_name, spec, breaks, _ = self.current_generated assert spec is not None # Synthesize the waveform if not vocoder.is_loaded(): self.init_vocoder() def vocoder_progress(i, seq_len, b_size, gen_rate): real_time_factor = (gen_rate / Synthesizer.sample_rate) * 1000 line = "Waveform generation: %d/%d (batch size: %d, rate: %.1fkHz - %.2fx real time)" \ % (i * b_size, seq_len * b_size, b_size, gen_rate, real_time_factor) self.ui.log(line, "overwrite") self.ui.set_loading(i, seq_len) if self.ui.current_vocoder_fpath is not None: self.ui.log("") wav = vocoder.infer_waveform(spec, progress_callback=vocoder_progress) else: self.ui.log("Waveform generation with Griffin-Lim... ") wav = Synthesizer.griffin_lim(spec) self.ui.set_loading(0) self.ui.log(" Done!", "append") # Add breaks b_ends = np.cumsum(np.array(breaks) * Synthesizer.hparams.hop_size) b_starts = np.concatenate(([0], b_ends[:-1])) wavs = [wav[start:end] for start, end, in zip(b_starts, b_ends)] breaks = [np.zeros(int(0.15 * Synthesizer.sample_rate))] * len(breaks) wav = np.concatenate([i for w, b in zip(wavs, breaks) for i in (w, b)]) # Play it wav = wav / np.abs(wav).max() * 0.97 self.ui.play(wav, Synthesizer.sample_rate) fref = '-'.join([self.ui.current_dataset_name, self.ui.current_speaker_name, self.ui.current_utterance_name]) ftime = '{}'.format(int(time.time())) ftext = self.ui.text_prompt.toPlainText() fms = int(len(wav) * 1000 / Synthesizer.sample_rate) fname = filename_formatter('{}_{}_{}ms_{}.wav'.format(fref, ftime, fms, ftext)) audio.save_wav(wav, _out_wav_dir.joinpath(fname), Synthesizer.sample_rate) # save # Compute the embedding # TODO: this is problematic with different sampling rates, gotta fix it if not encoder.is_loaded(): self.init_encoder() encoder_wav = encoder.preprocess_wav(wav) embed, partial_embeds, _ = encoder.embed_utterance(encoder_wav, return_partials=True) # Add the utterance name = speaker_name + "_gen_%05d" % int(time.time()) utterance = Utterance(name, speaker_name, wav, spec, embed, partial_embeds, True) np.save(_out_embed_dir.joinpath(name + '.npy'), embed, allow_pickle=False) # save self.utterances.add(utterance) # Plot it self.ui.draw_embed(embed, name, "generated") self.ui.draw_umap_projections(self.utterances)
def text_to_speech(text: str, play_sound: bool = True): start = timer() texts = [text] embeds = [embed] print("Creating the MEL spectrogram") with nostdout(): specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Generating audio") with nostdout(): generated_wav = vocoder.infer_waveform(spec) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate)) # Trim excess silences to compensate for gaps in spectrograms (issue #53) generated_wav = encoder.preprocess_wav(generated_wav) filename = f'tts_generated{filenum}.wav' sf.write(filename, generated_wav.astype(np.float32), synthesizer.sample_rate) print(text) if play_sound: os.system(f'afplay {filename} &') elapsed_time = timer() - start print(f'Generated in {elapsed_time}')
def vocode(self): speaker_name, spec, breaks, _ = self.current_generated assert spec is not None # Synthesize the waveform if not vocoder.is_loaded(): self.init_vocoder() def vocoder_progress(i, seq_len, b_size, gen_rate): real_time_factor = (gen_rate / Synthesizer.sample_rate) * 1000 line = "Waveform generation: %d/%d (batch size: %d, rate: %.1fkHz - %.2fx real time)" \ % (i * b_size, seq_len * b_size, b_size, gen_rate, real_time_factor) self.ui.log(line, "overwrite") self.ui.set_loading(i, seq_len) if self.ui.current_vocoder_fpath is not None: self.ui.log("") wav = vocoder.infer_waveform(spec, progress_callback=vocoder_progress) else: self.ui.log("Waveform generation with Griffin-Lim... ") wav = Synthesizer.griffin_lim(spec) self.ui.set_loading(0) self.ui.log(" Done!", "append") # Add breaks b_ends = np.cumsum(np.array(breaks) * Synthesizer.hparams.hop_size) b_starts = np.concatenate(([0], b_ends[:-1])) wavs = [wav[start:end] for start, end, in zip(b_starts, b_ends)] breaks = [np.zeros(int(0.15 * Synthesizer.sample_rate))] * len(breaks) wav = np.concatenate([i for w, b in zip(wavs, breaks) for i in (w, b)]) # Play it wav = wav / np.abs(wav).max() * 0.97 self.ui.play(wav, Synthesizer.sample_rate) self.ui.save_button.setDisabled(False) # Compute the embedding # TODO: this is problematic with different sampling rates, gotta fix it if not encoder.is_loaded(): self.init_encoder() encoder_wav = encoder.preprocess_wav(wav) embed, partial_embeds, _ = encoder.embed_utterance( encoder_wav, return_partials=True) # Add the utterance if not speaker_name is None: name = speaker_name else: name = "unknown" name = name + "_gen_%05d" % np.random.randint(100000) utterance = Utterance(name, speaker_name, wav, spec, embed, partial_embeds, True) self.utterances.add(utterance) self.ui.register_utterance(utterance) # Plot it self.ui.draw_embed(embed, name, "generated") self.ui.draw_umap_projections(self.utterances)
def generate_voice(self, in_fpath, text, out_fpath): try: preprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) print("Loaded file successfully") embed = encoder.embed_utterance(preprocessed_wav) print("Created the embedding") texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = self.synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Created the mel spectrogram") ## Generating the waveform print("Synthesizing the waveform:") generated_wav = vocoder.infer_waveform(spec) generated_wav = np.pad(generated_wav, (0, self.synthesizer.sample_rate), mode="constant") librosa.output.write_wav(out_fpath, generated_wav.astype(np.float32), self.synthesizer.sample_rate) print("\nSaved output as %s\n\n" % out_fpath) except Exception as e: print("Caught exception: %s" % repr(e)) print("Restarting\n")
def synth(text, audio_file): """ Parameters ---------- text : string text to be said in synthesized voice audio_file : filepath filepath for audio file in wav format Returns ------- generated_wav : numpy.ndarray Numpy padded array of synthesized audio signal """ in_fpath = Path("audio.wav") original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) print("Synthesizing new audio...") specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") return generated_wav
def initialize(self): print("Running a test of your configuration...\n") if not torch.cuda.is_available(): print( "Your PyTorch installation is not configured to use CUDA. If you have a GPU ready " "for deep learning, ensure that the drivers are properly installed, and that your " "CUDA version matches your PyTorch installation. CPU-only inference is currently " "not supported.") quit(-1) print("PyTorch is available and working...") device_id = torch.cuda.current_device() gpu_properties = torch.cuda.get_device_properties(device_id) print( "Found %d GPUs available. Using GPU %d (%s) of compute capability %d.%d with " "%.1fGb total memory.\n" % (torch.cuda.device_count(), device_id, gpu_properties.name, gpu_properties.major, gpu_properties.minor, gpu_properties.total_memory / 1e9)) ## Load the models one by one. print("Preparing the encoder, the synthesizer and the vocoder...") encoder.load_model(self.enc_model_fpath) vocoder.load_model(self.voc_model_fpath) ## Run a test print("Testing your configuration with small inputs.") print("\tTesting the encoder...") encoder.embed_utterance(np.zeros(encoder.sampling_rate)) embed = np.random.rand(speaker_embedding_size) embed /= np.linalg.norm(embed) embeds = [embed, np.zeros(speaker_embedding_size)] texts = ["test 1", "test 2"] print( "\tTesting the synthesizer... (loading the model will output a lot of text)" ) mels = self.synthesizer.synthesize_spectrograms(texts, embeds) mel = np.concatenate(mels, axis=1) no_action = lambda *args: None print("\tTesting the vocoder...") vocoder.infer_waveform(mel, target=200, overlap=50, progress_callback=no_action) print("All test passed! You can now synthesize speech.\n\n")
def synthesize(embed, text): print("Synthesizing new audio...") #with io.capture_output() as captured: specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") print(type(generated_wav)) return
def synthesize(embed, text): specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") print(synthesize.sample_rate) sf.write('infer5.wav', generated_wav, synthesizer.sample_rate, 'PCM_24')
def synthesize_and_save(inpath, outpath): spec=np.load(inpath).T generated_wav = vocoder.infer_waveform(spec) ## Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. generated_wav = np.pad(generated_wav, (0, 16000), mode="constant") # Save it on the disk librosa.output.write_wav(outpath, generated_wav.astype(np.float32), 16000) pass
async def generate_wav(text, filename): user_id = "russell" embed_path = "user_data/embeds/{}.npy".format(user_id) embed_path = Path(embed_path) if embed_path.is_file(): embed = np.load(embed_path) print("load embedding in {}".format(embed_path)) else: raise ("user embedding not found") # ================== synthesizer ================== start_time = time.time() # The synthesizer works in batch, so you need to put your data in a list or numpy array texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Created the mel spectrogram") print("--- synthesizer: %s seconds ---" % (time.time() - start_time)) # ================== vocoder ================== start_time = time.time() # If seed is specified, reset torch seed and reload vocoder if args.seed is not None: torch.manual_seed(args.seed) vocoder.load_model(args.voc_model_fpath) # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. generated_wav = vocoder.infer_waveform(spec) print("") print("--- vocoder: %s seconds ---" % (time.time() - start_time)) # ================== post generation ================== start_time = time.time() # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") # Trim excess silences to compensate for gaps in spectrograms (issue #53) generated_wav = encoder.preprocess_wav(generated_wav) print("--- post generation: %s seconds ---" % (time.time() - start_time)) sf.write("./user_data/generated_voice/%s/"%(user_id) + "%s.wav"%filename, \ generated_wav.astype(np.float32), synthesizer.sample_rate)
def clone_voice(sentence, results_file): """Adapted from 'demo_cli.py'""" u_path = Path('utterance.wav') results_path = Path(results_file) preprocessed_wav = encoder.preprocess_wav(u_path) embed = encoder.embed_utterance(preprocessed_wav) specs = synthesizer.synthesize_spectrograms([sentence], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") librosa.output.write_wav(results_path, generated_wav.astype(np.float32), synthesizer.sample_rate)
def text_callback(bot, update): spectrogram = synthesizer.synthesize_spectrograms([update.message.text], [embedding]) wav = vocoder.infer_waveform(spectrogram[0]) wav = np.pad(wav, (0, synthesizer.sample_rate), mode="constant") librosa.output.write_wav(f"{output_path}{update.effective_chat.id}.wav", wav, sr=synthesizer.sample_rate) bot.send_voice(chat_id=update.effective_chat.id, voice=open(f"{output_path}{update.effective_chat.id}.wav", "rb"), timeout=100)
def generate_wav(text, num_generated, synthesizer, sampling_rate, embed, debug = False): texts = [text] embeds = [embed] specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] generated_wav = vocoder.infer_waveform(spec, True, False) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") print("zeros=", np.count_nonzero(generated_wav==0), "\n") fpath = "output_%02d.wav" % num_generated librosa.output.write_wav(fpath, generated_wav.astype(np.float32), synthesizer.sample_rate) if debug: sd.stop() sd.play(generated_wav, synthesizer.sample_rate)
def audioFromEmbeds(filename, embed): textPath = dataPath + "/" + filename + ".txt" #reading the text file and prepare the text string for the synthesizer textFile = open(textPath) text = textFile.read().replace("\n", " ") textFile.close() #synthesize the text together with the embeds specs = synthesizer.synthesize_spectrograms([text], [embed]) #generate the audio using the vocoder generated_wav = vocoder.infer_waveform(specs[0]) return np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant")
def gen_audio(self, ref_audio, text): try: in_fpath = Path(ref_audio.replace("\"", "").replace("\'", "")) ## Computing the embedding # First, we load the wav using the function that the speaker encoder provides. This is # important: there is preprocessing that must be applied. # The following two methods are equivalent: # - Directly load from the filepath: # preprocessed_wav = encoder.preprocess_wav(in_fpath) # - If the wav is already loaded: original_wav, sampling_rate = librosa.load(in_fpath, sr=None) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) print("Loaded file succesfully, rate=%s" % sampling_rate) # Then we derive the embedding. There are many functions and parameters that the # speaker encoder interfaces. These are mostly for in-depth research. You will typically # only use this function (with its default parameters): embed = encoder.embed_utterance(preprocessed_wav) print("Created the embedding") ## Generating the spectrogram # The synthesizer works in batch, so you need to put your data in a list or numpy array texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = self.synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Created the mel spectrogram") ## Generating the waveform print("Synthesizing the waveform:") # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. generated_wav = vocoder.infer_waveform(spec) ## Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. # generated_wav = np.pad(generated_wav, (0, self.synthesizer.sample_rate), mode="constant") print("\n samples = %s @ %s" % (len(generated_wav), self.synthesizer.sample_rate)) return generated_wav except Exception as e: traceback.print_exc() print("Caught exception: %s" % repr(e)) print("Restarting\n")
def DeepTalk_vocoder(synthesized_mel, breaks, model_save_path, normalize=True): vocoder.load_model(model_save_path) no_action = lambda *args: None wav1 = vocoder.infer_waveform(synthesized_mel, progress_callback=no_action, normalize=normalize) # Add breaks b_ends = np.cumsum(np.array(breaks) * Synthesizer.hparams.hop_size) b_starts = np.concatenate(([0], b_ends[:-1])) wavs = [wav1[start:end] for start, end, in zip(b_starts, b_ends)] breaks = [np.zeros(int(0.15 * Synthesizer.sample_rate))] * len(breaks) wav1 = np.concatenate([i for w, b in zip(wavs, breaks) for i in (w, b)]) wav1 = wav1 / np.abs(wav1).max() * 0.97 return wav1
def tts(input_dict): ''' Flow: 0) Check if audio has embeddings (Not yet) 1) Encode the audio 2) Synthesizer the text with embeddings 3) Vocoder out the fake wav ''' # init output_dict = {"data": {}} # loop the input for audio_name, raw_audio in input_dict["data"].items(): wav_name_no_ext = Path(audio_name).stem saved_path_obj = Path.cwd() / "data/output" # step 1 print("Step 1") raw_audio_np, sample_rate = librosa.load(io.BytesIO(raw_audio)) preprocessed_wav = encoder.preprocess_wav(raw_audio_np, sample_rate) embeddings = encoder.embed_utterance(preprocessed_wav) # step 2 print("Step 2") splitted_text = input_dict["text"].split(".") clean_text_list = [text for text in splitted_text if len(text) > 0] if len(clean_text_list) == 0: raise Exception("Empty text field") sentence_count = len(clean_text_list) embeddings_list = [embeddings] * sentence_count specs = synthesizer.synthesize_spectrograms(clean_text_list, embeddings_list) # step 3 print("Step 3") for index, spec in enumerate(specs): generated_wav = vocoder.infer_waveform(spec) # needed to 1 second for playback capability generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") file_name = "{}_tts_{}.wav".format(wav_name_no_ext, index) file_path = saved_path_obj / file_name sf.write(str(file_path), generated_wav.astype(np.float32), synthesizer.sample_rate, 'PCM_16') output_dict["data"][index] = file_path return output_dict
def generate(): text_to_be_analyzed = request.get_json()["text"] session_client = dialogflow.SessionsClient() session = session_client.session_path(DIALOGFLOW_PROJECT_ID, SESSION_ID) text_input = dialogflow.types.TextInput( text=text_to_be_analyzed, language_code=DIALOGFLOW_LANGUAGE_CODE) query_input = dialogflow.types.QueryInput(text=text_input) try: response = session_client.detect_intent(session=session, query_input=query_input) except InvalidArgument: raise text = response.query_result.fulfillment_text in_fpath = Path("audio.wav") reprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") generated_wav = encoder.preprocess_wav(generated_wav) if os.path.exists("temp.wav"): os.remove("temp.wav") else: print("The file does not exist") if os.path.exists("temp.mp3"): os.remove("temp.mp3") sf.write("temp.wav", generated_wav, synthesizer.sample_rate) AudioSegment.from_wav("temp.wav").export("temp.mp3", format="mp3") encoded_gen_wav_string = "data:audio/mp3;base64," with open("temp.mp3", "rb") as f1: encoded_f1 = base64.b64encode(f1.read()) encoded_gen_wav_string += str(encoded_f1, 'ascii', 'ignore') # encoded_gen_wav_bytes= base64.b64encode(generated_wav) # encoded_gen_wav_string = str(encoded_gen_wav_bytes,'ascii', 'ignore') res = { "data": encoded_gen_wav_string, "rate": synthesizer.sample_rate, "text": text } # sf.write("demo_output.wav", generated_wav.astype(np.float32), synthesizer.sample_rate) return jsonify(res), 200
def synth(): text = "hey welcome to programming hut" #@param {type:"string"} print("Now recording for 10 seconds, say what you will...") ### record record(30) print("Audio recording complete") in_fpath = Path("audio.wav") reprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) print("Synthesizing new audio...") with io.capture_output() as captured: specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") display(Audio(generated_wav, rate=synthesizer.sample_rate))
def convert(self, text, in_fpath, outfn): print(f"converting\ntext:\n {text}\n\n wavfn: {in_fpath}") print(f"outfn: {outfn}") try: preprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) print("Loaded file succesfully") embed = encoder.embed_utterance(preprocessed_wav) print("Created the embedding") # The synthesizer works in batch, so you need to put your data in a list or numpy array texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = self.synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] print("Created the mel spectrogram") ## Generating the waveform print("Synthesizing the waveform:") # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. generated_wav = vocoder.infer_waveform(spec) #TODO: check this necessary generated_wav = np.pad(generated_wav, (0, self.synthesizer.sample_rate), mode="constant") #TODO: Save it on the disk? #fpath = "demo_output_%02d.wav" % num_generated print(generated_wav.dtype) librosa.output.write_wav(outfn, generated_wav.astype(np.float32), self.synthesizer.sample_rate) #num_generated += 1 print("\nSaved output as %s\n\n" % outfn) return True except Exception as e: print("Caught exception: %s" % repr(e)) return False
def vocode(self, spec, breaks): ## Generating the waveform print("Synthesizing the waveform:") # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. wav = vocoder.infer_waveform(spec) # Add breaks b_ends = np.cumsum(np.array(breaks) * Synthesizer.hparams.hop_size) b_starts = np.concatenate(([0], b_ends[:-1])) wavs = [wav[start:end] for start, end, in zip(b_starts, b_ends)] breaks = [np.zeros(int(0.15 * Synthesizer.sample_rate))] * len(breaks) wav = np.concatenate([i for w, b in zip(wavs, breaks) for i in (w, b)]) ## Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. wav = np.pad(wav, (0, self.synthesizer.sample_rate), mode="constant") return wav
def vocalize(n_clicks, celebrity, value): text = value embed = embeddings[celebrity] print("Synthesizing new audio...") # with io.capture_output() as captured: specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") audio = Audio(generated_wav, rate=synthesizer.sample_rate) # display(audio) # return json.dumps({'sample_rate':synthesizer.sample_rate,'audio':generated_wav.tolist()}) write('generated_via_flask_api.mp3', synthesizer.sample_rate, generated_wav, normalized=True) folder_id = '1hOJ9GrsOHLRGe75YwLhS8tfb9zi2bvB9' file1 = drive.CreateFile({ "parents": [{ "kind": "drive#fileLink", "id": folder_id }], 'title': 'audio.mp3', }) file1.SetContentFile('generated_via_flask_api.mp3') file1.Upload() # # Fetch permissions. permissions = file1.GetPermissions() permission = file1.InsertPermission({ 'type': 'anyone', 'value': 'anyone', 'role': 'reader' }) token = permissions[0]['selfLink'].split('/')[-3] return "http://docs.google.com/uc?export=open&id=" + token
def synthesized_voice(text, speaker_name): """ Parameters ---------- text : string text to be said in synthesized voice audio_file : filepath filepath for audio file in wav format Returns ------- generated_wav : numpy.ndarray Numpy padded array of synthesized audio signal """ sample_dir = "src\samples\Original Samples" in_fpath = os.path.join( sample_dir, speaker_name + '.mp3' ) # Audio file to be synthesized, can be changed to audio file of choice, refer synthesizer.py reprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) print("Synthesizing new audio...") with io.capture_output() as captured: specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") print("Synthesized audio generated") ## For saving samples you can call save_audio_local ## save_audio_local(generated_wav, speaker_name, synthesizer.sample_rate) return generated_wav, synthesizer.sample_rate
def clone_voice(self, embed): synthesizer = Synthesizer("synthesizer/saved_models/logs-pretrained/taco_pretrained") vocoder.load_model("vocoder/saved_models/pretrained/pretrained.pt") with open(self.json_text) as text_json: data = json.load(text_json) for x in data: text = x['translation'] # The synthesizer works in batch, so you need to put your data in a list or numpy array texts = [text] embeds = [embed] # If you know what the attention layer alignments are, you can retrieve them here by # passing return_alignments=True specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] ## Generating the waveform print("\nSynthesizing the waveform:") # Synthesizing the waveform is fairly straightforward. Remember that the longer the # spectrogram, the more time-efficient the vocoder. generated_wav = vocoder.infer_waveform(spec) ## Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") # Save it on the disk output_dir = '../temp' try: if not os.path.exists(output_dir): os.makedirs(output_dir) except: pass fpath = "%s/%d.wav" % (output_dir, x['index']) generated_wav *= 32767 / max(0.01, np.max(np.abs(generated_wav))) wavfile.write(fpath, synthesizer.sample_rate, generated_wav.astype(np.int16))
def generate(): """Generates wav file from text and source audio file""" if 'file' not in request.files: return jsonify({'error': 'no audio file'}) text = request.form.get('text') if not text: return jsonify({'error': 'no text provided'}) source_audio_file = request.files['file'].save('/tmp/original.wav') preprocessed_wav = encoder.preprocess_wav(Path('/tmp/original.wav')) original_wav, sampling_rate = librosa.load('/tmp/original.wav') preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) texts = [text] embeds = [embed] specs = synthesizer.synthesize_spectrograms(texts, embeds) spec = specs[0] generated_wav = vocoder.infer_waveform(spec) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") generated_wav = encoder.preprocess_wav(generated_wav) # filename = "/tmp/{}.wav".format(int(time.time())) fio = io.BytesIO() sf.write(fio, generated_wav.astype(np.float32), synthesizer.sample_rate, None, None, 'WAV') fio.seek(0) return send_file(fio, as_attachment=True, attachment_filename='generated.wav', mimetype='audio/wav')
def convert_voice(): if request.method == 'POST': file = request.files['file'] extension = os.path.splitext(file.filename)[1] f_name = str(uuid.uuid4()) + extension file.save(os.path.join(app.config['UPLOAD_FOLDER'], f_name)) file_path = os.path.join(app.config['UPLOAD_FOLDER'], f_name) # @param {type:"string"} text = "This is being said in my own voice. The computer has learned to do an impression of me." in_fpath = Path( "/home/naman/melnetCode/clone_in_5_sec/dataset/data_voice/test.wav" ) reprocessed_wav = encoder.preprocess_wav(in_fpath) original_wav, sampling_rate = librosa.load(in_fpath) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) embed = encoder.embed_utterance(preprocessed_wav) print("Synthesizing new audio...") specs = synthesizer.synthesize_spectrograms([text], [embed]) generated_wav = vocoder.infer_waveform(specs[0]) generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") # display(Audio(generated_wav, rate=synthesizer.sample_rate)) librosa.output.write_wav('output.wav', generated_wav, synthesizer.sample_rate) path_to_file = "output.wav" return send_file(path_to_file, mimetype="audio/wav", as_attachment=True, attachment_filename="output.wav")
mels = synthesizer.synthesize_spectrograms(texts, embeds) # The vocoder synthesizes one waveform at a time, but it's more efficient for long ones. We # can concatenate the mel spectrograms to a single one. mel = np.concatenate(mels, axis=1) # The vocoder can take a callback function to display the generation. More on that later. For # now we'll simply hide it like this: no_action = lambda *args: None print("\tTesting the vocoder...") # For the sake of making this test short, we'll pass a short target length. The target length # is the length of the wav segments that are processed in parallel. E.g. for audio sampled # at 16000 Hertz, a target length of 8000 means that the target audio will be cut in chunks of # 0.5 seconds which will all be generated together. The parameters here are absurdly short, and # that has a detrimental effect on the quality of the audio. The default parameters are # recommended in general. vocoder.infer_waveform(mel, target=200, overlap=50, progress_callback=no_action) print("All test passed! You can now synthesize speech.\n\n") ## Interactive speech generation print("This is a GUI-less example of interface to SV2TTS. The purpose of this script is to " "show how you can interface this project easily with your own. See the source code for " "an explanation of what is happening.\n") print("Interactive generation loop") num_generated = 0 # Get the reference audio filepath #message = "Reference voice: enter an audio filepath of a voice to be cloned(Введите путь до клонируемого файла, например ex.wav) (mp3, " \ # "wav, m4a, flac, ...):\n"
def clone(audio=None, audio_url=None, sentence=""): try: if not 10 <= len(sentence.split(" ")) <= 30: return {"error": "Sentence is invalid! (length must be 10 to 30 words)"} audio_data = audio if audio_url: # Link if "http://" in audio_url or "https://" in audio_url: header = {'User-Agent': 'Mozilla/5.0 (Windows NT x.y; Win64; x64; rv:9.0) Gecko/20100101 Firefox/10.0'} # Check if audio file has less than 5Mb r = requests.head(audio_url, headers=header, allow_redirects=True) size = r.headers.get('content-length', 0) size = int(size) / float(1 << 20) log.info("File size: {:.2f} Mb".format(size)) if size > 10: return {"error": "Input audio file is too large! (max 10Mb)"} r = requests.get(audio_url, headers=header, allow_redirects=True) audio_data = r.content # Base64 elif len(audio_url) > 500: audio_data = base64.b64decode(audio_url) audio_path = generate_uid() + ".audio" with open(audio_path, "wb") as f: f.write(audio_data) # Load the models one by one. log.info("Preparing the encoder, the synthesizer and the vocoder...") encoder.load_model(Path("rtvc/encoder/saved_models/pretrained.pt")) synthesizer = Synthesizer(Path("rtvc/synthesizer/saved_models/logs-pretrained/taco_pretrained")) vocoder.load_model(Path("rtvc/vocoder/saved_models/pretrained/pretrained.pt")) # Computing the embedding original_wav, sampling_rate = librosa.load(audio_path) preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate) log.info("Loaded file successfully") if os.path.exists(audio_path): os.remove(audio_path) embed = encoder.embed_utterance(preprocessed_wav) log.info("Created the embedding") specs = synthesizer.synthesize_spectrograms([sentence], [embed]) spec = np.concatenate(specs, axis=1) # spec = specs[0] log.info("Created the mel spectrogram") # Generating the waveform log.info("Synthesizing the waveform:") generated_wav = vocoder.infer_waveform(spec, progress_callback=lambda *args: None) # Post-generation # There's a bug with sounddevice that makes the audio cut one second earlier, so we # pad it. generated_wav = np.pad(generated_wav, (0, synthesizer.sample_rate), mode="constant") # Save it on the disk fp = tempfile.TemporaryFile() librosa.output.write_wav(fp, generated_wav.astype(np.float32), synthesizer.sample_rate) return {"audio": fp.read()} except Exception as e: log.error(e) traceback.print_exc() return {"error": "Fail"}
def test_config(self): ## Print some environment information (for debugging purposes) print("Running a test of your configuration...\n") try: if not torch.cuda.is_available(): print("Your PyTorch installation is not configured to use CUDA. If you have a GPU ready " "for deep learning, ensure that the drivers are properly installed, and that your " "CUDA version matches your PyTorch installation. CPU-only inference is currently " "not supported.", file=sys.stderr) quit(-1) device_id = torch.cuda.current_device() gpu_properties = torch.cuda.get_device_properties(device_id) print("Found %d GPUs available. Using GPU %d (%s) of compute capability %d.%d with " "%.1fGb total memory.\n" % (torch.cuda.device_count(), device_id, gpu_properties.name, gpu_properties.major, gpu_properties.minor, gpu_properties.total_memory / 1e9)) ## Load the models one by one. print("Preparing the encoder, the synthesizer and the vocoder...") encoder.load_model(self.enc_model_fpath) print("Loaded Encoder") self.synthesizer = Synthesizer(self.syn_model_dir.joinpath("taco_pretrained"), low_mem=self.low_mem) print("Loaded Synth") vocoder.load_model(self.voc_model_fpath) print("Loaded Vocoder") ## Run a test print("Testing your configuration with small inputs.") # Forward an audio waveform of zeroes that lasts 1 second. Notice how we can get the encoder's # sampling rate, which may differ. # If you're unfamiliar with digital audio, know that it is encoded as an array of floats # (or sometimes integers, but mostly floats in this projects) ranging from -1 to 1. # The sampling rate is the number of values (samples) recorded per second, it is set to # 16000 for the encoder. Creating an array of length <sampling_rate> will always correspond # to an audio of 1 second. print("\tTesting the encoder...") encoder.embed_utterance(np.zeros(encoder.sampling_rate)) # Create a dummy embedding. You would normally use the embedding that encoder.embed_utterance # returns, but here we're going to make one ourselves just for the sake of showing that it's # possible. embed = np.random.rand(speaker_embedding_size) # Embeddings are L2-normalized (this isn't important here, but if you want to make your own # embeddings it will be). embed /= np.linalg.norm(embed) # The synthesizer can handle multiple inputs with batching. Let's create another embedding to # illustrate that embeds = [embed, np.zeros(speaker_embedding_size)] texts = ["test 1", "test 2"] print("\tTesting the synthesizer... (loading the model will output a lot of text)") mels = self.synthesizer.synthesize_spectrograms(texts, embeds) # The vocoder synthesizes one waveform at a time, but it's more efficient for long ones. We # can concatenate the mel spectrograms to a single one. mel = np.concatenate(mels, axis=1) # The vocoder can take a callback function to display the generation. More on that later. For # now we'll simply hide it like this: no_action = lambda *args: None print("\tTesting the vocoder...") # For the sake of making this test short, we'll pass a short target length. The target length # is the length of the wav segments that are processed in parallel. E.g. for audio sampled # at 16000 Hertz, a target length of 8000 means that the target audio will be cut in chunks of # 0.5 seconds which will all be generated together. The parameters here are absurdly short, and # that has a detrimental effect on the quality of the audio. The default parameters are # recommended in general. vocoder.infer_waveform(mel, target=200, overlap=50, progress_callback=no_action) print("\tAll test passed!") return("All test passed!") except Exception as e: return("Caught exception: %s" % repr(e))