def versionInfo(): res = [] buildInfoFile = "/wikispeech/wikispeech_server/build_info.txt" if os.path.isfile(buildInfoFile): with open(buildInfoFile) as fp: lines = fp.readlines() fp.close() for l in lines: res.append(l.strip()) else: res.append("Application name: wikispeech") res.append("Build timestamp: n/a") res.append("Built by: user") try: tag = subprocess.check_output(["git", "describe", "--tags"]).decode("utf-8").strip() branch = subprocess.check_output( ["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() log.info(tag) log.info(branch) res.append(("Release: %s on branch %s") % (tag, branch)) except: log.warning("couldn't retrieve git release info: %s" % sys.exc_info()[1]) res.append("Release: unknown") res.append("Started: " + startedAt) return res
def static_proxy_audio(path): audio_tmpdir = config.config.get("Audio settings","audio_tmpdir") audio_file_name = audio_tmpdir+"/"+path log.info("Looking for audio file %s" % audio_file_name) # send_static_file will guess the correct MIME type #return send_from_directory("tmp", path) return send_from_directory(os.getcwd()+"/"+audio_tmpdir, path)
def mapperMapFromMary(trans, lang, voice): log.info("mapperMapFromMary( %s , %s , %s )" % (trans, lang, voice)) if "mapper" in voice: #Bad names.. It should be perhaps "external" and "internal" instead of "from" and "to" to_symbol_set = voice["mapper"]["from"] from_symbol_set = voice["mapper"]["to"] else: log.info("No marytts mapper defined for language %s" % lang) return trans ## hl remove quote_plus 20170613 ## hb "" b j A: rn . % 9 j doesn't work, testing to put quote back in 20170626 ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote_plus(trans)) url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote(trans)) ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, trans) log.debug("MAPPER URL before requests: %s" % url) r = requests.get(url) log.debug("MAPPER URL: "+r.url) response = r.text #log.debug("RESPONSE: %s" % response) try: response_json = json.loads(response) #log.debug("RESPONSE_JSON: %s" % response_json) new_trans = response_json["Result"] except: log.error("unable to map %s, from %s to %s. response was %s" % (trans, from_symbol_set, to_symbol_set, response)) raise #log.debug("NEW TRANS: %s" % new_trans) return new_trans
def __init__(self, cconfig): self.type = "TextprocComponent" self.module = cconfig["module"] self.call = cconfig["call"] if "mapper" in cconfig: try: self.mapper = Mapper(cconfig["mapper"]["from"], cconfig["mapper"]["to"]) except MapperException as e: raise TextprocComponentException(e) if "module" in cconfig and cconfig["module"] == "adapters.marytts_adapter": log.info("Trying to create marytts component: %s" % cconfig) #For testing marytts_adapter #TODO? move to test function in marytts_adapter try: marytts_url = config.config.get("Services", "marytts") payload = { "INPUT_TYPE": "TEXT", "OUTPUT_TYPE": "INTONATION", "LOCALE": "en_US", "INPUT_TEXT": "test" } r = requests.get(marytts_url, params=payload) log.debug("CALLING MARYTTS: %s" % r.url) xml = r.text except Exception as e: raise TextprocComponentException(e)
def static_proxy_audio(path): audio_tmpdir = config.config.get("Audio settings","audio_tmpdir") audio_file_name = audio_tmpdir+"/"+path log.info("Looking for audio file %s" % audio_file_name) # send_static_file will guess the correct MIME type #return send_from_directory("tmp", path) return send_from_directory(os.getcwd()+"/"+audio_tmpdir, path)
def lexserver_proxy(url): redirect_url = "http://localhost:8787/%s%s" % ( (url, "?" + request.query_string.decode("utf-8") if request.query_string else "")) log.info("Lexserver proxy to: %s" % redirect_url) req = requests.get(redirect_url, stream=True) return Response(stream_with_context(req.iter_content()), content_type=req.headers['content-type'])
def loadJsonConfigurationFiles(): global textprocessor_configs, voice_configs textprocessor_configs = [] voice_configs = [] cf_dir = "wikispeech_server" if config.config.has_option("Voice config", "config_files_location"): cf_dir = config.config.get("Voice config", "config_files_location") #The testing config file should always be there #config_files = ["voice_config_for_testing.json"] #OR maybe it shouldn't?? config_files = [] if config.config.has_option("Voice config", "config_files"): #print(config.config.get("Voice config", "config_files")) cfs = config.config.get("Voice config", "config_files").split("\n") for cf in cfs: if cf not in config_files and cf != "": config_files.append(cf) for config_file in config_files: if os.path.isfile(config_file): path = config_file elif os.path.isfile("%s/%s" % (cf_dir, config_file)): path = "%s/%s" % (cf_dir, config_file) else: print("Config file %s or %s not found" % (config_file, "%s/%s" % (cf_dir, config_file))) sys.exit() with open(path) as json_file: log.info("Reading config file: %s" % path) json_like = json_file.read() json_str = remove_comments(json_like) #cf = json.load(json_file) cf = json.loads(json_str) if "textprocessor_configs" in cf: for tconf in cf["textprocessor_configs"]: #Is the tp name already in the list? addTp = True for tc in textprocessor_configs: if tc["name"] == tconf["name"]: log.warning("Textprocessor %s defined more than once: file %s" % (tconf["name"], path)) addTp = False if addTp: tconf["config_file"] = path textprocessor_configs.append(tconf) if "voice_configs" in cf: for vconf in cf["voice_configs"]: #Is the voice name already in the list? addVoice = True for vc in voice_configs: if vc["name"] == vconf["name"]: log.warning("Voice %s defined more than once: file %s" % (vconf["name"], path)) addVoice = False if addVoice: vconf["config_file"] = path voice_configs.append(vconf)
def loadVoice(voice_config): try: log.info("Loading voice %s" % (voice_config["name"])) v = Voice(voice_config) log.info("Done loading voice %s" % (voice_config["name"])) voices.append(v) except VoiceException as e: log.warning("Failed to load voice from %s. Reason:\n%s" % (voice_config,e)) raise
def loadTextprocessor(tp_config): try: log.info("Loading textprocessor %s" % (tp_config["name"])) tp = Textprocessor(tp_config) log.info("Done loading textprocessor %s" % (tp_config["name"])) textprocessors.append(tp) except TextprocessorException as e: log.warning("Failed to load textprocessor from %s. Reason:\n%s" % (tp_config,e)) raise
def lexserver_proxy(url): lexicon_host = config.config.get("Services", "lexicon") redirect_url = "%s/%s%s" % ( (lexicon_host, url, "?" + request.query_string.decode("utf-8") if request.query_string else "")) log.info("Lexserver proxy to: %s" % redirect_url) req = requests.get(redirect_url, stream=True) return Response(stream_with_context(req.iter_content()), content_type=req.headers['content-type'])
def mapperMapToMary(trans, lang, voice): log.debug("mapperMapToMary( %s, %s, %s)" % (trans, lang, voice)) if "mapper" in voice: to_symbol_set = voice["mapper"]["to"] from_symbol_set = voice["mapper"]["from"] log.info("marytts mapper defined for language %s\nFrom: %s\nTo: %s" % (lang, from_symbol_set, to_symbol_set)) else: log.info("No marytts mapper defined for language %s" % lang) return trans ## hl remove quote_plus 20170613 ## hb "" b j A: rn . % 9 j doesn't work, using quote 20170626 ## url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote_plus(trans)) url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote(trans)) ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, trans) log.debug("MAPPER URL before requests: %s" % url) r = requests.get(url) log.debug("MAPPER URL: %s" % r.url) response = r.text log.debug("MAPPER RESPONSE: %s" % response) try: response_json = json.loads(response) except json.JSONDecodeError: log.error("JSONDecodeError:") log.error("RESPONSE: %s" % response) raise new_trans = response_json["Result"] #Special cases for Swedish pre-r allophones that are not handled by the mapper (because mary uses an old version of the phoneme set that desn't distinguish between normal and r-coloured E/{ (always E) and 2/9 (always 9). This should change in mary later on. if lang == "sv": new_trans = re.sub("{ - ",r"E - ", new_trans) new_trans = re.sub("{ ",r"E ", new_trans) new_trans = re.sub("2(:? -) r? ",r"9\1 r", new_trans) log.debug("NEW TRANS: %s" % new_trans) return new_trans
def mapperMapToMary(trans, lang, voice): log.debug("mapperMapToMary( %s, %s, %s)" % (trans, lang, voice)) if "mapper" in voice: to_symbol_set = voice["mapper"]["to"] from_symbol_set = voice["mapper"]["from"] log.info("marytts mapper defined for language %s\nFrom: %s\nTo: %s" % (lang, from_symbol_set, to_symbol_set)) else: log.info("No marytts mapper defined for language %s" % lang) return trans ## hl remove quote_plus 20170613 ## hb "" b j A: rn . % 9 j doesn't work, using quote 20170626 ## url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote_plus(trans)) url = mapper_url + "/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote(trans)) ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, trans) log.debug("MAPPER URL before requests: %s" % url) r = requests.get(url) log.debug("MAPPER URL: %s" % r.url) response = r.text log.debug("MAPPER RESPONSE: %s" % response) try: response_json = json.loads(response) except json.JSONDecodeError: log.error("JSONDecodeError:") log.error("RESPONSE: %s" % response) raise new_trans = response_json["Result"] #Special cases for Swedish pre-r allophones that are not handled by the mapper (because mary uses an old version of the phoneme set that desn't distinguish between normal and r-coloured E/{ (always E) and 2/9 (always 9). This should change in mary later on. if lang == "sv": new_trans = re.sub("{ - ", r"E - ", new_trans) new_trans = re.sub("{ ", r"E ", new_trans) new_trans = re.sub("2(:? -) r? ", r"9\1 r", new_trans) log.debug("NEW TRANS: %s" % new_trans) return new_trans
def synthesise(lang, voice, utterance, presynth=False, hostname=None): log.info("Utterance: %s" % utterance) input = utterance['original_text'] log.info("Text: %s" % input) words = get_orth(utterance) log.info("Words: %s" % words) hashstring=input+'&Lang='+lang+'&Voice='+voice['name'] try: hash_object=hashlib.md5(hashstring.encode('latin-1')) except: try: hash_object=hashlib.md5(hashstring.encode('utf-8')) except: hash_object=hashlib.md5(hashstring.encode()) hashnumber=hash_object.hexdigest() inputfile=open("%s/bin/tts_%s.txt" % (ahotts_dir, hashnumber),"wb") inputfile.write(input.encode('latin-1')+'\n'.encode('latin-1')) inputfile.close() ahotts_command = "cd %s/bin ; ./tts_client -SetDur=y -Speed=%s -IP=%s -Port=%s -InputFile=tts_%s.txt -OutputFile=tts_%s.wav -WordFile=tts_%s.wrd -PhoFile=tts_%s.pho ; mv tts_%s.wav %s/%s/tts_%s.wav ; rm tts_%s.txt" % (ahotts_dir, ahotts_speed, ahotts_server_ip, ahotts_server_port, hashnumber, hashnumber, hashnumber, hashnumber, hashnumber, cwdir, tmpdir, hashnumber, hashnumber) log.info("Ahotts command: %s" % ahotts_command) os.system(ahotts_command) audio_url = "%s%s/%s" % (hostname, "audio",'tts_%s.wav' % hashnumber) words_times_file=open(ahotts_dir+'/tts_'+hashnumber+'.wrd','r') words_times=words_times_file.readlines() words_times_file.close() os.remove(ahotts_dir+'/tts_'+hashnumber+'.wrd') os.remove(ahotts_dir+'/tts_'+hashnumber+'.pho') log.info(str(words)) words_times=list(map(lambda x:x[:-1].split(' ')[1],words_times)) tokens = [] starttime=0.0 lastendtime=0.0 for word_ind in range(len(words)): word=words[word_ind] if word_ind>len(words_times)-1: endtime=lastendtime else: endtime=float(words_times[word_ind])/1000 tokens.append({"orth":word, "starttime":starttime, "endtime":endtime}) starttime=endtime lastendtime=endtime return (audio_url, tokens)
def testVoice(self): log.info("Testing voice %s" % self.name) if self.engine == "marytts": voice_host = config.config.get("Services", "marytts") url = re.sub("process","voices",voice_host) log.debug("Calling url: %s" % url) try: r = requests.get(url) except: msg = "Marytts server not found at url %s" % (url) log.error(msg) raise VoiceException(msg) response = r.text log.debug("Response:\n%s" % response) marytts_voicenames = self.getMaryttsVoicenames(response) if not self.name in marytts_voicenames: msg = "Voice %s not found at url %s" % (self.name, url) log.error(msg) raise VoiceException(msg) else: log.info("Voice found at url %s" % url)
def testVoice(self): log.info("Testing voice %s" % self.name) if self.engine == "marytts": voice_host = config.config.get("Services", "marytts") url = re.sub("process","voices",voice_host) log.debug("Calling url: %s" % url) try: r = requests.get(url) except: msg = "Marytts server not found at url %s" % (url) log.error(msg) raise VoiceException(msg) response = r.text log.debug("Response:\n%s" % response) marytts_voicenames = self.getMaryttsVoicenames(response) if not self.name in marytts_voicenames: msg = "Voice %s not found at url %s" % (self.name, url) log.error(msg) raise VoiceException(msg) else: log.info("Voice found at url %s" % url) elif self.engine == "ahotts": cwdir = os.getcwd() tmpdir = config.config.get("Audio settings","audio_tmpdir") ahotts_dir = config.config.get("Services", "ahotts_dir") ahotts_server_ip = config.config.get("Services", "ahotts_server_ip") ahotts_server_port = config.config.get("Services", "ahotts_server_port") ahotts_command = "cd %s/bin; echo \"Hasierako proba\" > ahotts_test.txt; ./tts_client -IP=%s -Port=%s -InputFile=ahotts_test.txt -OutputFile=ahotts_test.wav ; mv ahotts_test.wav %s/%s/ahotts_test.wav ; rm ahotts_test.txt" % (ahotts_dir, ahotts_server_ip, ahotts_server_port, cwdir, tmpdir) os.system(ahotts_command) try: wavfile=open('%s/%s/ahotts_test.wav'%(cwdir, tmpdir),'r') wavfile.close() os.remove('%s/%s/ahotts_test.wav'%(cwdir, tmpdir)) except: msg = "AhoTTS server not found at IP %s and Port %s" % (ahotts_server_ip,ahotts_server_port) log.error(msg) raise VoiceException(msg)
def testLookup(self): lexicon_name = "wikispeech_testdb:sv" lexicon = Lexicon(lexicon_name) orth = "apa" #expected = [{'entryValidations': [], 'preferred': False, 'lexiconId': 2, 'partOfSpeech': 'NN', 'wordParts': 'apa', 'id': 74078, 'transcriptions': [{'language': 'sv-se', 'id': 79414, 'strn': '"" A: . p a', 'sources': [], 'entryId': 74078}], 'lemma': {'paradigm': 's1a-flicka', 'id': 8764, 'strn': 'apa', 'reading': ''}, 'status': {'id': 74078, 'source': 'nst', 'timestamp': '2017-04-06T09:40:10Z', 'current': True, 'name': 'imported'}, 'language': 'sv-se', 'strn': 'apa', 'morphology': 'SIN|IND|NOM|UTR'}, {'entryValidations': [], 'preferred': False, 'lexiconId': 2, 'partOfSpeech': 'VB', 'wordParts': 'apa', 'id': 74079, 'transcriptions': [{'language': 'sv-se', 'id': 79415, 'strn': '"" A: . p a', 'sources': [], 'entryId': 74079}], 'lemma': {'paradigm': 's1a-flicka', 'id': 8764, 'strn': 'apa', 'reading': ''}, 'status': {'id': 74079, 'source': 'nst', 'timestamp': '2017-04-06T09:40:10Z', 'current': True, 'name': 'imported'}, 'language': 'sv-se', 'strn': 'apa', 'morphology': ''}, {'entryValidations': [], 'preferred': False, 'lexiconId': 2, 'partOfSpeech': 'VB', 'wordParts': 'apa', 'id': 74080, 'transcriptions': [{'language': 'sv-se', 'id': 79416, 'strn': '"" A: . p a', 'sources': [], 'entryId': 74080}], 'lemma': {'paradigm': 's1a-flicka', 'id': 8764, 'strn': 'apa', 'reading': ''}, 'status': {'id': 74080, 'source': 'nst', 'timestamp': '2017-04-06T09:40:10Z', 'current': True, 'name': 'imported'}, 'language': 'sv-se', 'strn': 'apa', 'morphology': 'AKT|INF-IMP'}] #expected = [{'entryValidations': [], 'partOfSpeech': 'NN', 'language': 'sv-se', 'transcriptions': [{'id': 79410, 'entryId': 74074, 'sources': [], 'language': 'sv-se', 'strn': '"" A: . p a'}], 'id': 74074, 'preferred': False, 'morphology': 'SIN|IND|NOM|UTR', 'lemma': {'id': 8764, 'paradigm': 's1a-flicka', 'reading': '', 'strn': 'apa'}, 'wordParts': 'apa', 'strn': 'apa', 'lexiconId': 1, 'status': {'name': 'imported', 'id': 74074, 'current': True, 'source': 'nst', 'timestamp': '2017-05-12T10:55:49Z'}}, {'entryValidations': [], 'partOfSpeech': 'VB', 'language': 'sv-se', 'transcriptions': [{'id': 79411, 'entryId': 74075, 'sources': [], 'language': 'sv-se', 'strn': '"" A: . p a'}], 'id': 74075, 'preferred': False, 'morphology': '', 'lemma': {'id': 8764, 'paradigm': 's1a-flicka', 'reading': '', 'strn': 'apa'}, 'wordParts': 'apa', 'strn': 'apa', 'lexiconId': 1, 'status': {'name': 'imported', 'id': 74075, 'current': True, 'source': 'nst', 'timestamp': '2017-05-12T10:55:49Z'}}, {'entryValidations': [], 'partOfSpeech': 'VB', 'language': 'sv-se', 'transcriptions': [{'id': 79412, 'entryId': 74076, 'sources': [], 'language': 'sv-se', 'strn': '"" A: . p a'}], 'id': 74076, 'preferred': False, 'morphology': 'AKT|INF-IMP', 'lemma': {'id': 8764, 'paradigm': 's1a-flicka', 'reading': '', 'strn': 'apa'}, 'wordParts': 'apa', 'strn': 'apa', 'lexiconId': 1, 'status': {'name': 'imported', 'id': 74076, 'current': True, 'source': 'nst', 'timestamp': '2017-05-12T10:55:49Z'}}] #expected = [{'lexRef': {'DBRef': 'wikispeech_testdb', 'LexName': 'sv'}, 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'reading': '', 'id': 8764}, 'strn': 'apa', 'transcriptions': [{'entryId': 74074, 'strn': '"" A: . p a', 'id': 79410, 'language': 'sv-se', 'sources': []}], 'preferred': False, 'partOfSpeech': 'NN', 'wordParts': 'apa', 'id': 74074, 'morphology': 'SIN|IND|NOM|UTR', 'status': {'source': 'nst', 'id': 74074, 'timestamp': '2017-08-17T11:57:08Z', 'name': 'imported', 'current': True}, 'language': 'sv-se', 'entryValidations': []}, {'lexRef': {'DBRef': 'wikispeech_testdb', 'LexName': 'sv'}, 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'reading': '', 'id': 8764}, 'strn': 'apa', 'transcriptions': [{'entryId': 74075, 'strn': '"" A: . p a', 'id': 79411, 'language': 'sv-se', 'sources': []}], 'preferred': False, 'partOfSpeech': 'VB', 'wordParts': 'apa', 'id': 74075, 'morphology': '', 'status': {'source': 'nst', 'id': 74075, 'timestamp': '2017-08-17T11:57:08Z', 'name': 'imported', 'current': True}, 'language': 'sv-se', 'entryValidations': []}, {'lexRef': {'DBRef': 'wikispeech_testdb', 'LexName': 'sv'}, 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'reading': '', 'id': 8764}, 'strn': 'apa', 'transcriptions': [{'entryId': 74076, 'strn': '"" A: . p a', 'id': 79412, 'language': 'sv-se', 'sources': []}], 'preferred': False, 'partOfSpeech': 'VB', 'wordParts': 'apa', 'id': 74076, 'morphology': 'AKT|INF-IMP', 'status': {'source': 'nst', 'id': 74076, 'timestamp': '2017-08-17T11:57:08Z', 'name': 'imported', 'current': True}, 'language': 'sv-se', 'entryValidations': []}] expected = [{'wordParts': 'apa', 'entryValidations': [], 'id': 74074, 'lexRef': {'LexName': 'sv', 'DBRef': 'wikispeech_testdb'}, 'status': {'timestamp': '2017-08-25T08:43:56Z', 'current': True, 'name': 'imported', 'source': 'nst', 'id': 74074}, 'morphology': 'SIN|IND|NOM|UTR', 'strn': 'apa', 'language': 'sv-se', 'partOfSpeech': 'NN', 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'id': 8764, 'reading': ''}, 'transcriptions': [{'strn': '"" A: . p a', 'entryId': 74074, 'sources': [], 'language': 'sv-se', 'id': 79410}], 'preferred': False}, {'wordParts': 'apa', 'entryValidations': [], 'id': 74075, 'lexRef': {'LexName': 'sv', 'DBRef': 'wikispeech_testdb'}, 'status': {'timestamp': '2017-08-25T08:43:56Z', 'current': True, 'name': 'imported', 'source': 'nst', 'id': 74075}, 'morphology': '', 'strn': 'apa', 'language': 'sv-se', 'partOfSpeech': 'VB', 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'id': 8764, 'reading': ''}, 'transcriptions': [{'strn': '"" A: . p a', 'entryId': 74075, 'sources': [], 'language': 'sv-se', 'id': 79411}], 'preferred': False}, {'wordParts': 'apa', 'entryValidations': [], 'id': 74076, 'lexRef': {'LexName': 'sv', 'DBRef': 'wikispeech_testdb'}, 'status': {'timestamp': '2017-08-25T08:43:56Z', 'current': True, 'name': 'imported', 'source': 'nst', 'id': 74076}, 'morphology': 'AKT|INF-IMP', 'strn': 'apa', 'language': 'sv-se', 'partOfSpeech': 'VB', 'lemma': {'strn': 'apa', 'paradigm': 's1a-flicka', 'id': 8764, 'reading': ''}, 'transcriptions': [{'strn': '"" A: . p a', 'entryId': 74076, 'sources': [], 'language': 'sv-se', 'id': 79412}], 'preferred': False}] result = lexicon.lookup(orth) log.info("RESULT: %s" % result) expected_first_trans = expected[0]['transcriptions'][0]['strn'] result_first_trans = result[0]['transcriptions'][0]['strn'] self.assertEqual(expected_first_trans,result_first_trans)
def testVoice(voice_config): voice_host = config.config.get("Services", "marytts") url = re.sub("process", "voices", voice_host) name = voice_config["name"] log.debug("Calling url: %s" % url) try: r = requests.get(url) except: msg = "Marytts server not found at url %s" % (url) log.error(msg) raise VoiceException(msg) response = r.text log.debug("Response:\n%s" % response) marytts_voicenames = getMaryttsVoicenames(response) log.debug("marytts_voicenames: %s" % marytts_voicenames) if not name in marytts_voicenames: msg = "Voice %s not found at url %s" % (name, url) log.error(msg) raise VoiceException(msg) else: log.info("Voice found at url %s" % url)
def mapperMapFromMary(trans, lang, voice): log.info("mapperMapFromMary( %s , %s , %s )" % (trans, lang, voice)) if "mapper" in voice: #Bad names.. It should be perhaps "external" and "internal" instead of "from" and "to" to_symbol_set = voice["mapper"]["from"] from_symbol_set = voice["mapper"]["to"] else: log.info("No marytts mapper defined for language %s" % lang) return trans ## hl remove quote_plus 20170613 ## hb "" b j A: rn . % 9 j doesn't work, testing to put quote back in 20170626 ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote_plus(trans)) url = mapper_url + "/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, quote(trans)) ##url = mapper_url+"/mapper/map/%s/%s/%s" % (from_symbol_set, to_symbol_set, trans) log.debug("MAPPER URL before requests: %s" % url) r = requests.get(url) log.debug("MAPPER URL: " + r.url) response = r.text #log.debug("RESPONSE: %s" % response) try: response_json = json.loads(response) #log.debug("RESPONSE_JSON: %s" % response_json) new_trans = response_json["Result"] except: log.error("unable to map %s, from %s to %s. response was %s" % (trans, from_symbol_set, to_symbol_set, response)) raise #log.debug("NEW TRANS: %s" % new_trans) return new_trans
def synthesise(lang, voice, utterance, presynth=False): log.info("Utterance: %s" % utterance) words = get_orth(utterance) log.info("Words: %s" % words) ahotts_command = "cd ~/ahotts-code/bin; echo \"%s\" > input.txt; ./tts; mv Output.wav %s/%s/%s" % (" ".join(words), cwdir, tmpdir, wavfile_name) log.info("Ahotts command: %s" % ahotts_command) os.system(ahotts_command) audio_url = "%s/%s" % (prefix,wavfile_name) tokens = [] for word in words: tokens.append({"orth":word, "endtime":0}) return (audio_url, tokens)
def wikispeech(): global hostname from urllib.parse import urlparse parsed_uri = urlparse(request.url) hostname = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) # log.debug("request.url: %s" % hostname) log.info("request: %s" % request) log.info("request.url: %s" % request.url) log.info("hostname: %s" % hostname) if not hostname.endswith("/"): hostname = hostname+"/" if "wikispeech.morf.se" in hostname: ## HL 20171121: force https for wikispeech.morf.se hostname = hostname.replace("http://","https://") log.debug("hostname: %s" % hostname) lang = getParam("lang") input = getParam("input") input_type = getParam("input_type", "text") output_type = getParam("output_type", "json") #For use with synthesis only presynth = getParam("presynth", False) if presynth == "True": presynth = True else: presynth = False textprocessor_name = getParam("textprocessor", "default_textprocessor") voice_name = getParam("voice", "default_voice") log.debug("WIKISPEECH CALL - LANG: %s, INPUT_TYPE: %s, OUTPUT_TYPE: %s, INPUT: %s" % (lang, input_type, output_type, input)) supported_languages = getSupportedLanguages() if not lang or not input: return render_template("usage.html", server=hostname, languages=supported_languages) if lang not in supported_languages: return "Language %s not supported. Supported languages are: %s" % (lang, supported_languages) if input == "TEST_EXAMPLE": return json.dumps(getTestExample(lang)) if input_type in ["text","ssml"]: markup = textproc(lang, textprocessor_name, input, input_type=input_type) if type(markup) == type(""): log.debug("RETURNING MESSAGE: %s" % markup) return markup else: return "input_type %s not supported" % input_type if output_type == "json": result = synthesise(lang, voice_name, markup,"markup",output_type, hostname=hostname, presynth=presynth) if type(result) == type(""): log.debug("RETURNING MESSAGE: %s" % result) return result #TODO #The player being developed at wikimedia depends on the output matching input exactly #phabricator T147547 #Some special characters, like "—" (em-dash) aren't returned properly by the TTS-server. This breaks the token-to-HTML mapping, since it relies on finding the exact same strings in the HTML as the tokens orth values. #Add a test for that here, #And then require adapter components to conform to this? #how, exactly ... msg = checkInputAndOutputTokens(input,result["tokens"]) if msg: result["message"] = msg json_data = json.dumps(result) return Response(json_data, mimetype='application/json') else: return "output_type %s not supported" % output_type
from wikispeech_server.voice import Voice, VoiceException import os.path import datetime import pytz from pytz import timezone import subprocess ################# # # Test opusenc before anything else # ################ log.info("\nOPUSENC\n\nChecking that opusenc is installed on your system..") retval = os.system("opusenc -V") if retval != 0: os.system("opusenc -V") log.error("ERROR: opusenc was not found. You should probably run something like\nsudo apt install opus-tools\n") sys.exit(1) else: log.info("opusenc found.\n\nEND OPUSENC\n") ############### # # Load textprocessors and voices # ###############
def wikispeech(): global hostname from urllib.parse import urlparse parsed_uri = urlparse(request.url) hostname = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) # log.debug("request.url: %s" % hostname) log.debug("request: %s" % request) log.info("request.url: %s" % request.url) log.debug("hostname: %s" % hostname) if not hostname.endswith("/"): hostname = hostname+"/" if "wikispeech.morf.se" in hostname: ## HL 20171121: force https for wikispeech.morf.se hostname = hostname.replace("http://","https://") log.debug("hostname: %s" % hostname) lang = getParam("lang") input = getParam("input") input_type = getParam("input_type", "text") output_type = getParam("output_type", "json") textprocessor_name = getParam("textprocessor", "default_textprocessor") voice_name = getParam("voice", "default_voice") log.debug("WIKISPEECH CALL - LANG: %s, INPUT_TYPE: %s, OUTPUT_TYPE: %s, INPUT: %s" % (lang, input_type, output_type, input)) supported_languages = getSupportedLanguages() if not lang or not input: return render_template("usage.html", server=hostname, languages=supported_languages) if lang not in supported_languages: return "Language %s not supported. Supported languages are: %s" % (lang, supported_languages) if input == "TEST_EXAMPLE": return json.dumps(getTestExample(lang)) if input_type in ["text","ssml"]: markup = textproc(lang, textprocessor_name, input, input_type=input_type) if type(markup) == type(""): log.debug("RETURNING MESSAGE: %s" % markup) return markup else: return "input_type %s not supported" % input_type if output_type in ["json", "html"]: result = synthesise(lang, voice_name, markup,"markup",output_type, hostname=hostname) if type(result) == type(""): log.debug("RETURNING MESSAGE: %s" % result) return result #TODO #The player being developed at wikimedia depends on the output matching input exactly #phabricator T147547 #Some special characters, like "—" (em-dash) aren't returned properly by the TTS-server. This breaks the token-to-HTML mapping, since it relies on finding the exact same strings in the HTML as the tokens orth values. #Add a test for that here, #And then require adapter components to conform to this? #how, exactly ... msg = checkInputAndOutputTokens(input,result["tokens"]) if msg: result["message"] = msg json_data = json.dumps(result) if output_type == "json": return Response(json_data, mimetype='application/json') elif output_type == "html": newtokens = [] starttime = 0 for token in result["tokens"]: token["starttime"] = starttime token["dur"] = token["endtime"]-starttime newtokens.append(token) starttime = token["endtime"] return render_template("output.html", audio_data=result["audio_data"], tokens=newtokens) else: return "output_type %s not supported" % output_type
return "{name:%s, lang:%s}" % (self.name, self.lang) def __str__(self): return {"name":self.name, "lang":self.lang} #return "{name:%s, lang:%s}" % (self.name, self.lang) if __name__ == "__main__": log.log_level = "debug" #debug, info, warning, error voice_config = { "lang":"sv", "name":"stts_sv_nst-hsmm", "engine":"marytts", "adapter":"adapters.marytts_adapter", "mapper": { "from":"sv-se_ws-sampa", "to":"sv-se_sampa_mary" } } try: v = Voice(voice_config) log.info("Created voice %s from %s" % (v, voice_config)) except VoiceException as e: log.error("Failed to create voice for %s\nException message was:\n%s" % (voice_config, e))
def lexserver_proxy(url): lexicon_host = config.config.get("Services","lexicon") redirect_url = "%s/%s%s" % ((lexicon_host, url, "?" + request.query_string.decode("utf-8") if request.query_string else "")) log.info("Lexserver proxy to: %s" % redirect_url) req = requests.get(redirect_url, stream = True) return Response(stream_with_context(req.iter_content()), content_type = req.headers['content-type'])
def static_proxy_js(): filename = "wikispeech_simple_player.js" root_dir = os.getcwd() log.info("Looking for static file %s/%s" % (root_dir, filename)) return send_from_directory(root_dir, filename)
import sys import json if __name__ == "__main__": import os sys.path.append(os.path.dirname(os.path.abspath(__file__))+"/..") #print(sys.path) import wikispeech_server.wikispeech as ws import wikispeech_server.log as log log.info("RUNNING VOICE CONFIG TESTS") #HB 171127 host = "http://localhost:10000/" host = "/" test_client = ws.app.test_client() r = test_client.options(host) api_info = json.loads(r.data.decode('utf-8')) supported_languages = api_info["GET"]["parameters"]["lang"]["allowed"] def test_default_settings(): #1) Test default settings for supported languages for lang in supported_languages: log.debug("START: %s" % lang) # GET: curl "http://localhost:10000/?lang=en&input=test." r = test_client.get("%s?lang=%s&input=test." % (host,lang)) log.debug(r.data.decode('utf-8'))
l.append("config_file: %s" % (self.config["config_file"])) return "{%s}" % ", ".join(l) def __str__(self): return self.__repr__() if __name__ == "__main__": log.log_level = "debug" #debug, info, warning, error voice_config = { "lang":"sv", "name":"stts_sv_nst-hsmm", "engine":"marytts", "adapter":"adapters.marytts_adapter", "mapper": { "from":"sv-se_ws-sampa", "to":"sv-se_sampa_mary" } } try: v = Voice(voice_config) log.info("Created voice %s from %s" % (v, voice_config)) except VoiceException as e: log.error("Failed to create voice for %s\nException message was:\n%s" % (voice_config, e))
def static_test(): log.info("Looking for static file %s" % "test.html") hostname = "http://localhost:10000/wikispeech" return render_template("test.html", server=hostname)
def static_proxy_js2(): filename = "wikispeech_simple_player.js" root_dir = os.getcwd() log.info("Looking for static file %s/%s" % (root_dir, filename)) return send_from_directory(root_dir, filename)
use_json_conf = False if config.config.has_option("Voice config", "config_files_location"): use_json_conf = True #if use_json_conf, the json files defined in *.conf will be loaded, replacing voice_config.py ### ################# # # Test opusenc before anything else # ################ log.info("\nOPUSENC\n\nChecking that opusenc is installed on your system..") retval = os.system("opusenc -V") if retval != 0: os.system("opusenc -V") log.error("ERROR: opusenc was not found. You should probably run something like\nsudo apt install opus-tools\n") sys.exit(1) else: log.info("opusenc found.\n\nEND OPUSENC\n") ################ # # Flask app # ###############
def static_test(): log.info("Looking for static file %s" % "test.html") #HB this is wrong (won't work on morf) #hostname = "http://localhost:10000" hostname = request.url_root return render_template("test.html", server=hostname)
def test_all_settings(): #2) Test all settings for supported languages. #foreach language #foreach textproc_config #foreach voice #test for lang in supported_languages: log.debug("START: %s" % lang) r = test_client.get("%stextprocessing/textprocessors/%s" % (host, lang)) #log.debug(r.url) textproc_configs = json.loads(r.data.decode('utf-8')) for textproc_config in textproc_configs: log.debug(textproc_config) tp_name = textproc_config["name"] if "skip_test" in textproc_config and textproc_config[ "skip_test"] == True: log.info("SKIPPING synthesis test with %s" % (tp_name)) continue log.debug("START %s" % tp_name) url = "%stextprocessing/?input=test.&lang=%s&textprocessor=%s" % ( host, lang, tp_name) log.debug("url: %s" % url) r = test_client.get(url) tmp = json.loads(r.data.decode('utf-8')) #tmp = r.data.decode('utf-8') log.debug("TP OUTPUT: %s" % tmp) url = "%ssynthesis/voices/%s" % (host, lang) log.debug("trying url: %s" % url) r = test_client.get(url) voices = json.loads(r.data.decode('utf-8')) for voice in voices: log.debug(voice) voice_name = voice["name"] log.debug("START %s" % voice_name) if "skip_test" in voice and voice["skip_test"] == True: log.info("SKIPPING synthesis test with %s (%s)" % (voice_name, voice["engine"])) continue payload = { "input": json.dumps(tmp), "lang": lang, "voice": voice_name, "output_type": "test" } url = "%ssynthesis/" % (host) r = test_client.post(url, data=payload) if r.status_code != 200: log.fatal("test call to URL %s failed, server error: %s" % (url, r)) res = json.loads(r.data.decode('utf-8')) log.debug(res) log.debug("DONE %s" % voice_name) log.debug("DONE %s" % tp_name) log.debug("DONE: %s" % lang)
def static_proxy_workflow(path): filename = "workflow_demo/"+path root_dir = os.getcwd() log.info("Looking for static file %s/%s" % (root_dir, filename)) return send_from_directory(root_dir, filename)
def static_proxy_audio(path): audio_file_name = "tmp/" + path log.info("Looking for audio file %s" % audio_file_name) # send_static_file will guess the correct MIME type return send_from_directory("tmp", path)
import sys import json import wikispeech_server.wikispeech as ws import wikispeech_server.log as log log.info("RUNNING VOICE CONFIG TESTS") host = "http://localhost:10000/wikispeech/" test_client = ws.app.test_client() r = test_client.options(host) api_info = json.loads(r.data.decode('utf-8')) supported_languages = api_info["GET"]["parameters"]["lang"]["allowed"] def test_default_settings(): #1) Test default settings for supported languages for lang in supported_languages: log.debug("START: %s" % lang) # GET: curl "http://localhost:10000/wikispeech/?lang=en&input=test." r = test_client.get("%s?lang=%s&input=test." % (host, lang)) log.debug(r.data.decode('utf-8')) log.debug("DONE: %s" % lang) def test_all_settings(): #2) Test all settings for supported languages. #foreach language #foreach textproc_config
def static_test(): log.info("Looking for static file %s" % "test.html") #HB this is wrong (won't work on morf) #hostname = "http://localhost:10000" hostname = request.url_root return render_template("test.html", server=hostname)
if __name__ == "__main__": log.log_level = "debug" #debug, info, warning, error tp_config = { "name":"wikitextproc_sv", "lang":"sv", "components":[ { "module":"adapters.marytts_adapter", "call":"marytts_preproc", "mapper": { "from":"sv-se_ws-sampa", "to":"sv-se_sampa_mary" }, }, { "module":"adapters.lexicon_client", "call":"lexLookup", "lexicon":"wikispeech_testdb:sv" } ] } try: tp = Textprocessor(tp_config) log.info("Created textprocessor %s from %s" % (tp, tp_config)) except TextprocessorException as e: log.error("Failed to create textprocessor for %s\nException message was:\n%s" % (tp_config, e))
def static_proxy_workflow(path): filename = "workflow_demo/" + path root_dir = os.getcwd() log.info("Looking for static file %s/%s" % (root_dir, filename)) return send_from_directory(root_dir, filename)