def download_sounds(self):
if self.settings["soundbackup"] is True:
import py7zr
name = self.settings["Name"]
if not os.path.isfile(self.file_system.path+"/nonesounds.7z"):
free_mb = psutil.disk_usage('/')[2] / 1024 / 1024
if free_mb <= 1500:
self.settings["soundbackup"] = False
self.log.info("no space: Sound Download not possible")
return
else:
self.log.info("downloading soundbackup")
wget.download('http://downloads.tuxfamily.org/pdsounds/pdsounds_march2009.7z', self.file_system.path+"/nonesounds.7z")
#onlyfiles = next(os.walk(self.settings["file_path"]+name+"/not-wake-word/noises"))[2]
#if len(onlyfiles) <= 30:
if not os.path.isdir(self.file_system.path+"/noises"):
os.makedirs(self.file_system.path+"/noises")
if not os.path.isdir(self.file_system.path+"/noises/mp3"):
self.log.info("unzip soundbackup")
py7zr.unpack_7zarchive(self.file_system.path+"/nonesounds.7z", self.file_system.path+"/noises")
self.log.info("download sucess, start convert")
onlyfiles = next(os.walk(self.file_system.path+"/noises/noises"))[2]
if len(onlyfiles) <= 30:
folder = self.file_system.path+"/noises/mp3/"
fileformat = '.mp3'
i = 1
while i <= 2:
for root, dirs, files in os.walk(folder):
for f in files:
filename = os.path.join(root, f)
if filename.endswith(fileformat):
self.log.info("Filename: "+filename)
soundfile = filename.replace(fileformat, '').replace(folder, '')
if not os.path.isdir(self.file_system.path+"/noises/noises"):
os.makedirs(self.file_system.path+"/noises/noises")
subprocess.call(["ffmpeg -i "+filename+" -acodec pcm_s16le -ar 16000 -ac 1 -f wav "+
self.file_system.path+"/noises/noises/"+soundfile+".wav"],
preexec_fn=os.setsid, shell=True)
self.log.info("extratct: "+filename)
folder = self.file_system.path+"/noises/otherformats/"
fileformat = '.flac'
i = i + 1
self.speak_dialog("download.success")
if not os.path.isdir(self.settings["file_path"]+name+"/not-wake-word"):
os.makedirs(self.settings["file_path"]+"/"+name+"/not-wake-word")
if not os.path.isdir(self.settings["file_path"]+name+"/not-wake-word/noises"):
self.log.info("Make Filelink")
os.symlink(self.file_system.path+"/noises/noises/", self.settings["file_path"]+name+"/not-wake-word/noises")
else:
return True
def intent_match(self, saved_utt, skill):
vocs = []
best_match = [0.5]
for path in self.paths_gen(skill):
self.log.info("search on path "+path)
for root, dirs, files in os.walk(path):
for f in files:
### for intent files
filename = os.path.join(root, f)
if filename.endswith(".intent"):
i = filename.replace(".intent", "")
#for l in self.read_intent_lines(i, filename):
#self.log.info("test2"+str(self._lines_from_path(filename)))
try:
match, confidence = match_one(saved_utt, self._lines_from_path(filename))
self.log.info("match "+str(match)+ " confidence "+ str(confidence))
if confidence > best_match[0]:
self.log.info("better match "+str(match)+ " confidence "+ str(confidence))
best_match = [confidence, saved_utt, filename, match, skill]
self.log.info("save"+str(best_match))
except:
self.log.info("fail load intent: "+filename)
else:
self.log.info(str(best_match[0])+" and")
if len(best_match) > 1:
match, saved_utt = self.var_found(best_match[1], best_match[3])
self.ask_save_intent_dialog(best_match[1], best_match[2], best_match[3], best_match[4])
self.acknowledge()
match = self.filter_sentence(match)
self.bus.emit(Message('recognizer_loop:utterance',
{"utterances": [match],
"lang": self.lang,
"session": skill.name}))
else:
self.speak_dialog("no.old.inquiry")
def work_on_dialog(self, message):
skill = self.get_skill(message.data.get("skill", None))
for path in self.paths_gen(skill):
self.speak_dialog("read.for")
for root, dirs, files in os.walk(path):
for f in files:
### for dialog files
filename = os.path.join(root, f)
if filename.endswith(".dialog") and self.lang in filename:
e = open(filename, 'r')
self.log.info("filename "+filename)
line = e.readline()
self.log.info(str(line))
confirm = self.ask_yesno("line", data={"line":self.dialog_renderer.render(line)})
if confirm == "yes":
match = line[:10]
saved_utt = self.get_response("found.output", data={"match": match})
self.log.info("utt "+str(saved_utt))
while saved_utt is None:
saved_utt = self.get_response("found.output", data={"match": match})
else:
self.log.info("bevor match "+str(saved_utt))
match, saved_utt = self.var_found(saved_utt, match)
self.log.info("after match "+str(saved_utt))
self.ask_save_intent_dialog(saved_utt, filename, match, skill)
self.log.info("after save "+str(saved_utt))
a = self.ask_yesno("continue")
if a == "yes":
continue
else:
self.acknowledge()
return True
else:
continue
def dialog_match(self, saved_dialog, skill):
for path in self.paths_gen(skill):
for root, dirs, files in os.walk(path):
for f in files:
### for intent files
filename = os.path.join(root, f)
if self.lang in filename: ## reduce selection to reduce load list size
if filename.endswith(".dialog"):
try:
match, confidence = match_one(
self.saved_answer,
self._lines_from_path(filename))
self.log.info("match " + str(match) +
" confidence " + str(confidence))
if confidence > 0.8:
saved_utt = self.get_response(
"found.output", data={"match": match})
if saved_utt is not None:
match, saved_utt = self.var_found(
saved_utt, match)
self.ask_save_intent_dialog(
saved_utt, filename, match, skill)
self.acknowledge()
else:
self.speak_dialog("cancel")
except:
self.log.info("fail load match: " + filename)
def get_parent_items():
menu = []
apps_dir = join(settings.PROJECT_PATH, 'apps')
for dirname in next(os.walk(apps_dir))[1]:
if dirname not in ['accounts'] and exists(join(apps_dir, '{}/admin.py'.format(dirname))):
menu.append(ParentItem(_('{}'.format(dirname)), app='{}'.format(dirname), icon='fa fa-heart'))
return menu
def getObjectsByCollectionId(self, collection_id, logger, log_marker):
objects = []
for root_dir_path, _, file_names in os.walk(os.path.join(self.__data_dir_path, "record")):
for file_name in file_names:
file_path = os.path.join(root_dir_path, file_name)
if not file_path.endswith(".xml"):
os.rename(file_path, file_path + ".xml")
file_path = file_path + ".xml"
objects.append(self.__map_oai_pmh_record(collection_id=collection_id, file_path=file_path))
return ImmutableList.copyOf(objects)
def improve_intent(self, message):
name = self.config_core.get('listener', {}).get('wake_word').replace(' ', '-')
i = 1
if os.path.isdir(self.settings["sell_path"]):
onlyfiles = next(os.walk(self.settings["sell_path"]))[2]
if len(onlyfiles) <= self.settings["improve"]:
selling = len(onlyfiles)
else:
selling = self.settings["improve"]
self.speak_dialog('improve', data={'name': name, "selling": selling})
self.log.info("search wake word in: "+self.settings["sell_path"])
wait_while_speaking()
for root, dirs, files in os.walk(self.settings["sell_path"]):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.wav'):
if i <= selling:
self.log.info("play file")
play_wav(filename)
wait_while_speaking()
time.sleep(3)
sell = self.ask_yesno("ask.sell", data={'i': i})
wait_while_speaking()
i = i+1
path = None
if sell == "yes":
path = self.settings["file_path"]+name+"/wake-word/"+self.lang[:2]+"-short/"
elif sell == "no":
path = self.settings["file_path"]+name+"/not-wake-word/"+self.lang[:2]+"-short-not/"
if not path is None:
if not os.path.isdir(path):
os.makedirs(path)
file = path+name+"-"+self.lang[:2]+"-"+str(uuid.uuid1())+".wav"
shutil.move(filename, file)
self.log.info("move File: "+file)
else:
os.remove(filename)
else:
self.speak_dialog('improve.no.file', data={'name': name})
else:
self.speak_dialog('improve.no.file', data={'name': name})
def download_sounds(self):
if self.settings["soundbackup"] is True:
import py7zr
name = self.settings["name"]
if not os.path.isfile(self.file_system.path + "/nonesounds.7z"):
self.log.info("downloading soundbackup")
wget.download(
'http://downloads.tuxfamily.org/pdsounds/pdsounds_march2009.7z',
self.file_system.path + "/nonesounds.7z")
if not os.path.isdir(self.settings["file_path"] + "/" + name +
"/not-wake-word/noises"):
if not os.path.isdir(self.file_system.path + "/noises"):
os.makedirs(self.file_system.path + "/noises")
if not os.path.isdir(self.file_system.path + "/noises/mp3"):
self.log.info("unzip soundbackup")
py7zr.unpack_7zarchive(
self.file_system.path + "/nonesounds.7z",
self.file_system.path + "/noises")
self.log.info("download sucess, start convert")
if not os.path.isdir(self.file_system.path + "/noises/noises"):
for root, dirs, files in os.walk(self.file_system.path +
"/noises/mp3/"):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.mp3'):
self.log.info("Filename: " + filename)
if not os.path.isdir(self.file_system.path +
"/noises/noises"):
os.makedirs(self.file_system.path +
"/noises/noises")
self.soundbackup_convert = subprocess.Popen(
[
"ffmpeg -i " + filename +
" -acodec pcm_s16le -ar 16000 -ac 1 -f wav "
+ self.file_system.path +
"/noises/noises/noises-" +
str(uuid.uuid1()) + ".wav"
],
preexec_fn=os.setsid,
shell=True)
self.log.info("extratct: " + filename)
self.log.info("Make Filelink")
if not os.path.isdir(self.settings["file_path"] + "/" + name +
"/not-wake-word"):
os.makedirs(self.settings["file_path"] + "/" + name +
"/not-wake-word")
os.symlink(
self.file_system.path + "/noises/noises/",
self.settings["file_path"] + "/" + name +
"/not-wake-word/noises")
else:
return True
def _parse_records(self):
objects_by_id = OrderedDict()
for root_dir_path, _, file_names in os.walk(os.path.join(self._data_dir_path, 'extracted', self._institution_id, 'record')):
for file_name in file_names:
file_path = os.path.join(root_dir_path, file_name)
if not file_path.endswith('.xml'):
os.rename(file_path, file_path + '.xml')
file_path = file_path + '.xml'
record_etree = ElementTree.parse(file_path)
result = self._parse_record(record_etree)
if result is None:
continue
object_id, object_ = result
objects_by_id[object_id] = object_
return objects_by_id
def main(argv):
directory = argv[1]
for dirname, dirnames, filenames in os.walk(directory):
os.chdir(dirname)
for filename in filenames:
if filename.endswith('.jar'):
print filename
jar = zipfile.ZipFile(filename, 'r')
for info in jar.infolist():
if info.filename == 'META-INF/MANIFEST.MF':
data = jar.read(info.filename)
lines = data.splitlines()
for line in lines:
imp_version = line.find('Implementation-Version')
if imp_version != -1:
print line + '\n'
def scan_intent(self, skill):
location = os.path.dirname(os.path.realpath(__file__))
location = location + '/../' # get skill parent directory path
self.log.info("old uttr: "+str(self.saved_utt))
self.speak_dialog("please.wait")
for name in os.listdir(location):
path = os.path.join(location, name)
file = path.replace(location, '')
if str(skill) in str(file):
if os.path.isdir(path):
#self.log.info('find skill folder: '+path)
for root, dirs, files in os.walk(str(path)):
for f in files:
#self.log.info('search file')
filename = os.path.join(root, f)
if filename.endswith('.intent'):
self.work_on_intent(filename, skill, location)
def getSubDirs(directory, word):
print("In GetSubDirs method")
foundDirs = list()
if os.path.isdir(directory):
for root, dirs, files in os.walk(directory):
"""for name in dirs:
print(name)"""
print("Root: " + root)
for d in dirs:
print(d)
if d == word:
print("Hurra!!")
p = os.path.join(root, d)
print(p)
level = getDirLevel(p)
print(level)
tup = (level, p)
foundDirs.append(tup)
tup = (99, os.path.join(directory, word))
foundDirs.append(tup)
return foundDirs
def prepaire_repo(self, name):
name = name.replace('-', '').replace(' ', '')
##### Model Files
self.log.info("make repo ready vor upload")
presiceversion = linecache.getline(
self.file_system.path +
"/precise/mycroft_precise.egg-info/PKG-INFO",
3).replace('Version: ', '')[:5]
modelzip = self.precisefolder + "/" + name + "/models/" + name + "-" + self.lang[:3] + presiceversion + "-" + time.strftime(
"%Y%m%d") + "-" + self.settings.get('localgit') + ".tar.gz"
if not os.path.isdir(self.precisefolder + "/" + name + "/models/"):
os.makedirs(self.precisefolder + "/" + name + "/models/")
tar = tarfile.open(modelzip, "w:gz")
for nams in [
self.file_system.path + "/" + name + ".pb",
self.file_system.path + "/" + name + ".pbtxt",
self.file_system.path + "/" + name + ".pb.params"
]:
tar.add(nams)
#### calculating info
traininfo = linecache.getline(
self.file_system.path + "/" + name + ".logs/output.txt", 2)
#### generate Readme.md
readmefile = self.precisefolder + "/" + name + "/models/README.md"
file = open(readmefile, "a")
if not os.path.isfile(readmefile):
file.write("# " + name + "\n")
file.write("\n### " + name + "-" + self.lang[:3] +
time.strftime("%Y%m%d") + "\n")
file.write(presiceversion + " " + traininfo[:1] +
". Use Public Domain Sounds Backup:" +
str(self.settings["soundbackup"]) +
", automatically generated by wakeword trainer skill \n")
file.close()
###### licenses
licensefile = self.precisefolder + "/licenses/license-" + time.strftime(
"%Y%m%d") + "-" + self.settings.get('localgit') + ".txt"
fobj_in = open(self.precisefolder + "/licenses/license-template.txt",
"r")
fobj_out = open(licensefile, "w")
for line in fobj_in:
line = line.replace(
"I, [author name]", "I, " + self.settings.get('localgit') +
' (https://github.com/' + self.settings.get('localgit') + ')')
line = line.replace(
"/file/name/1",
"automatically generated by gras64 wakeword trainer skill"
).replace("/file/name/2", "")
fobj_out.write(str(line))
modelzipfile = modelzip.replace(self.precisefolder + "/", "")
fobj_out.write(modelzipfile + "\n")
for root, dirs, files in os.walk(self.precisefolder + "/" + name +
"/" + self.lang[:2]):
for f in files:
filename = os.path.join(root, f)
self.log.info("filename: " + filename)
if filename.endswith('.wav'):
filename = filename.replace(self.precisefolder, "")
fobj_out.write(filename + "\n")
fobj_in.close()
fobj_out.close()
##### Copy all wav file
if not self.settings["onlyPrecise"]:
source = self.settings[
"file_path"] + name + "/wake-word/" + self.lang[:2] + "-short/"
destination = self.precisefolder + "/" + name + "/" + self.lang + "/"
if not os.path.isdir(destination):
os.makedirs(destination)
fobj_out = open(licensefile, "a")
for filename in os.listdir(source):
if filename.endswith('.wav'):
shutil.copy(source + filename, destination)
fobj_out.write("/" + name + "/" + self.lang[:2] + "/" +
filename + "\n")
source = self.settings[
"file_path"] + name + "/test/wake-word/" + self.lang[:2] + "-short/"
for filename in os.listdir(source):
if filename.endswith('.wav'):
shutil.copy(source + filename, destination)
fobj_out.write("/" + name + "/" + self.lang[:2] + "/" +
filename + "\n")
fobj_out.close()
def record_file_mover(self, yespath, nopath, source):
#### wake words with 4 test files
i = 1
if os.path.isdir(self.settings["file_path"] + self.new_name + "/test" +
yespath):
onlyfiles = next(
os.walk(self.settings["file_path"] + self.new_name + "/test" +
yespath))
i = 4 - len(onlyfiles)
else:
i = 1
os.makedirs(self.settings["file_path"] + self.new_name + "/test" +
yespath)
for root, dirs, files in os.walk(source + yespath):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.wav'):
if i <= 4:
shutil.move(
filename,
self.settings["file_path"] + self.new_name +
"/test" + yespath + self.new_name + "-" +
self.lang[:2] + "-" + str(uuid.uuid1()) + ".wav")
self.log.info("move file: " + filename)
i = i + 1
else:
if not os.path.isdir(self.settings["file_path"] +
self.new_name + yespath):
os.makedirs(self.settings["file_path"] +
self.new_name + yespath)
shutil.move(
filename, self.settings["file_path"] +
self.new_name + yespath + self.new_name + "-" +
self.lang[:2] + "-" + str(uuid.uuid1()) + ".wav")
self.log.info("move file: " + filename)
i = i + 1
#### not wakeword with 4 test files
i = 1
if os.path.isdir(self.settings["file_path"] + self.new_name + "/test" +
nopath):
onlyfiles = next(
os.walk(self.settings["file_path"] + self.new_name + "/test" +
nopath))
i = 4 - len(onlyfiles)
else:
i = 1
os.makedirs(self.settings["file_path"] + self.new_name + "/test" +
nopath)
for root, dirs, files in os.walk(source + nopath):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.wav'):
if i <= 4:
if not os.path.isdir(self.settings["file_path"] +
self.new_name + "/test" + nopath):
os.makedirs(self.settings["file_path"] +
self.new_name + "/test/" + nopath)
shutil.move(
filename,
self.settings["file_path"] + self.new_name +
"/test" + nopath + "not" + self.new_name + "-" +
self.lang[:2] + "-" + str(uuid.uuid1()) + ".wav")
self.log.info("move file: " + filename)
i = i + 1
else:
if not os.path.isdir(self.settings["file_path"] +
self.new_name + nopath):
os.makedirs(self.settings["file_path"] +
self.new_name + nopath)
shutil.move(
filename,
self.settings["file_path"] + self.new_name +
nopath + "not" + self.new_name + "-" +
self.lang[:2] + "-" + str(uuid.uuid1()) + ".wav")
self.log.info("move file: " + filename)
i = i + 1
def train_wake_word_intent(self, message):
if message.data.get("name"):
name = message.data.get("name")
name = name.replace(' ', '-')
if os.path.isdir(self.settings["file_path"]+name):
if self.ask_yesno("model.available",
data={"name": name}) == "yes":
if os.path.isdir(self.settings["file_path"]+name):
rmtree(self.settings["file_path"]+name)
if os.path.isdir("/tmp/mycroft_wakeword/"):
rmtree("/tmp/mycroft_wakeword/")
self.speak_dialog("word.wake",
data={"name": name})
wait_while_speaking()
# Throw away any previous recording
time.sleep(4)
i = 1
source = "/tmp/mycroft_wakeword/"+name
nopath = "/not-wake-word/"+ self.lang[:2] + "-short/"
yespath = "/wake-word/"+ self.lang[:2] + "-short/"
### Record test files to tmp
while i <= 24:
if i < 12:
play_wav(self.piep)
path = source+yespath
soundfile = name+ "-" + self.lang[:2] +"-"+str(uuid.uuid1())+".wav"
elif i == 12:
stop = ""
self.speak_dialog("none.wake.word", stop, expect_response=True)
if stop == "stop":
rmtree(source)
return
play_wav(self.piep)
path = source+nopath
soundfile = "not"+name+"-"+ self.lang[:2] +"-"+str(uuid.uuid1())+".wav"
else:
play_wav(self.piep)
path = source+nopath
soundfile = "not"+name+"-"+ self.lang[:2] +"-"+str(uuid.uuid1())+".wav"
time.sleep(2)
wait_while_speaking()
i = i+1
self.start_recording(name,i,path,soundfile)
else:
#### Save wakewords in data folder
if self.ask_yesno("is.all.ok") == "no":
rmtree(source)
return
wait_while_speaking()
#### wake words with 4 test files
i = 1
if os.path.isdir(self.settings["file_path"]+name+"/test"+yespath):
onlyfiles = next(os.walk(self.settings["file_path"]+name+"/test"+yespath))
i = 4 - len(onlyfiles)
else:
i = 1
os.makedirs(self.settings["file_path"]+name+"/test"+yespath)
for root, dirs, files in os.walk(source+yespath):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.wav'):
if i <= 4:
shutil.move(filename, self.settings["file_path"]+name+"/test"+yespath+
name+ "-" + self.lang[:2] +"-"+str(uuid.uuid1())+".wav")
self.log.info("move file: "+filename)
i = i + 1
else:
if not os.path.isdir(self.settings["file_path"]+name+yespath):
os.makedirs(self.settings["file_path"]+name+yespath)
shutil.move(filename, self.settings["file_path"]+name+yespath+
name+ "-" + self.lang[:2] +"-"+str(uuid.uuid1())+".wav")
self.log.info("move file: "+filename)
i = i + 1
#### not wakeword with 4 test files
i = 1
if os.path.isdir(self.settings["file_path"]+name+"/test"+nopath):
onlyfiles = next(os.walk(self.settings["file_path"]+name+"/test"+nopath))
i = 4 - len(onlyfiles)
else:
i = 1
os.makedirs(self.settings["file_path"]+name+"/test"+nopath)
for root, dirs, files in os.walk(source+nopath):
for f in files:
filename = os.path.join(root, f)
if filename.endswith('.wav'):
if i <= 4:
if not os.path.isdir(self.settings["file_path"]+name+"/test"+nopath):
os.makedirs(self.settings["file_path"]+name+"/test/"+nopath)
shutil.move(filename, self.settings["file_path"]+name+"/test"+nopath+
"not"+name+"-"+ self.lang[:2] +"-"+str(uuid.uuid1())+".wav")
self.log.info("move file: "+filename)
i = i + 1
else:
if not os.path.isdir(self.settings["file_path"]+name+nopath):
os.makedirs(self.settings["file_path"]+name+nopath)
shutil.move(filename, self.settings["file_path"]+name+nopath+
"not"+name+"-"+ self.lang[:2] +"-"+str(uuid.uuid1())+".wav")
self.log.info("move file: "+filename)
i = i + 1
self.speak_dialog("start.calculating")
self.calculating_intent(name, message)