def test_get_all_models(self):
guessKeeper = gk.GuessKeeper(threshold=self.thresholds)
# Expect up for file1
guessKeeper.add_guess("file1", "up", 0.001)
guessKeeper.add_guess("file1", "down", 0.25)
guessKeeper.add_guess("file1", "left", 0.005)
guessKeeper.add_guess("file1", "right", 0.15)
# Expect silence for file 2
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
# Expect other for file 3
guessKeeper.add_guess("file3", "up", 0.5)
guessKeeper.add_guess("file3", "down", 0.5)
guessKeeper.add_guess("file3", "left", 0.5)
guessKeeper.add_guess("file3", "right", 0.5)
# Expect left for file 4
guessKeeper.add_guess("file4", "up", 0.5)
guessKeeper.add_guess("file4", "down", 0.25)
guessKeeper.add_guess("file4", "left", 0.005)
guessKeeper.add_guess("file4", "right", 0.15)
models = guessKeeper.get_unique_models()
self.assertTrue(len(models) == 4)
self.assertTrue('up' in models)
self.assertTrue('silence' in models)
self.assertTrue('other' in models)
self.assertTrue('left' in models)
guessKeeper = None
def test_guess_keeper_counts_by_guesses(self):
guessKeeper = gk.GuessKeeper(threshold=self.thresholds)
guessKeeper.add_guess("file1", "up", 0.5)
guessKeeper.add_guess("file1", "down", 0.25)
guessKeeper.add_guess("file1", "left", 0.005)
guessKeeper.add_guess("file1", "right", 0.15)
guessKeeper.add_guess("file2", "up", 0.5)
guessKeeper.add_guess("file2", "down", 0.25)
guessKeeper.add_guess("file2", "left", 0.005)
guessKeeper.add_guess("file2", "right", 0.15)
guessKeeper.add_guess("file3", "up", 0.5)
guessKeeper.add_guess("file3", "down", 0.25)
guessKeeper.add_guess("file3", "left", 0.005)
guessKeeper.add_guess("file3", "right", 0.15)
self.assertEqual(len(guessKeeper.get_files_by_model('left')), 3)
self.assertEqual(len(guessKeeper.get_files_by_model('up')), 0)
self.assertEqual(len(guessKeeper.get_files_by_model('down')), 0)
self.assertEqual(len(guessKeeper.get_files_by_model('right')), 0)
self.assertEqual(len(guessKeeper.get_files_by_model('silence')), 0)
guessKeeper = None
def test_guesses_keeper_by_file(self):
guessKeeper = gk.GuessKeeper(threshold=self.thresholds)
guessKeeper.add_guess("file1", "on", 0.5)
guessKeeper.add_guess("file1", "on", 0.25)
guessKeeper.add_guess("file1", "on", 0.05)
guessKeeper.add_guess("file1", "on", 0.15)
guessKeeper.add_guess("file1", "on", 1.15)
guessKeeper.add_guess("file2", "off", 0.08)
guessKeeper.add_guess("file3", "off", 1.08)
files = guessKeeper.get_all_guesses()
filenames = [x['filename'] for x in files]
self.assertTrue(len(files) == 3)
self.assertTrue("file1" in filenames)
self.assertTrue("file2" in filenames)
self.assertTrue("file3" in filenames)
guessKeeper = None
def test_get_word_counts(self):
guessKeeper = gk.GuessKeeper(threshold=self.thresholds)
# Expect up for file1
guessKeeper.add_guess("file1", "up", 0.001)
guessKeeper.add_guess("file1", "down", 0.25)
guessKeeper.add_guess("file1", "left", 0.005)
guessKeeper.add_guess("file1", "right", 0.15)
# Expect silence for file 2
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
guessKeeper.add_guess("file2", "silence", -1.0)
# Expect up for file 3
guessKeeper.add_guess("file3", "up", 0.005)
guessKeeper.add_guess("file3", "down", 0.5)
guessKeeper.add_guess("file3", "left", 0.5)
guessKeeper.add_guess("file3", "right", 0.5)
# Expect other for file 4
guessKeeper.add_guess("file4", "up", 0.5)
guessKeeper.add_guess("file4", "down", 0.5)
guessKeeper.add_guess("file4", "left", 0.5)
guessKeeper.add_guess("file4", "right", 0.5)
word_counts = guessKeeper.get_word_counts()
self.assertTrue("up" in word_counts.keys())
self.assertTrue("silence" in word_counts.keys())
self.assertTrue("other" in word_counts.keys())
self.assertTrue(word_counts["up"] == 2)
self.assertTrue(word_counts["silence"] == 1)
self.assertTrue(word_counts["other"] == 1)
guessKeeper = None
def test_get_model_by_name(self):
guessKeeper = gk.GuessKeeper(threshold=self.thresholds)
# Expect up for file1
guessKeeper.add_guess("file1", "up", 0.001)
self.assertEqual("up", guessKeeper.get_model("file1"))
class MultiEvaluator:
guesses_file = None
thresholds = {
'on': 0.000000000125,
'off': 0.00000125,
'up': 0.00000125,
'down': 0.00000125,
'left': 0.00000125,
'right': 0.00000125,
'stop': 0.1,
'go': 0.00000125,
'yes': 0.00000125,
'no': 0.00000125
}
guessKeeper = gk.GuessKeeper(threshold=thresholds)
model = None
class_indices = dict()
reported_categories = [
'stop', 'off', 'yes', 'no', 'go', 'up', 'down', 'left', 'right', 'on'
]
saved_model_dir = "../saved_models"
def __init__(self):
self.guesses_file = open("guesses.csv".format(ts), "w")
def get_last_file(self, extension):
list_of_files = glob.glob(
extension) # * means all if need specific format then *.csv
latest_file = max(list_of_files, key=os.path.getctime)
print("Using file: {0}".format(latest_file))
return latest_file
# def get_file_processing_list(self):
# files = list()
# for category in os.listdir(self.training_file_root_directory):
# if category in self.training_categories:
# training_path = os.path.join(self.training_file_root_directory, category)
# for filename in [x for x in os.listdir(training_path) if x.endswith('.wav')]:
# fullpath = "{0}/{1}".format(training_path, filename)
# files.append({'category': category, 'filename': fullpath})
# return files
def get_evaluation_files(self, w, path):
files = list()
eval_directory = "{0}/{1}".format(w, path)
for filename in [
x for x in os.listdir(eval_directory) if x.endswith('.wav')
]:
full_path = "{0}/{1}".format(eval_directory, filename)
files.append(full_path)
return files
def save_file_output(self, filename, scores):
self.guesses_file.write(filename)
scorestring = ""
for score in scores:
self.guesses_file.write(str(score))
scorestring = "{0},{1}".format(scorestring, score["score"])
print("{0}{1}".format(filename.split("/")[-1], scorestring))
def load_model_by_name(self, model_name):
print("Loading model.")
print(
"Loading model configuration from file {0}. One moment...".format(
model_name))
model = None
try:
model = load_model("{0}/{1}.h5".format(self.saved_model_dir,
model_name))
model.summary()
print("Model loaded.")
except Exception as exp:
print("Error loading {0}...".format(model_name))
print(exp.message)
return model
def get_filter_bank_features(self, f):
(rate, sig) = wav.read(f)
max_vol = max(sig)
# Calculate the mfcc features based on the file data
#filter_bank_features = mfcc(sig, rate, nfft=1200)
# Calculate the filterbank from the audio file
filter_bank_features = logfbank(sig, rate, nfft=1600)
filter_bank_features = filter_bank_features.T
if filter_bank_features.shape[0] < 26 or filter_bank_features.shape[
1] < 99:
zeros = np.zeros((26, 99), dtype=np.int32)
zeros[:filter_bank_features.shape[0], :filter_bank_features.
shape[1]] = filter_bank_features
return zeros, max_vol
else:
return filter_bank_features, max_vol
def evaluate(self, path, subdirectories, models):
fig = 0
total_count = 0
total_correct = 0
word_counts = dict()
total = 0
print("Evaluating")
for i in subdirectories:
eval_files = self.get_evaluation_files(path, i)
lnf = len(eval_files)
# Initialize everything to other...
self.guessKeeper.initialize_files(eval_files, 'other', 1.0)
total_count = total_count + 1
for k in self.reported_categories:
model = models[k]
cnt = 0
print("processing {0}. Files found: {1}".format(k, lnf))
for fileobj in eval_files:
f = fileobj
cnt = cnt + 1
if cnt > 10000 and cnt % 10000 == 0:
print("{0} of {1}... {2}%".format(
cnt, lnf, int((float(cnt) / float(lnf)) * 100.0)))
e = self.evaluate_file(model, f)
if e == -1:
modelname = "silence"
else:
modelname = k
self.guessKeeper.add_guess(filename=f,
modelname=modelname,
score=e)
word_counts = self.guessKeeper.get_word_counts()
plt.bar(range(len(word_counts)), word_counts.values(), align='center')
plt.xticks(range(len(word_counts)), word_counts.keys())
print("-------------------------------------")
print("\n\nTotal: {0} Correct: {1} Final accuracy: {2}".format(
total_count, total_correct, total_correct / total_count))
plt.tight_layout()
plt.show()
for i in word_counts.items():
print("{0}: {1}".format(i[0], i[1]))
return self.guessKeeper.get_all_guesses()
def evaluate_file(self, model, filename):
filter_bank_features, max_vol = self.get_filter_bank_features(filename)
c = None
guess = None
if max_vol < 1000:
guess = -1
else:
scale = 255.0 / np.amax(filter_bank_features)
filter_bank_features = filter_bank_features * scale
if filter_bank_features.shape[
0] == 26 and filter_bank_features.shape[1] == 99:
filter_bank_features = np.reshape(filter_bank_features,
(26, 99, 1))
filter_bank_features = np.expand_dims(filter_bank_features,
axis=0)
c = model.predict(filter_bank_features,
batch_size=1,
verbose=0)
guess = c[0][0]
return guess