def main(): args = TrainData.parse_args(create_parser(usage)) inject_params(args.model) data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) train, test = data.load(args.use_train, not args.use_train, shuffle=False) inputs, targets = train if args.use_train else test filenames = sum(data.train_files if args.use_train else data.test_files, []) predictions = Listener.find_runner(args.model)(args.model).predict(inputs) stats = Stats(predictions, targets, filenames) print('Data:', data) if not args.no_filenames: fp_files = stats.calc_filenames(False, True, args.threshold) fn_files = stats.calc_filenames(False, False, args.threshold) print('=== False Positives ===') print('\n'.join(fp_files)) print() print('=== False Negatives ===') print('\n'.join(fn_files)) print() print(stats.counts_str(args.threshold)) print() print(stats.summary_str(args.threshold))
def main(): parser = create_parser(usage) parser.add_argument( 'models', nargs='*', help='List of model filenames in format: wake-word.yy-mm-dd.net') args = TrainData.parse_args(parser) if not (bool(args.pocketsphinx_dict) == bool(args.pocketsphinx_folder) == bool(args.pocketsphinx_wake_word)): parser.error('Must pass all or no Pocketsphinx arguments') data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) data_files = data.train_files if args.use_train else data.test_files print('Data:', data) metrics = {} if args.pocketsphinx_dict and args.pocketsphinx_folder and args.pocketsphinx_wake_word: if not isfile(args.pocketsphinx_dict): parser.error('No such file: ' + args.pocketsphinx_dict) if not isdir(args.pocketsphinx_folder): parser.error('No such folder: ' + args.pocketsphinx_folder) listener = PocketsphinxListener(args.pocketsphinx_wake_word, args.pocketsphinx_dict, args.pocketsphinx_folder, args.pocketsphinx_threshold) stats = test_pocketsphinx(listener, data_files) metrics[args.pocketsphinx_dict] = stats_to_dict(stats) for model_name in args.models: print('Calculating', model_name + '...') inject_params(model_name) train, test = data.load(args.use_train, not args.use_train) inputs, targets = train if args.use_train else test predictions = Listener.find_runner(model_name)(model_name).predict( inputs) stats = Stats(predictions, targets, sum(data_files, [])) print('----', model_name, '----') print(stats.counts_str()) print() print(stats.summary_str()) print() metrics[model_name] = stats.to_dict(args.threshold) print('Writing to:', args.output) with open(args.output, 'w') as f: json.dump(metrics, f)
def calc_stats(model_files, loader, use_train, filenames): model_data = {} for model in model_files: train, test = loader.load_for(model) inputs, targets = train if use_train else test print('Running network...') predictions = Listener.find_runner(model)(model).predict(inputs) print(inputs.shape, targets.shape) print('Generating statistics...') stats = Stats(predictions, targets, filenames) print('\n' + stats.counts_str() + '\n\n' + stats.summary_str() + '\n') model_name = basename(splitext(model)[0]) model_data[model_name] = stats return model_data
def main(): args = TrainData.parse_args(create_parser(usage)) inject_params(args.model) data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) train, test = data.load(args.use_train, not args.use_train) inputs, targets = train if args.use_train else test filenames = sum(data.train_files if args.use_train else data.test_files, []) predictions = Listener.find_runner(args.model)(args.model).predict(inputs) stats = calc_stats(filenames, targets, predictions) print('Data:', data) show_stats(stats, not args.no_filenames)
def run(self): args = self.args data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) data_files = data.train_files if args.use_train else data.test_files print('Data:', data) metrics = {} if self.is_pocketsphinx: script = PocketsphinxTestScript.create( key_phrase=args.pocketsphinx_wake_word, dict_file=args.pocketsphinx_dict, hmm_folder=args.pocketsphinx_folder, threshold=args.pocketsphinx_threshold) ww_files, nww_files = data_files script.run_test(ww_files, 'Wake Word', 1.0) script.run_test(nww_files, 'Not Wake Word', 0.0) stats = script.get_stats() metrics[args.pocketsphinx_dict] = stats.to_dict(args.threshold) for model_name in args.models: print('Calculating', model_name + '...') inject_params(model_name) train, test = data.load(args.use_train, not args.use_train) inputs, targets = train if args.use_train else test predictions = Listener.find_runner(model_name)(model_name).predict( inputs) stats = Stats(predictions, targets, sum(data_files, [])) print('----', model_name, '----') print(stats.counts_str()) print() print(stats.summary_str()) print() metrics[model_name] = stats.to_dict(args.threshold) print('Writing to:', args.output) with open(args.output, 'w') as f: json.dump(metrics, f)
def main(): args = TrainData.parse_args(create_parser(usage)) inject_params(args.model) data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) #print(data) #print(args.use_train) train, test = data.load(args.use_train, not args.use_train) #print(train) #print(test) inputs, targets = train if args.use_train else test #print(inputs) filenames = sum(data.train_files if args.use_train else data.test_files, []) #for i in range(0, len(targets)): # print(filenames[i], "\t", targets[i]) print(args.model) predictions = Listener.find_runner(args.model)(args.model).predict(inputs) #for i, filename in enumerate(filenames): #print(filename, "\t", predictions[i], targets[i]) #print(predictions[i]) #print (predictions) #print (len(predictions)) stats = calc_stats(filenames, targets, predictions) print('Data:', data) show_stats(stats, not args.no_filenames)
def __init__(self): self.args = create_parser(usage).parse_args() inject_params(self.args.model) self.runner = Listener.find_runner(self.args.model)(self.args.model) self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)
def __init__(self, args): super().__init__(args) inject_params(self.args.model) self.runner = Listener.find_runner(self.args.model)(self.args.model) self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)