def evaluate_convs(parallel,
config_path: str,
convs: List[str],
eval_config: dict,
showprog=False):
import trainNN.evaluate
totals = {}
results = {}
if "weights_file" not in eval_config and 'random_baseline' not in eval_config and eval_config[
"epoch"] == "best":
eval_config["epoch"], eval_config[
"weights_file"] = trainNN.evaluate.get_best_epoch(
load_config(config_path))
convids = [
"{}-{}".format(conv, channel) for conv in convs
for channel in ["A", "B"]
]
tasks = [
delayed(evaluate_conv)(config_path, convid, eval_config)
for convid in convids
]
if showprog:
tasks = tqdm(tasks)
for convid, result in parallel(tasks):
results[convid] = result
for k, v in result.items():
if k == 'confusion_matrix':
totals.setdefault(k, np.zeros_like(v))
totals[k] = totals.get(k, 0) + v
result.update(precision_recall(result))
totals.update(precision_recall(totals))
return dict(config=eval_config, totals=totals) # , details=results)
def get_net_output(convid: str, path: List[str]):
if path[-1].endswith(".smooth"):
path[-1] = path[-1][:-len(".smooth")]
smooth = True
else:
smooth = False
version, id = path
version_path = os.path.relpath(os.path.realpath(os.path.join("trainNN", "out", version)))
config_path = os.path.join(version_path, "config.json")
config = util.load_config(config_path)
features = Features(config, config_path)
eval_conf = evaluate.get_best_eval_config(config_path, filter=want_margin_filter)
if smooth:
return config_path, eval_conf, features.smooth(convid, id, eval_conf['smoother'])
else:
return config_path, eval_conf, features.get_multidim_net_output(convid, id)
def findAllNets():
import os
from os.path import join, isdir, isfile
folder = join("trainNN", "out")
rootList = []
for netversion in sorted(os.listdir(folder)):
path = join(folder, netversion)
if not isdir(path) or not isfile(join(path, "train.log")):
continue
net_conf_path = join(path, "config.json")
if isfile(net_conf_path):
curList = []
rootList.append({'name': netversion, 'children': curList})
net_conf = util.load_config(net_conf_path)
stats = net_conf['train_output']['stats']
curList.append("best")
curList.append("best.smooth")
curList.append("best.smooth.thres")
for id, info in stats.items():
curList.append(id)
return rootList
def __init__(self, reader: DBReader, client_socket):
self.reader = reader
self.client_socket = client_socket
self.featurenames = []
self.previous_end_offset_samples = 0
self.client_sample_rate = 8000
self.client_dtype = 'float32'
self.buffer_length_s = 60 # must be the same as on client side (AudioRecorder.bufferDuration_s)
self.client_microphone = Audio(np.zeros(self.client_sample_rate * self.buffer_length_s, dtype='int16'),
self.client_sample_rate)
self.nn_adc = Audio(np.zeros(self.client_sample_rate * self.buffer_length_s, dtype='int16'),
self.client_sample_rate)
self.previous_end_offset_samples = 0
self.frame_window_ms = 32 # config['extract_config']['sample_window_ms']
self.window_shift_ms = 10
self.was_above_thres_prev = False
self.do_nn = "trainNN/out/v050-finunified-65-ge1015c2-dirty:lstm-best-features-raw_power,pitch,ffv_live/config.json"
self.dimensions = {
'ffv': 7,
'raw_power': 1,
'pitch': 1
}
for featname in self.reader.config['extract_config']['input_features']:
pref, featnamepart = featname[0:4], featname[4:]
if pref != "get_":
raise Exception(f"invalid featname {featname}")
self.featurenames.append(featnamepart)
self.features = {featurename: Feature(
np.zeros((round(1000 * self.buffer_length_s / self.window_shift_ms), self.dimensions[featurename]),
dtype=np.float32),
infofrom=dict(
frame_window_ms=self.frame_window_ms,
frame_shift_ms=self.window_shift_ms,
initial_frame_offset_ms=0
))
for featurename in self.featurenames}
if self.do_nn is not None:
self.nn_config = util.load_config(self.do_nn)
self.eval_config = evaluate.get_best_eval_config(self.do_nn, filter=want_margin_filter)
epoch, _ = trainNN.evaluate.get_best_epoch(self.nn_config)
layers, self.nn_fn = trainNN.evaluate.get_network_outputter(self.do_nn, epoch, batch_size=None)
self.context_frames = self.nn_config['train_config']['context_frames']
self.context_stride = self.nn_config['train_config']['context_stride']
self.context_range = self.context_frames * self.context_stride
self.smoother_context_range = 1000 // self.window_shift_ms # 500ms
self.smoother = self.reader.features.smoothing_for_live(self.window_shift_ms, self.eval_config['smoother'])
def feat():
return Feature(
np.zeros((round(1000 * self.buffer_length_s / self.window_shift_ms), 1 if single_nn_out_dim else 2),
dtype=np.float32),
infofrom=dict(
frame_window_ms=self.frame_window_ms,
frame_shift_ms=self.window_shift_ms,
initial_frame_offset_ms=0
))
self.features["nn"] = feat()
self.features["nn.smooth"] = feat()
self.features["nn.smooth.bc"] = self.nn_adc
import random
st = random.choice(write_wavs.good_bc_sample_tracks)
print(f"bcs from {st}")
self.bc_samples = list(
write_wavs.bcs_to_samples(
reader,
write_wavs.get_boring_bcs(reader.config_path, st)))
def start_server():
start_server = websockets.serve(handler, "0.0.0.0", 8765)
print("server started")
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
def fill_caches(config_path: str):
reader = DBReader(config_path, originalDb=True)
micro = MicrophoneHandler(reader, None)
for st in write_wavs.good_bc_sample_tracks:
list(
write_wavs.bcs_to_samples(
reader,
write_wavs.get_boring_bcs(reader.config_path, st)))
if __name__ == '__main__':
config_path = sys.argv[1]
config = util.load_config(config_path)
origReader = DBReader(config_path, originalDb=True)
conversations = readDB.read_conversations(config)
netsTree = findAllNets()
start_server()
def train():
from . import network_model, evaluate
global reader
global backchannels
global config_path
config_path = sys.argv[1]
config = load_config(config_path)
version = subprocess.check_output("git describe --dirty",
shell=True).decode('ascii').strip()
if config_path.startswith("trainNN/out"):
out_dir = os.path.dirname(config_path)
print("Continuing training from folder " + out_dir)
load_stats = config['train_output']['stats']
load_epoch = max([int(epoch) for epoch in load_stats.keys()])
load_params = os.path.join(
out_dir,
config['train_output']['stats'][str(load_epoch)]['weights'])
print(
f"Continuing training from folder {out_dir}, epoch={load_epoch}, params={load_params}"
)
config.setdefault('train_output_old',
{})[load_epoch] = config['train_output']
else:
load_epoch = -1
load_stats = {}
load_params = None
out_dir = os.path.join("trainNN", "out",
version + ":" + config['name'])
if os.path.isdir(out_dir):
print(
"Output directory {} already exists, aborting".format(out_dir))
sys.exit(1)
os.makedirs(out_dir, exist_ok=True)
LOGFILE = os.path.join(out_dir, "train.log")
logging.root.handlers.clear()
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)-8s %(message)s',
handlers=[logging.FileHandler(LOGFILE),
logging.StreamHandler()])
logging.debug("version={}:{}".format(version, config['name']))
reader = readDB.loadDBReader(config_path)
train_config = config['train_config']
context_stride = train_config['context_stride']
context_frames = int(train_config['context_ms'] / 10 / context_stride)
train_config['context_frames'] = context_frames
gaussian = train_config['gaussian']
out_all = {'all': True, 'single': False}[train_config['output_type']]
if gaussian:
raise Exception("not implemented")
# train_data = load_numpy_file(os.path.join(dir, train_config['files']['train']))
# validate_data = load_numpy_file(os.path.join(dir, train_config['files']['validate']))
# train_inputs, train_outputs = train_data[:, :input_dim], train_data[:, input_dim]
# validate_inputs, validate_outputs = validate_data[:, :input_dim], validate_data[:, input_dim]
else:
batchers = {}
for t in 'train', 'validate':
# with open(os.path.join(dir, train_config['files'][t]['ids'])) as f:
# meta = json.load(f)
# groups = [slice(begin, end) for begin, end in meta['ranges']]
# inputs = load_numpy_file(os.path.join(dir, train_config['files'][t]['input']))
# outputs = load_numpy_file(os.path.join(dir, train_config['files'][t]['output']))
convos = readDB.read_conversations(config)
balance_method = train_config.get('balance_method', None)
uttids = [
bc for bc in readDB.all_uttids(config_path, convos[t])
if extract(bc) is not None
]
if balance_method is None:
backchannels = list(readDB.balance_data(config_path, uttids))
elif balance_method == "weighted":
backchannels = list(
readDB.get_balanced_weights(config_path, uttids))
else:
raise Exception(f"unknown balance method {balance_method}")
input_dim = extract(backchannels[0])[0].shape[1]
logging.debug(f"set input dim to {input_dim}")
inxtoname = {
**{v: k
for k, v in reader.category_to_index.items()}, 0: None
}
train_config['input_dim'] = input_dim
if config['extract_config'].get('categories', None) is not None:
category_names = [
inxtoname[inx]
for inx in range(len(reader.categories) + 1)
]
train_config['category_names'] = category_names
train_config['num_labels'] = len(category_names)
else:
train_config['num_labels'] = 2
logging.debug(f"input dim = {input_dim}")
context_stride = train_config['context_stride']
context_length = int(train_config['context_ms'] / 10 /
context_stride)
sequence_length = int(
(reader.method['nbc'][1] - reader.method['nbc'][0]) * 1000 /
10)
inner_indices = windowed_indices(sequence_length, context_length,
context_stride)
all_elements = list(itertools.product(backchannels, inner_indices))
batchers[t] = partial(iterate_minibatches, train_config,
all_elements, out_all)
# batchers[t] = iterate_faster_minibatches(train_config, all_elements, out_all)
before = time.perf_counter()
before_cpu = time.process_time()
logging.debug("loading data into ram")
i = 0
for backchannel in tqdm(backchannels):
extract(backchannel)
logging.debug(
f"loading data took {time.perf_counter () - before:.3f}s (cpu: {time.process_time()-before_cpu:.3f}s)"
)
create_network = getattr(network_model, train_config['model_function'])
model = create_network(train_config)
out_layer = model['output_layer']
resume_parameters = train_config.get('resume_parameters', None)
finetune_config = train_config.get("finetune", None)
if finetune_config is not None:
import lasagne
if load_params is not None or resume_parameters is not None:
raise Exception("cant finetune and load")
ft_config_path = finetune_config['config']
epoch = finetune_config['epoch']
which_layers = finetune_config['layers']
ft_layers, _ = evaluate.get_network_outputter(ft_config_path,
epoch,
batch_size=250)
layers = lasagne.layers.get_all_layers(out_layer)
for inx, (layer_config, layer,
ft_layer) in enumerate(zip(which_layers, layers, ft_layers)):
do_load = layer_config['load']
do_freeze = layer_config['freeze']
if do_load:
for param, ft_param in zip(layer.get_params(),
ft_layer.get_params()):
param.set_value(ft_param.get_value())
logging.info(
f"loaded layer {inx} ({ {repr(p): p.get_value().shape for p in layer.get_params()} })"
)
if do_freeze:
logging.info(f"freezing layer {inx}")
train_func.freeze(layer)
stats_generator = train_func.train_network(
network=out_layer,
twodimensional_output=False,
scheduling_method=None,
start_epoch=load_epoch + 1,
resume=load_params if load_params is not None else resume_parameters,
l2_regularization=train_config.get("l2_regularization", None),
# scheduling_params=(0.8, 0.000001),
update_method=train_config['update_method'],
num_epochs=train_config['epochs'],
learning_rate_num=train_config['learning_rate'],
iterate_minibatches_train=batchers['train'],
iterate_minibatches_validate=batchers['validate'],
categorical_output=not gaussian,
output_prefix=os.path.join(out_dir, "epoch"))
config_out = os.path.join(out_dir, "config.json")
for stats in stats_generator:
for k, v in stats.items():
v['weights'] = os.path.basename(v['weights'])
with open(config_out, "w") as f:
json.dump(
{
**config, 'train_output': {
'stats': {
**load_stats,
**stats
},
'source': config_path,
'environment': dict(os.environ)
}
},
f,
indent='\t')
logging.info("Wrote output to " + config_out)
latest_path = os.path.join("trainNN", "out", "latest")
with contextlib.suppress(FileNotFoundError):
os.remove(latest_path)
os.symlink(version, latest_path)
"sw4028-B", "sw3662", "sw2073", "sw3105", "sw2307", "sw3942", "sw2307",
"sw3715", "sw2027", "sw2849", "sw2787", "sw3357", "sw2389"
]
# assume problems are symmetric
bad_eval_convos = [track.split("-")[0] for track in bad_eval_tracks]
good_eval_tracks = []
if __name__ == '__main__':
config_path = sys.argv[1]
args = config_path.split("/")
version = "None"
if len(args) == 4:
_, _, version, _ = args
config = load_config(config_path)
reader = loadDBReader(config_path)
conversations = read_conversations(config)
eval_conversations = sorted(conversations['eval'])
eval_conversations = [
convo for convo in eval_conversations if convo not in bad_eval_convos
]
# valid_conversations = sorted(conversations['validate'])
write_wavs(reader,
eval_conversations,
1e10,
version,
good_bc_sample_tracks,
write_mono=True,
write_nn=True,
write_orig=False,
def main():
config_path = sys.argv[1]
dowhat = sys.argv[2]
if dowhat == "allmargins":
# auto find best params for all margins
manual_analysis = False
all_margins = True
elif dowhat == "bestmargin":
manual_analysis = False
all_margins = False
else:
raise Exception("dowhat?")
# return stat(config_path)
_, _, version, _ = config_path.split("/")
out_dir = os.path.join("evaluate", "out", version)
if os.path.isdir(out_dir):
print("Output directory {} already exists, aborting".format(out_dir))
sys.exit(1)
os.makedirs(out_dir, exist_ok=True)
logging.root.handlers.clear()
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)-8s %(message)s',
handlers=[
# logging.FileHandler(LOGFILE),
logging.StreamHandler()
])
config = load_config(config_path)
conversations = read_conversations(config)
res = []
eval_conversations = sorted(conversations['eval'])
valid_conversations = sorted(conversations['validate'])
do_baseline = config['eval_config'].get('do_random_baseline', None)
with Parallel(n_jobs=int(os.environ.get('JOBS', '1'))) as parallel:
print(f"filling caches...")
eval_config = default_config
if do_baseline is not None:
eval_config['random_baseline'] = do_baseline
evaluate_convs(parallel,
config_path,
[*valid_conversations, *eval_conversations],
default_config,
showprog=True)
if not manual_analysis:
itera = gpyopt_all(parallel,
config_path,
valid_conversations,
eval_conversations,
only_best=not all_margins)
for inx, r in enumerate(itera):
print(f" itera {inx} ({len(r)} results)")
print(
f"bayesian search done, checking scores on eval data set")
res.extend(r)
with open(os.path.join(out_dir, "results.json"), "w") as f:
json.dump(res, f, indent='\t')
return
else:
if do_detailed_analysis:
confs = list(detailed_analysis(config))
else:
confs = list(general_interesting_2(config))
# manual search
for inx, eval_config in enumerate(confs):
if do_baseline is not None:
eval_config['random_baseline'] = do_baseline
print(f"\n{inx}/{len(confs)}: {eval_config}\n")
ev = evaluate_convs(parallel, config_path, eval_conversations,
eval_config)
va = evaluate_convs(parallel, config_path, valid_conversations,
eval_config)
res.append(
nptolist(
dict(config=ev['config'],
totals={
'eval': ev['totals'],
'valid': va['totals']
})))
with open(os.path.join(out_dir, "results.json"), "w") as f:
json.dump(res, f, indent='\t')