def train():
logging.info('train bss model')
sig1_list = util.load_data(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT)
# sig1_list = util.load_data_from_matlab(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT)
sig2 = util.read_wav(config.DATA_TRAIN_STATIC_SIGNAL)
# sig2 = util.read_wav_from_matlab(config.DATA_TRAIN_STATIC_SIGNAL)
# print('sig1_list is', sig1_list)
# print('sig2 is', sig2)
if config.NORMALIZATION:
sig1_list = [_normalize(sig) for sig in sig1_list]
sig2 = _normalize(sig2)
logging.info('mix signals')
util.mkdir_p(config.DATA_TRAIN_MIX_ROOT)
mix_filenames = _gen_mix_filename(config.DATA_TRAIN_LIST,
config.DATA_TRAIN_STATIC_SIGNAL,
config.DATA_TRAIN_MIX_ROOT)
sig2_list = [sig2 for _ in range(len(sig1_list))]
sig1_list, sig2_list, mix_list = util.mix(sig1_list,
sig2_list,
output=mix_filenames)
# print('sig1_list is', sig1_list)
# print('sig2_list is', sig2_list)
# print('mix_list is', mix_list)
# logging.info('extract stft features')
# sig1_stft = _extract_stft(sig1_list)
# print('sig1_stft', sig1_stft)
# sig2_mrcg = _extract_stft(sig2_list)
# mix_mrcg = _extract_stft(mix_list)
logging.info('extract mrcg features')
sig1_mrcg = _extract_mrcg(sig1_list)
print('sig1_mrcg is', sig1_mrcg)
sig2_mrcg = _extract_mrcg(sig2_list)
print('sig2_mrcg is', sig2_mrcg)
mix_mrcg = _extract_mrcg(mix_list)
print('mix_mrcg is', mix_mrcg)
logging.info('train neural network')
sig1_mrcg = util.r_abs(sig1_mrcg)
sig2_mrcg = util.r_abs(sig2_mrcg)
mix_mrcg = util.r_abs(mix_mrcg)
train_x, train_y = mix_mrcg, []
for s, n in zip(sig1_mrcg, sig2_mrcg): # 将signal和noise拼接作为nn输出
train_y.append(np.concatenate((s, n), axis=1))
train_x = [_extend(sig, config.EXTEND_NUM) for sig in train_x]
# 各层神经元数量
layer_size = [len(train_x[0][0])] # 输入层
layer_size.extend(config.LAYER_SIZE) # 隐藏层
layer_size.append(len(train_y[0][0])) # 输出层
dnn = nn.NNet(layer_size)
util.mkdir_p(config.MODEL_ROOT)
dnn.train(train_x,
train_y,
model_path=config.MODEL_NN_PATH,
training_epochs=config.EPOCH,
learning_rate=config.LEARNING_RATE)
def observe(self, pobs):
if self.design.files:
self.mix, self.refs = util.mix_files(
self.design.fn1,
self.design.fn2,
snr=self.design.snr,
fs=self.fs,
desired_length_sec=self.mix_dur)
else:
self.mix, self.refs = util.mix(self.design.in1,
self.design.in2,
snr=self.design.snr,
fs=self.fs,
desired_length_sec=self.mix_dur)
if defaults.DO_NOISE:
noise_var = numpy.var(self.mix) * 10**(-defaults.NOISING_SNR / 10)
noise = numpy.sqrt(noise_var) * numpy.random.randn(len(self.mix), )
# x = x + noise
self.refs += noise[:, numpy.newaxis] / 2
self.mix = numpy.sum(self.refs, axis=1)
self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop)
if pobs is None:
self.pobs = dict()
self.pobs['ft'] = util.normalize(numpy.abs(self.X))
else:
self.pobs = pobs
if self.do_write: # write audio in
for s in range(self.S):
util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) +
'.wav',
sound=self.refs[:, s],
fs=self.fs)
util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav',
sound=self.mix,
fs=self.fs)
def handle_item_effects() -> None:
#TODO, finish this
pass
if c.BOMBOMB_EFFECT: #mix items
u.mix(c.ALL_ITEMS)
c.BOMBOMB_EFFECT = False #reset flag
if c.POW_BUTTON_EFFECT: #all, minus one point
for player in c.PLAYERS:
player.points -= 1
c.POW_BUTTON_EFFECT = False #reset flag
if c.FEATHER_EFFECT:
# print("change feather effect to something more interesting.")
u.rotate_players_left()
FEATHER_EFFECT = False #reset flag
if c.STAR_EFFECT:
# print("change star effect to something more interesting.")
STAR_EFFECT = False #reset flag
if c.QUESTION_BLOCK_EFFECT:
# print("change star effect to something more interesting.")
QUESTION_BLOCK_EFFECT = False #reset flag
def observe(self, pobs=None):
if self.design.files:
self.mix, self.refs = util.mix_files(
self.design.fn1,
self.design.fn2,
snr=self.design.snr,
fs=self.fs,
desired_length_sec=self.mix_dur)
else:
self.mix, self.refs = util.mix(self.design.in1,
self.design.in2,
snr=self.design.snr,
fs=self.fs,
desired_length_sec=self.mix_dur)
if defaults.DO_NOISE:
noise_var = numpy.var(self.mix) * 10**(-defaults.NOISING_SNR / 10)
noise = numpy.sqrt(noise_var) * numpy.random.randn(len(self.mix), )
self.refs += noise[:, numpy.newaxis] / 2
self.mix = numpy.sum(self.refs, axis=1)
self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop)
if pobs is None:
self.pobs = util.observe(x=self.mix,
N=self.N,
M=self.M,
Q=self.Q,
L=self.L,
hop=self.hop,
fs=self.fs,
R=self.R)
else:
self.pobs = pobs
self.R = self.pobs['R'] # in case R becomes odd
if self.do_write: # write audio in
print('\nsaving experiment design and q_init at %s' %
self.info['in_dir'])
for s in range(self.S):
util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) +
'.wav',
sound=self.refs[:, s],
fs=self.fs)
util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav',
sound=self.mix,
fs=self.fs)
def observe(self, pobs):
if self.design.files:
self.mix, self.refs = util.mix_files(self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur)
else:
self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur)
if defaults.DO_NOISE:
noise_var = numpy.var(self.mix)*10**(-defaults.NOISING_SNR/10)
noise = numpy.sqrt(noise_var)*numpy.random.randn(len(self.mix), )
# x = x + noise
self.refs += noise[:, numpy.newaxis]/2
self.mix = numpy.sum(self.refs, axis=1)
self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop)
if pobs is None:
self.pobs = dict()
self.pobs['ft'] = util.normalize(numpy.abs(self.X))
else:
self.pobs = pobs
if self.do_write: # write audio in
for s in range(self.S):
util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs)
util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def observe(self, pobs=None):
if self.design.files:
self.mix, self.refs = util.mix_files(self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur)
else:
self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur)
if defaults.DO_NOISE:
noise_var = numpy.var(self.mix)*10**(-defaults.NOISING_SNR/10)
noise = numpy.sqrt(noise_var)*numpy.random.randn(len(self.mix), )
self.refs += noise[:, numpy.newaxis]/2
self.mix = numpy.sum(self.refs, axis=1)
self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop)
if pobs is None:
self.pobs = util.observe(x=self.mix, N=self.N, M=self.M, Q=self.Q, L=self.L, hop=self.hop, fs=self.fs, R=self.R)
else:
self.pobs = pobs
self.R = self.pobs['R'] # in case R becomes odd
if self.do_write: # write audio in
print('\nsaving experiment design and q_init at %s' % self.info['in_dir'])
for s in range(self.S):
util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs)
util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def on_key_release(symbol, modifiers):
# options screen
if u.is_options_screen():
if symbol == key.UP:
options.selector_up()
elif symbol == key.DOWN:
options.selector_down()
elif symbol == key.B:
c.SCREEN = Screens.TITLE
elif symbol == key.LEFT:
options.selector_left()
elif symbol == key.RIGHT:
options.selector_right()
elif symbol == key.SPACE:
options.toggle_item()
# title screen
elif u.is_title_screen():
if symbol == key.UP:
title.selector_up()
elif symbol == key.DOWN:
title.selector_down()
#make a selection
elif symbol == key.ENTER:
if title.is_game_selected():
c.SCREEN = Screens.GAME
elif title.is_options_selected():
c.SCREEN = Screens.OPTIONS
# game screen
elif u.is_game_screen():
"""
Digits: 1
Letters: ABDFOSUX
Arrows: Left Right Up
"""
player = u.player_in_front()
if symbol == key.B:
c.SCREEN = Screens.TITLE
elif not u.any_movement():
if symbol == key._1:
if not problem.showing:
#TODO,
# determine what item the player will get,
# pass it to the problem instance
# the problem class should update itself based on the item passed to it.
problem.question.draw()
problem.toggle()
player.use_item()
yammy.wave_wand()
c.TRANSFER_ITEM = u.remove_item_from_platform()
i.add_item()
elif symbol == key.LEFT:
u.rotate_players_left()
elif symbol == key.RIGHT:
u.rotate_players_right()
elif symbol == key.UP:
c.PLAYERS = u.mix(c.PLAYERS)
#plus one point
elif symbol == key.O:
u.right_answer(player)
u.rotate_players_left()
if problem.showing:
problem.toggle()
problem.decided = False # clear the flag
#delete prior problem letter sprites
#minus one point
elif symbol == key.X:
u.wrong_answer(player)
if player.item:
player.item.poof()
player.item = None
u.rotate_players_left()
if problem.showing:
problem.toggle()
problem.decided = False # clear the flag
#delete prior problem letter sprites
elif symbol == key.A:
u.rotate_items_left()
elif symbol == key.D:
u.rotate_items_right()
elif symbol == key.S:
c.ALL_ITEMS = u.mix(c.ALL_ITEMS)
elif symbol == key.U:
player.use_item()
def test():
logging.info('test denoising model')
logging.info("read test signal according to %s, read noise from %s" %
(config.DATA_TEST_LIST, config.DATA_TEST_STATIC_SIGNAL))
sig1_list = util.load_data(config.DATA_TEST_LIST, config.DATA_TEST_ROOT)
sig2 = util.read_wav(config.DATA_TEST_STATIC_SIGNAL)
if config.NORMALIZATION:
sig1_list = [_normalize(sig) for sig in sig1_list]
sig2 = _normalize(sig2)
logging.info('mix signals')
util.mkdir_p(config.DATA_TEST_MIX_ROOT)
mix_filenames = _gen_mix_filename(config.DATA_TEST_LIST,
config.DATA_TEST_STATIC_SIGNAL,
config.DATA_TEST_MIX_ROOT)
sig2_list = [sig2 for _ in range(len(sig1_list))]
sig1_list, sig2_list, mix_list = util.mix(sig1_list,
sig2_list,
output=mix_filenames)
logging.info('extract stft features')
mix_mrcg = _extract_mrcg(mix_list)
mix_data = util.r_abs(mix_mrcg)
mix_data = [_extend(sig, config.EXTEND_NUM) for sig in mix_data]
logging.info('run neural network')
# 各层神经元数量
layer_size = [len(mix_data[0][0])] # 输入层
layer_size.extend(config.LAYER_SIZE) # 隐藏层
layer_size.append(len(mix_mrcg[0][0]) * 2) # 输出层
dnn = nn.NNet(layer_size)
sig1_sig2 = dnn.run(mix_data, model_path=config.MODEL_NN_PATH)
# 神经网络输出为预测signal1与signal2的拼接，此处分离
sig1_mrcg, sig2_mrcg = _separate(sig1_sig2)
logging.info('Time-Frequency Masking')
mask = []
for s, n in zip(sig1_mrcg, sig2_mrcg):
mask.append(_build_ibm(s, n))
sig1_mrcg = [np.multiply(mix, m) for mix, m in zip(mix_mrcg, mask)]
sig2_mrcg = [
np.subtract(mix, sig) for mix, sig in zip(mix_mrcg, sig1_mrcg)
]
logging.info('Inverse Short-Time Fourier Transformation')
# 从频域转换到时域
sep_sig1 = _istft(sig1_mrcg)
sep_sig2 = _istft(sig2_mrcg)
logging.info("write test audio into dir %s" % config.DATA_TEST_MIX_ROOT)
util.mkdir_p(config.DATA_TEST_MIX_ROOT)
file_list = _gen_mix_filename(config.DATA_TEST_LIST,
config.DATA_TEST_STATIC_SIGNAL,
config.DATA_TEST_MIX_ROOT)
for ss, sn, f in zip(sep_sig1, sep_sig2, file_list):
sf.write(f + '.sep.sig1.wav', ss, config.SAMPLE_RATE)
sf.write(f + '.sep.sig2.wav', sn, config.SAMPLE_RATE)
logging.info('run bss evaluation')
sdr, sir, sar = _bss_eval()
logging.info('SDR: %g, SIR: %g, SAR: %g' % (sdr, sir, sar))
def effect(self) -> None:
"""Randomly mix the order of items on the screen."""
c.PLAYERS = u.mix(c.PLAYERS)
def __getitem__(self, idx):
# return format:
# (
# mixed multichannel audio, (C,L)
# array of groundtruth with reverb for each target, (N, C, L)
# array of direction of targets, (N,)
# array of multichannel ideal groundtruths for each target, (N, C, L)
# noise (C, L)
# )
# check cache first
if idx >= self.length:
return None
if self.cache_folder is not None:
cache_path = self.cache_folder + '/' + str(idx) + '-' + str(self.seed) + '.npz'
if cache_path not in self.cache_history:
self.cache_history.append(cache_path)
if self.cache_max is not None and self.cache_max == len(self.cache_history):
# delete first one
first = self.cache_history[0]
os.remove(first)
self.cache_history = self.cache_history[1:]
if os.path.exists(cache_path):
cache_result = np.load(cache_path, allow_pickle=True)['data']
return cache_result[0], cache_result[1], cache_result[2], cache_result[3], cache_result[4],\
cache_result[5]
else:
cache_path = None
np.random.seed(self.__seed_for_idx(idx))
# n_source=np.random.randint(self.max_source)+1
n_source = 3
room_result = simulateRoom(n_source, self.min_room_dim, self.max_room_dim, self.min_gap, self.min_dist,
self.min_angle_diff)
if room_result is None:
return self.__getitem__(idx + 1) # backoff
room_dim, R_loc, source_loc, source_angles = room_result
voices = [self.truncator.process(self.voices[vi]) for vi in np.random.choice(len(self.voices), n_source)]
voices = [v * np.random.uniform(self.random_volume_range[0], self.random_volume_range[1]) for v in voices]
if self.special_noise_ratio < np.random.rand():
noise = self.truncator.process(self.noises[np.random.choice(len(self.noises))])
else:
noise = np.random.randn(self.truncator.get_length())
if self.randomize_material_ratio < np.random.rand():
ceiling, east, west, north, south = tuple(np.random.choice(wall_materials, 5)) # sample material
floor = np.random.choice(floor_materials) # sample material
mixed, premix_w_reverb, premix, R = simulateSound(room_dim, R_loc, source_loc, voices, 0,
(ceiling, east, west, north, south, floor),
self.max_order)
else:
rt60 = np.random.uniform(self.min_rt60, self.max_rt60)
mixed, premix_w_reverb, premix, R = simulateSound(room_dim, R_loc, source_loc, voices, rt60)
background = simulateBackground(noise)
snr = np.random.uniform(self.additive_noise_min_snr, self.additive_noise_max_snr)
# trucate to the same length
mixed = mixed[:, :truncator.get_length()]
background = background[:, :truncator.get_length()]
total, background = mix(mixed, background, snr)
# save cache
if cache_path is not None:
np.savez_compressed(cache_path, data=[total, premix_w_reverb, source_angles, premix, background, R])
if premix_w_reverb.shape[-1] <= 18000:
offset = 18000 - premix_w_reverb.shape[-1]
premix_w_reverb = np.pad(premix_w_reverb, ((0, 0), (0, 0), (0, offset)), "constant")
else:
premix_w_reverb = premix_w_reverb[..., :18000]
return total, premix_w_reverb, source_angles, premix, background, R, R_loc
quad_flat(stream, (left, top, width, height), (s0,t0,s1,t1), color)
offset += metrics["advance"]
def patch9_cell_flat(stream, patch9, x, y, (left, top, width, height), color):
x0,s0 = patch9.gridpoint(0, x+0, width)
x1,s1 = patch9.gridpoint(0, x+1, width)
y0,t0 = patch9.gridpoint(1, y+0, height)
y1,t1 = patch9.gridpoint(1, y+1, height)
quad_flat(stream, (left+x0, top+y0, x1-x0, y1-y0), (s0, t0, s1, t1), color)
def patch9_cell_gradient(stream, patch9, x, y, (left, top, width, height), (c0,c1,c2,c3)):
x0,s0 = patch9.gridpoint(0, x+0, width)
x1,s1 = patch9.gridpoint(0, x+1, width)
y0,t0 = patch9.gridpoint(1, y+0, height)
y1,t1 = patch9.gridpoint(1, y+1, height)
k0 = mix(c0, c1, float(x0)/width)
k1 = mix(c0, c1, float(x1)/width)
k2 = mix(c2, c3, float(x0)/width)
k3 = mix(c2, c3, float(x1)/width)
c0 = mix(k0, k2, float(y0)/height)
c1 = mix(k1, k3, float(y0)/height)
c2 = mix(k0, k2, float(y1)/height)
c3 = mix(k1, k3, float(y1)/height)
quad_gradient(stream, (left+x0, top+y0, x1-x0, y1-y0), (s0, t0, s1, t1), (c0,c1,c2,c3))
class Renderer(object):
def __init__(self, output, default_font, program=None):
self.output = output
self.default_font = default_font
self.stream = VertexStream(fmt)
self.textures = TextureCache()