def train(): logging.info('train bss model') sig1_list = util.load_data(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT) # sig1_list = util.load_data_from_matlab(config.DATA_TRAIN_LIST, config.DATA_TRAIN_ROOT) sig2 = util.read_wav(config.DATA_TRAIN_STATIC_SIGNAL) # sig2 = util.read_wav_from_matlab(config.DATA_TRAIN_STATIC_SIGNAL) # print('sig1_list is', sig1_list) # print('sig2 is', sig2) if config.NORMALIZATION: sig1_list = [_normalize(sig) for sig in sig1_list] sig2 = _normalize(sig2) logging.info('mix signals') util.mkdir_p(config.DATA_TRAIN_MIX_ROOT) mix_filenames = _gen_mix_filename(config.DATA_TRAIN_LIST, config.DATA_TRAIN_STATIC_SIGNAL, config.DATA_TRAIN_MIX_ROOT) sig2_list = [sig2 for _ in range(len(sig1_list))] sig1_list, sig2_list, mix_list = util.mix(sig1_list, sig2_list, output=mix_filenames) # print('sig1_list is', sig1_list) # print('sig2_list is', sig2_list) # print('mix_list is', mix_list) # logging.info('extract stft features') # sig1_stft = _extract_stft(sig1_list) # print('sig1_stft', sig1_stft) # sig2_mrcg = _extract_stft(sig2_list) # mix_mrcg = _extract_stft(mix_list) logging.info('extract mrcg features') sig1_mrcg = _extract_mrcg(sig1_list) print('sig1_mrcg is', sig1_mrcg) sig2_mrcg = _extract_mrcg(sig2_list) print('sig2_mrcg is', sig2_mrcg) mix_mrcg = _extract_mrcg(mix_list) print('mix_mrcg is', mix_mrcg) logging.info('train neural network') sig1_mrcg = util.r_abs(sig1_mrcg) sig2_mrcg = util.r_abs(sig2_mrcg) mix_mrcg = util.r_abs(mix_mrcg) train_x, train_y = mix_mrcg, [] for s, n in zip(sig1_mrcg, sig2_mrcg): # 将signal和noise拼接作为nn输出 train_y.append(np.concatenate((s, n), axis=1)) train_x = [_extend(sig, config.EXTEND_NUM) for sig in train_x] # 各层神经元数量 layer_size = [len(train_x[0][0])] # 输入层 layer_size.extend(config.LAYER_SIZE) # 隐藏层 layer_size.append(len(train_y[0][0])) # 输出层 dnn = nn.NNet(layer_size) util.mkdir_p(config.MODEL_ROOT) dnn.train(train_x, train_y, model_path=config.MODEL_NN_PATH, training_epochs=config.EPOCH, learning_rate=config.LEARNING_RATE)
def observe(self, pobs): if self.design.files: self.mix, self.refs = util.mix_files( self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) else: self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) if defaults.DO_NOISE: noise_var = numpy.var(self.mix) * 10**(-defaults.NOISING_SNR / 10) noise = numpy.sqrt(noise_var) * numpy.random.randn(len(self.mix), ) # x = x + noise self.refs += noise[:, numpy.newaxis] / 2 self.mix = numpy.sum(self.refs, axis=1) self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop) if pobs is None: self.pobs = dict() self.pobs['ft'] = util.normalize(numpy.abs(self.X)) else: self.pobs = pobs if self.do_write: # write audio in for s in range(self.S): util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs) util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def handle_item_effects() -> None: #TODO, finish this pass if c.BOMBOMB_EFFECT: #mix items u.mix(c.ALL_ITEMS) c.BOMBOMB_EFFECT = False #reset flag if c.POW_BUTTON_EFFECT: #all, minus one point for player in c.PLAYERS: player.points -= 1 c.POW_BUTTON_EFFECT = False #reset flag if c.FEATHER_EFFECT: # print("change feather effect to something more interesting.") u.rotate_players_left() FEATHER_EFFECT = False #reset flag if c.STAR_EFFECT: # print("change star effect to something more interesting.") STAR_EFFECT = False #reset flag if c.QUESTION_BLOCK_EFFECT: # print("change star effect to something more interesting.") QUESTION_BLOCK_EFFECT = False #reset flag
def observe(self, pobs=None): if self.design.files: self.mix, self.refs = util.mix_files( self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) else: self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) if defaults.DO_NOISE: noise_var = numpy.var(self.mix) * 10**(-defaults.NOISING_SNR / 10) noise = numpy.sqrt(noise_var) * numpy.random.randn(len(self.mix), ) self.refs += noise[:, numpy.newaxis] / 2 self.mix = numpy.sum(self.refs, axis=1) self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop) if pobs is None: self.pobs = util.observe(x=self.mix, N=self.N, M=self.M, Q=self.Q, L=self.L, hop=self.hop, fs=self.fs, R=self.R) else: self.pobs = pobs self.R = self.pobs['R'] # in case R becomes odd if self.do_write: # write audio in print('\nsaving experiment design and q_init at %s' % self.info['in_dir']) for s in range(self.S): util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs) util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def observe(self, pobs): if self.design.files: self.mix, self.refs = util.mix_files(self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) else: self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) if defaults.DO_NOISE: noise_var = numpy.var(self.mix)*10**(-defaults.NOISING_SNR/10) noise = numpy.sqrt(noise_var)*numpy.random.randn(len(self.mix), ) # x = x + noise self.refs += noise[:, numpy.newaxis]/2 self.mix = numpy.sum(self.refs, axis=1) self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop) if pobs is None: self.pobs = dict() self.pobs['ft'] = util.normalize(numpy.abs(self.X)) else: self.pobs = pobs if self.do_write: # write audio in for s in range(self.S): util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs) util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def observe(self, pobs=None): if self.design.files: self.mix, self.refs = util.mix_files(self.design.fn1, self.design.fn2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) else: self.mix, self.refs = util.mix(self.design.in1, self.design.in2, snr=self.design.snr, fs=self.fs, desired_length_sec=self.mix_dur) if defaults.DO_NOISE: noise_var = numpy.var(self.mix)*10**(-defaults.NOISING_SNR/10) noise = numpy.sqrt(noise_var)*numpy.random.randn(len(self.mix), ) self.refs += noise[:, numpy.newaxis]/2 self.mix = numpy.sum(self.refs, axis=1) self.X, _ = util.ec_stft(x=self.mix, N=self.N, hop=self.hop) if pobs is None: self.pobs = util.observe(x=self.mix, N=self.N, M=self.M, Q=self.Q, L=self.L, hop=self.hop, fs=self.fs, R=self.R) else: self.pobs = pobs self.R = self.pobs['R'] # in case R becomes odd if self.do_write: # write audio in print('\nsaving experiment design and q_init at %s' % self.info['in_dir']) for s in range(self.S): util.save_sound(sound_fn=self.info['in_dir'] + '/in' + str(s) + '.wav', sound=self.refs[:, s], fs=self.fs) util.save_sound(sound_fn=self.info['in_dir'] + '/mix.wav', sound=self.mix, fs=self.fs)
def on_key_release(symbol, modifiers): # options screen if u.is_options_screen(): if symbol == key.UP: options.selector_up() elif symbol == key.DOWN: options.selector_down() elif symbol == key.B: c.SCREEN = Screens.TITLE elif symbol == key.LEFT: options.selector_left() elif symbol == key.RIGHT: options.selector_right() elif symbol == key.SPACE: options.toggle_item() # title screen elif u.is_title_screen(): if symbol == key.UP: title.selector_up() elif symbol == key.DOWN: title.selector_down() #make a selection elif symbol == key.ENTER: if title.is_game_selected(): c.SCREEN = Screens.GAME elif title.is_options_selected(): c.SCREEN = Screens.OPTIONS # game screen elif u.is_game_screen(): """ Digits: 1 Letters: ABDFOSUX Arrows: Left Right Up """ player = u.player_in_front() if symbol == key.B: c.SCREEN = Screens.TITLE elif not u.any_movement(): if symbol == key._1: if not problem.showing: #TODO, # determine what item the player will get, # pass it to the problem instance # the problem class should update itself based on the item passed to it. problem.question.draw() problem.toggle() player.use_item() yammy.wave_wand() c.TRANSFER_ITEM = u.remove_item_from_platform() i.add_item() elif symbol == key.LEFT: u.rotate_players_left() elif symbol == key.RIGHT: u.rotate_players_right() elif symbol == key.UP: c.PLAYERS = u.mix(c.PLAYERS) #plus one point elif symbol == key.O: u.right_answer(player) u.rotate_players_left() if problem.showing: problem.toggle() problem.decided = False # clear the flag #delete prior problem letter sprites #minus one point elif symbol == key.X: u.wrong_answer(player) if player.item: player.item.poof() player.item = None u.rotate_players_left() if problem.showing: problem.toggle() problem.decided = False # clear the flag #delete prior problem letter sprites elif symbol == key.A: u.rotate_items_left() elif symbol == key.D: u.rotate_items_right() elif symbol == key.S: c.ALL_ITEMS = u.mix(c.ALL_ITEMS) elif symbol == key.U: player.use_item()
def test(): logging.info('test denoising model') logging.info("read test signal according to %s, read noise from %s" % (config.DATA_TEST_LIST, config.DATA_TEST_STATIC_SIGNAL)) sig1_list = util.load_data(config.DATA_TEST_LIST, config.DATA_TEST_ROOT) sig2 = util.read_wav(config.DATA_TEST_STATIC_SIGNAL) if config.NORMALIZATION: sig1_list = [_normalize(sig) for sig in sig1_list] sig2 = _normalize(sig2) logging.info('mix signals') util.mkdir_p(config.DATA_TEST_MIX_ROOT) mix_filenames = _gen_mix_filename(config.DATA_TEST_LIST, config.DATA_TEST_STATIC_SIGNAL, config.DATA_TEST_MIX_ROOT) sig2_list = [sig2 for _ in range(len(sig1_list))] sig1_list, sig2_list, mix_list = util.mix(sig1_list, sig2_list, output=mix_filenames) logging.info('extract stft features') mix_mrcg = _extract_mrcg(mix_list) mix_data = util.r_abs(mix_mrcg) mix_data = [_extend(sig, config.EXTEND_NUM) for sig in mix_data] logging.info('run neural network') # 各层神经元数量 layer_size = [len(mix_data[0][0])] # 输入层 layer_size.extend(config.LAYER_SIZE) # 隐藏层 layer_size.append(len(mix_mrcg[0][0]) * 2) # 输出层 dnn = nn.NNet(layer_size) sig1_sig2 = dnn.run(mix_data, model_path=config.MODEL_NN_PATH) # 神经网络输出为预测signal1与signal2的拼接,此处分离 sig1_mrcg, sig2_mrcg = _separate(sig1_sig2) logging.info('Time-Frequency Masking') mask = [] for s, n in zip(sig1_mrcg, sig2_mrcg): mask.append(_build_ibm(s, n)) sig1_mrcg = [np.multiply(mix, m) for mix, m in zip(mix_mrcg, mask)] sig2_mrcg = [ np.subtract(mix, sig) for mix, sig in zip(mix_mrcg, sig1_mrcg) ] logging.info('Inverse Short-Time Fourier Transformation') # 从频域转换到时域 sep_sig1 = _istft(sig1_mrcg) sep_sig2 = _istft(sig2_mrcg) logging.info("write test audio into dir %s" % config.DATA_TEST_MIX_ROOT) util.mkdir_p(config.DATA_TEST_MIX_ROOT) file_list = _gen_mix_filename(config.DATA_TEST_LIST, config.DATA_TEST_STATIC_SIGNAL, config.DATA_TEST_MIX_ROOT) for ss, sn, f in zip(sep_sig1, sep_sig2, file_list): sf.write(f + '.sep.sig1.wav', ss, config.SAMPLE_RATE) sf.write(f + '.sep.sig2.wav', sn, config.SAMPLE_RATE) logging.info('run bss evaluation') sdr, sir, sar = _bss_eval() logging.info('SDR: %g, SIR: %g, SAR: %g' % (sdr, sir, sar))
def effect(self) -> None: """Randomly mix the order of items on the screen.""" c.PLAYERS = u.mix(c.PLAYERS)
def __getitem__(self, idx): # return format: # ( # mixed multichannel audio, (C,L) # array of groundtruth with reverb for each target, (N, C, L) # array of direction of targets, (N,) # array of multichannel ideal groundtruths for each target, (N, C, L) # noise (C, L) # ) # check cache first if idx >= self.length: return None if self.cache_folder is not None: cache_path = self.cache_folder + '/' + str(idx) + '-' + str(self.seed) + '.npz' if cache_path not in self.cache_history: self.cache_history.append(cache_path) if self.cache_max is not None and self.cache_max == len(self.cache_history): # delete first one first = self.cache_history[0] os.remove(first) self.cache_history = self.cache_history[1:] if os.path.exists(cache_path): cache_result = np.load(cache_path, allow_pickle=True)['data'] return cache_result[0], cache_result[1], cache_result[2], cache_result[3], cache_result[4],\ cache_result[5] else: cache_path = None np.random.seed(self.__seed_for_idx(idx)) # n_source=np.random.randint(self.max_source)+1 n_source = 3 room_result = simulateRoom(n_source, self.min_room_dim, self.max_room_dim, self.min_gap, self.min_dist, self.min_angle_diff) if room_result is None: return self.__getitem__(idx + 1) # backoff room_dim, R_loc, source_loc, source_angles = room_result voices = [self.truncator.process(self.voices[vi]) for vi in np.random.choice(len(self.voices), n_source)] voices = [v * np.random.uniform(self.random_volume_range[0], self.random_volume_range[1]) for v in voices] if self.special_noise_ratio < np.random.rand(): noise = self.truncator.process(self.noises[np.random.choice(len(self.noises))]) else: noise = np.random.randn(self.truncator.get_length()) if self.randomize_material_ratio < np.random.rand(): ceiling, east, west, north, south = tuple(np.random.choice(wall_materials, 5)) # sample material floor = np.random.choice(floor_materials) # sample material mixed, premix_w_reverb, premix, R = simulateSound(room_dim, R_loc, source_loc, voices, 0, (ceiling, east, west, north, south, floor), self.max_order) else: rt60 = np.random.uniform(self.min_rt60, self.max_rt60) mixed, premix_w_reverb, premix, R = simulateSound(room_dim, R_loc, source_loc, voices, rt60) background = simulateBackground(noise) snr = np.random.uniform(self.additive_noise_min_snr, self.additive_noise_max_snr) # trucate to the same length mixed = mixed[:, :truncator.get_length()] background = background[:, :truncator.get_length()] total, background = mix(mixed, background, snr) # save cache if cache_path is not None: np.savez_compressed(cache_path, data=[total, premix_w_reverb, source_angles, premix, background, R]) if premix_w_reverb.shape[-1] <= 18000: offset = 18000 - premix_w_reverb.shape[-1] premix_w_reverb = np.pad(premix_w_reverb, ((0, 0), (0, 0), (0, offset)), "constant") else: premix_w_reverb = premix_w_reverb[..., :18000] return total, premix_w_reverb, source_angles, premix, background, R, R_loc
quad_flat(stream, (left, top, width, height), (s0,t0,s1,t1), color) offset += metrics["advance"] def patch9_cell_flat(stream, patch9, x, y, (left, top, width, height), color): x0,s0 = patch9.gridpoint(0, x+0, width) x1,s1 = patch9.gridpoint(0, x+1, width) y0,t0 = patch9.gridpoint(1, y+0, height) y1,t1 = patch9.gridpoint(1, y+1, height) quad_flat(stream, (left+x0, top+y0, x1-x0, y1-y0), (s0, t0, s1, t1), color) def patch9_cell_gradient(stream, patch9, x, y, (left, top, width, height), (c0,c1,c2,c3)): x0,s0 = patch9.gridpoint(0, x+0, width) x1,s1 = patch9.gridpoint(0, x+1, width) y0,t0 = patch9.gridpoint(1, y+0, height) y1,t1 = patch9.gridpoint(1, y+1, height) k0 = mix(c0, c1, float(x0)/width) k1 = mix(c0, c1, float(x1)/width) k2 = mix(c2, c3, float(x0)/width) k3 = mix(c2, c3, float(x1)/width) c0 = mix(k0, k2, float(y0)/height) c1 = mix(k1, k3, float(y0)/height) c2 = mix(k0, k2, float(y1)/height) c3 = mix(k1, k3, float(y1)/height) quad_gradient(stream, (left+x0, top+y0, x1-x0, y1-y0), (s0, t0, s1, t1), (c0,c1,c2,c3)) class Renderer(object): def __init__(self, output, default_font, program=None): self.output = output self.default_font = default_font self.stream = VertexStream(fmt) self.textures = TextureCache()