def make_super_session(bird_id, sess_str, depth='', raw_location='rw', ss_location='ss', super_sess_name=None): raw_data_folder = et.file_names(bird_id)['folders'][raw_location] sessions = glob.glob( os.path.join(raw_data_folder, sess_str + '*' + str(depth))) exp_files = list_experiment_files(bird_id, sess_str, depth=depth, raw_location=raw_location, file_type='data') sess_par = et.get_parameters(bird_id, os.path.split(sessions[0])[-1], location=raw_location) super_sess_par = sess_par.copy() if super_sess_name is None: super_sess_name = 'day-' + sess_str if depth != '': super_sess_name += '_{0}'.format(depth) fn = et.file_names(bird_id, super_sess_name, 0) super_sess_path = fn['folders'][ss_location] super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw']) lfp_file_path = os.path.join(super_sess_path, fn['structure']['ss_lfp']) module_logger.info("Making supersession {}".format(super_sess_name)) module_logger.info('super file path: {}'.format(super_file_path)) module_logger.info('Found {} experiment files'.format(len(exp_files))) et.mkdir_p(super_sess_path) make_super_file(super_file_path) make_super_file(lfp_file_path) with h5py.File(super_file_path, 'a') as super_file, h5py.File(lfp_file_path, 'a') as lfp_file: for experiment_path in exp_files: module_logger.info('Inserting file {0}'.format(experiment_path)) sess_fold = os.path.split(os.path.split(experiment_path)[0])[1] sess_par = et.get_parameters(bird_id, sess_fold, location=raw_location) kwd_chan_list, new_par_chan_config = new_channel_config( sess_par['channel_config']) with h5py.File(experiment_path, 'r') as raw_file: insert_experiment_groups(super_file, raw_file, kwd_chan_list) insert_experiment_groups(lfp_file, raw_file, kwd_chan_list, stream_type='lfp') super_file.flush() lfp_file.flush() super_sess_par['channel_config'] = new_par_chan_config.copy() save_ss_par(super_sess_par, bird_id, super_sess_name, ss_location=ss_location) return sessions
def process_recording_realtime(bird_id, sess_day_id, depth, raw_location='raw', ss_location='ss'): raw_data_folder = et.file_names(bird_id)['folders'][raw_location] sessions = glob.glob( os.path.join(raw_data_folder, sess_day_id + '*' + str(depth))) sess_par = et.get_parameters(bird_id, os.path.split(sessions[0])[-1], location=raw_location) data_processor = sess_par['rec_config']['processors']['data'] experiments = list( list_flatten([ glob.glob(os.path.join(s, '*_{}.raw.kwd'.format(data_processor)))[:] for s in sessions ])) experiments.sort() super_sess_name = 'day-' + sess_day_id + '_' + depth fn = et.file_names(bird_id, super_sess_name, 0) super_sess_path = fn['folders'][ss_location] super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw']) module_logger.info('Super session path {}'.format(super_file_path)) sess_list = make_super_session(bird_id, sess_day_id, depth, raw_location=raw_location, ss_location=ss_location) return sess_list
def find_song(bird_id, sess_day, raw_location='rw'): module_logger.info('Set to find song for bird {}, sess {}'.format( bird_id, sess_day)) # Parameters of the search bit_size = 50 # ms; size of sound bits refractory = 5 # bits; refractory period in bits bits_bout = 2 # bits; how many bouts together is considered a bout threshold = 20 # threshold in mads (median deviation of the median) # The band where we expect most of the energy of zf song to be in filt_lo = 10000 # Hz filt_hi = 300 # Hz raw_data_folder_bird = et.file_names(bird_id)['folders'][raw_location] ss_data_folder_bird = et.file_names(bird_id)['folders']['ss'] et.mkdir_p(ss_data_folder_bird) raw_file_list = get_bird_files(raw_data_folder_bird, only_days=[sess_day]) all_raw_file_list = [x for x in raw_file_list if 'auto' not in x] all_raw_file_list.sort() raw_file_path = all_raw_file_list[-1] raw_path, raw_fname = os.path.split(raw_file_path) #raw_path.replace('raw_data', 'ss_data') # create file handler which logs even debug messages log_f_name = os.path.join(ss_data_folder_bird, 'search_song_{}.log'.format(sess_day)) module_logger.info('Saving log to {}'.format(log_f_name)) fh = logging.FileHandler(log_f_name) fh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) # add the handlers to the logger module_logger.addHandler(fh) found_songs = [] for raw_file_path in all_raw_file_list: raw_path, raw_fname = os.path.split(raw_file_path) ss_path = raw_path.replace('raw_data', 'ss_data') et.mkdir_p(ss_path) bout_file_path = os.path.join( ss_path, '{}_auto.wav'.format(raw_fname.split('.')[0])) found = findsong.get_all_bouts(raw_file_path, bout_file_path=bout_file_path, bit_size=bit_size, refractory=refractory, bits_bout=bits_bout, threshold=threshold, filt_lo=filt_lo, filt_hi=filt_hi) found_songs.append(found) module_logger.info('done with the session {} : {}'.format( bird_id, sess_day)) return bird_id, sess_day
def __init__(self, bird, sess, fit_pars): logger.info('Init bird {0}, session {1}'.format(bird, sess)) self.bird = bird self.sess = sess self.fp = fit_pars self.s_f = None self.all_syl = None self.X = None self.X_t = None self.Y = None self.Y_t = None self.Z = None self.Z_t = None # load metadatas and files self.fn = et.file_names(self.bird, self.sess) self.exp_par = et.get_parameters(bird, sess) self.kwik_file = et.open_kwik(bird, sess) self.kwd_file = et.open_kwd(bird, sess) self.get_s_f() self.load_syl() self.load_units()
def list_experiment_files(bird_id, sess_str, depth=None, raw_location='rw', file_type='data'): """ List experiment files for a bird and a session base string (usually a day) and post string (usually a depth) :param bird_id: str, bird id :param sess_str: str, base identifier for a group of sessions :param depth: str, post_identifier :param raw_location: what is the location of the raw data (for the file_names returned structure) :return: """ raw_data_folder = et.file_names(bird_id)['folders'][raw_location] str_depth = '' if depth is None else '_{}'.format(depth) sessions = list( glob.glob(os.path.join(raw_data_folder, sess_str + '*' + str_depth))) module_logger.debug('Sessions: {}'.format(sessions)) sess_par = et.get_parameters(bird_id, os.path.split(sessions[0])[-1], location=raw_location) processor = sess_par['rec_config']['processors'][file_type] search_str = '*_{}.raw.kwd'.format(processor) module_logger.info('searching {}'.format(search_str)) experiments = list( list_flatten( [glob.glob(os.path.join(s, search_str))[:] for s in sessions])) experiments.sort() return experiments
def __init__(self, parent=None): super(BoutSearch, self).__init__(parent) self.setupUi(self) self.bird = None self.sess = None self.file = None self.s_f = None self.line_in_plot_span.setText(str(10000)) self.fn = et.file_names('') selection_functions = { 'select_bird': self.bird_change, 'select_sess': self.sess_change, 'select_file': self.file_change } for select_str in ['select_bird', 'select_sess', 'select_file']: q_box = getattr(self, select_str) q_box.currentIndexChanged.connect(selection_functions[select_str]) self.update_birds_list() # Init plot widgets connections self.plot_bar.sliderMoved.connect(self.update_pos_slider) self.line_in_plot_start.returnPressed.connect(self.update_pos_text) self.line_in_plot_span.returnPressed.connect(self.update_plot_span)
def search_pattern(bird_id, sess, rec, pattern_chunk): logger.info('Configuring search for rec {}'.format(rec)) fn = et.file_names(bird_id, sess, rec) mic_file_path = et.file_path(fn, 'ss', 'mic') logger.info('Loading the data from rec {}'.format(rec)) chan_sound = st.WavData2(mic_file_path) chan_chunk = st.Chunk(chan_sound) logger.info('{} samples loaded'.format(chan_sound.n_samples)) logger.info('Calling find_the_bouts') exp_par = et.get_parameters(bird_id, sess, 0) search_pars = exp_par['search_motiff'] search_pars['s_f'] = chan_sound.s_f cand_file_path_orig = et.file_path(fn, 'ss', 'cand') cand_file_parts = os.path.split(cand_file_path_orig) cand_file_path = os.path.join(cand_file_parts[0], cand_file_parts[1] + '.new') cand_grp = '/pattern_{0}/{1:03d}'.format(pattern, rec) candidates = fb.find_the_bouts(chan_chunk.data.flatten(), pattern_chunk.data.flatten(), search_pars, cand_file_path=cand_file_path, cand_grp=cand_grp) logger.info('Found {} candidates'.format(candidates.index.size)) return candidates
def process_awake_recording(bird_id, sess_day_id, depth, cond='', raw_location='raw'): raw_data_folder = et.file_names(bird_id)['folders'][raw_location] cond_string = cond if cond in ['', '*'] else '{}_'.format(cond) sess_day_str = cond_string + sess_day_id sessions = glob.glob( os.path.join(raw_data_folder, sess_day_str + '*' + str(depth))) sessions.sort() print(sessions) sess_par = et.get_parameters(bird_id, os.path.split(sessions[0])[-1], location=raw_location) data_processor = sess_par['rec_config']['processors']['data'] print([ glob.glob(os.path.join(s, '*_{}.raw.kwd'.format(data_processor)))[:] for s in sessions ]) experiments = list( list_flatten([ glob.glob(os.path.join(s, '*_{}.raw.kwd'.format(data_processor)))[:] for s in sessions ])) experiments.sort() super_sess_name = ('day-' + sess_day_str + '_' + depth).replace( '*', 'all_') fn = et.file_names(bird_id, super_sess_name, 0) super_sess_path = fn['folders']['ss'] super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw']) module_logger.info('Super session path {}'.format(super_file_path)) sess_list = make_super_session(bird_id, sess_day_str, depth, raw_location=raw_location, super_sess_name=super_sess_name) make_raw_bkp(bird_id, sess_list, raw_location=raw_location) extract_wav_chans(bird_id, super_sess_name) module_logger.info('Done making supersession') return super_sess_name
def extract_wav_chans(bird_id, sess, ch_name='mic', location='ss'): fn = et.file_names(bird_id, sess, 0) super_file_path = os.path.join(fn['folders'][location], fn['structure']['ss_raw']) exp_par = et.get_parameters(bird_id, sess) super_file = h5py.File(super_file_path, 'r') rec_list = super_file['/recordings'].keys() module_logger.info('Extract {0} chan to wav for recs {1}'.format( ch_name, rec_list)) for rec, rec_grp in super_file['/recordings'].items(): fn = et.file_names(bird_id, sess, int(rec)) data_set = rec_grp['data'] s_f = rec_grp.attrs['sample_rate'] wav_file_path = os.path.join(fn['folders']['ss'], fn['structure'][ch_name]) module_logger.info('Rec {0}: {1}'.format(rec, wav_file_path)) export_audio(data_set, exp_par['channel_config'][ch_name], wav_file_path, filter_func=filter_bp, args=band_pass_filter_pars(s_f, exp_par)) module_logger.info('Done extracting channels')
def kilo_to_kwik(bird, sess, file_names=None, location='ss', chan_group=0): module_logger.info('Creating kwik file for bird: {} sess: {}'.format( bird, sess)) if file_names is None: file_names = dict(clu='spike_clusters.npy', spk='spike_times.npy', grp='cluster_groups.csv', par='params.py', temp='spike_templates.npy', kwd='experiment.raw.kwd', kwk='experiment.kwik') fn = et.file_names(bird, sess) for key, value in file_names.items(): file_names[key] = os.path.join(fn['folders'][location], value) # Check whether there is manual sort or not: if not os.path.isfile(file_names['clu']): module_logger.info( 'Clu not found, will assume no manual sorting was done') file_names['clu'] = None file_names['grp'] = None module_logger.debug(file_names) else: module_logger.info( 'Found clu file, will attempt to unpack manual sorted data from kilosort' ) file_names['temp'] = None module_logger.debug(file_names) k = KwikFile(file_names, chan_group=chan_group) module_logger.info('Making spike tables') k.make_spk_tables() module_logger.info('Making rec tables (make_rec_groups)') k.make_rec_groups() module_logger.info('Making cluster group tables') k.make_clu_groups() module_logger.info('Moving files to their sort folder') sort_kilo_dir = os.path.join(fn['folders'][location], 'kilo_{:02d}'.format(chan_group)) et.mkdir_p(sort_kilo_dir) py_files = glob.glob(os.path.join(fn['folders'][location], '*.py')) npy_files = glob.glob(os.path.join(fn['folders'][location], '*.npy')) for src in py_files + npy_files: shutil.move(src, sort_kilo_dir) module_logger.info('Removing temporary .dat file') dat_file = os.path.join(fn['folders'][location], 'experiment.dat') os.remove(dat_file) module_logger.info('Done')
def make_binary(bird, sess, port=0): print('Making binary') fn = et.file_names(bird, sess) params = et.get_parameters(bird, sess) kwd_path = et.file_path(fn, 'ss', 'ss_raw') bin_path = et.file_path(fn, 'ss', 'ss_bin') neural_key = 'neural_{}'.format(port) logger.info('port is ' + neural_key) try: chan_list = params['channel_config'][neural_key] except KeyError: logger.info('not found chan group ' + neural_key) if port == 0: logger.info('Using default (neural) chan list') chan_list = params['channel_config']['neural'] logger.info('chan_list: {}'.format(chan_list)) else: raise h5f.kwd_to_binary(kwd_path, bin_path, chan_list=chan_list) print('done')
def make_moutainsort_config(bird, sess, detect_sign=-1, adjacency_radius=-1, ): fn = et.file_names(bird, sess) exp_par = et.get_parameters(bird, sess) # load the yml parameter file sort_dir = fn['folders']['ss'] logger.debug('local sort dir: {}'.format(local_sort_dir)) s_f = h5f.get_record_sampling_frequency(et.open_kwd(bird, sess)) params = { 'detect_sign': detect_sign, 'samplerate': s_f, } logger.debug(params) et.mkdir_p(sort_dir) with open(os.path.join(dest, 'params.json'), 'w') as output: json.dump(output_args, output)
def save_syn_streams(streams, bird, syn_sess=1, s_f=30000, new_s_f=30000, file_base='synth_bos'): stim_folder = et.file_names(bird)['folders']['stim'] syn_file = os.path.join(stim_folder, str(syn_sess).zfill(3), file_base + '.dat') np.savetxt(stream_resample(streams, s_f, new_s_f))
def load_syn_stream(bird, syn_sess=1, s_f=44100, new_s_f=30000, file_base='synth_bos'): stim_folder = et.file_names(bird)['folders']['stim'] syn_file = os.path.join(stim_folder, str(syn_sess).zfill(3), file_base + '.dat') return stream_resample(np.loadtxt(syn_file), s_f, new_s_f)
def save_ss_par(par, bird, sess, ss_location='ss'): fn = et.file_names(bird, sess) par_path = et.file_path(fn, ss_location, 'par') with open(par_path, 'w') as f: written = yaml.dump(par, f) return written
def update_files_list(self): self.fn = et.file_names(self.bird, self.sess) bout_folder = self.fn['folders']['rw'] all_files = glob.glob(os.path.join(bout_folder, '*.wav')) all_files.sort() self.select_file.addItems(all_files)
def update_sess_list(self): self.fn = et.file_names(self.bird) sess_list = et.list_raw_sessions(self.bird, location='rw')[0] sess_list.sort() self.select_sess.addItems(sess_list)