def make_super_session(bird_id,
sess_str,
depth='',
raw_location='rw',
ss_location='ss',
super_sess_name=None):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
sessions = glob.glob(
os.path.join(raw_data_folder, sess_str + '*' + str(depth)))
exp_files = list_experiment_files(bird_id,
sess_str,
depth=depth,
raw_location=raw_location,
file_type='data')
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
super_sess_par = sess_par.copy()
if super_sess_name is None:
super_sess_name = 'day-' + sess_str
if depth != '':
super_sess_name += '_{0}'.format(depth)
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders'][ss_location]
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
lfp_file_path = os.path.join(super_sess_path, fn['structure']['ss_lfp'])
module_logger.info("Making supersession {}".format(super_sess_name))
module_logger.info('super file path: {}'.format(super_file_path))
module_logger.info('Found {} experiment files'.format(len(exp_files)))
et.mkdir_p(super_sess_path)
make_super_file(super_file_path)
make_super_file(lfp_file_path)
with h5py.File(super_file_path,
'a') as super_file, h5py.File(lfp_file_path,
'a') as lfp_file:
for experiment_path in exp_files:
module_logger.info('Inserting file {0}'.format(experiment_path))
sess_fold = os.path.split(os.path.split(experiment_path)[0])[1]
sess_par = et.get_parameters(bird_id,
sess_fold,
location=raw_location)
kwd_chan_list, new_par_chan_config = new_channel_config(
sess_par['channel_config'])
with h5py.File(experiment_path, 'r') as raw_file:
insert_experiment_groups(super_file, raw_file, kwd_chan_list)
insert_experiment_groups(lfp_file,
raw_file,
kwd_chan_list,
stream_type='lfp')
super_file.flush()
lfp_file.flush()
super_sess_par['channel_config'] = new_par_chan_config.copy()
save_ss_par(super_sess_par,
bird_id,
super_sess_name,
ss_location=ss_location)
return sessions
def process_recording_realtime(bird_id,
sess_day_id,
depth,
raw_location='raw',
ss_location='ss'):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
sessions = glob.glob(
os.path.join(raw_data_folder, sess_day_id + '*' + str(depth)))
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
data_processor = sess_par['rec_config']['processors']['data']
experiments = list(
list_flatten([
glob.glob(os.path.join(s,
'*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
]))
experiments.sort()
super_sess_name = 'day-' + sess_day_id + '_' + depth
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders'][ss_location]
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
module_logger.info('Super session path {}'.format(super_file_path))
sess_list = make_super_session(bird_id,
sess_day_id,
depth,
raw_location=raw_location,
ss_location=ss_location)
return sess_list
def find_song(bird_id, sess_day, raw_location='rw'):
module_logger.info('Set to find song for bird {}, sess {}'.format(
bird_id, sess_day))
# Parameters of the search
bit_size = 50 # ms; size of sound bits
refractory = 5 # bits; refractory period in bits
bits_bout = 2 # bits; how many bouts together is considered a bout
threshold = 20 # threshold in mads (median deviation of the median)
# The band where we expect most of the energy of zf song to be in
filt_lo = 10000 # Hz
filt_hi = 300 # Hz
raw_data_folder_bird = et.file_names(bird_id)['folders'][raw_location]
ss_data_folder_bird = et.file_names(bird_id)['folders']['ss']
et.mkdir_p(ss_data_folder_bird)
raw_file_list = get_bird_files(raw_data_folder_bird, only_days=[sess_day])
all_raw_file_list = [x for x in raw_file_list if 'auto' not in x]
all_raw_file_list.sort()
raw_file_path = all_raw_file_list[-1]
raw_path, raw_fname = os.path.split(raw_file_path)
#raw_path.replace('raw_data', 'ss_data')
# create file handler which logs even debug messages
log_f_name = os.path.join(ss_data_folder_bird,
'search_song_{}.log'.format(sess_day))
module_logger.info('Saving log to {}'.format(log_f_name))
fh = logging.FileHandler(log_f_name)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
# add the handlers to the logger
module_logger.addHandler(fh)
found_songs = []
for raw_file_path in all_raw_file_list:
raw_path, raw_fname = os.path.split(raw_file_path)
ss_path = raw_path.replace('raw_data', 'ss_data')
et.mkdir_p(ss_path)
bout_file_path = os.path.join(
ss_path, '{}_auto.wav'.format(raw_fname.split('.')[0]))
found = findsong.get_all_bouts(raw_file_path,
bout_file_path=bout_file_path,
bit_size=bit_size,
refractory=refractory,
bits_bout=bits_bout,
threshold=threshold,
filt_lo=filt_lo,
filt_hi=filt_hi)
found_songs.append(found)
module_logger.info('done with the session {} : {}'.format(
bird_id, sess_day))
return bird_id, sess_day
def __init__(self, bird, sess, fit_pars):
logger.info('Init bird {0}, session {1}'.format(bird, sess))
self.bird = bird
self.sess = sess
self.fp = fit_pars
self.s_f = None
self.all_syl = None
self.X = None
self.X_t = None
self.Y = None
self.Y_t = None
self.Z = None
self.Z_t = None
# load metadatas and files
self.fn = et.file_names(self.bird, self.sess)
self.exp_par = et.get_parameters(bird, sess)
self.kwik_file = et.open_kwik(bird, sess)
self.kwd_file = et.open_kwd(bird, sess)
self.get_s_f()
self.load_syl()
self.load_units()
def list_experiment_files(bird_id,
sess_str,
depth=None,
raw_location='rw',
file_type='data'):
"""
List experiment files for a bird and a session base string (usually a day) and post string
(usually a depth)
:param bird_id: str, bird id
:param sess_str: str, base identifier for a group of sessions
:param depth: str, post_identifier
:param raw_location: what is the location of the raw data (for the file_names returned structure)
:return:
"""
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
str_depth = '' if depth is None else '_{}'.format(depth)
sessions = list(
glob.glob(os.path.join(raw_data_folder, sess_str + '*' + str_depth)))
module_logger.debug('Sessions: {}'.format(sessions))
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
processor = sess_par['rec_config']['processors'][file_type]
search_str = '*_{}.raw.kwd'.format(processor)
module_logger.info('searching {}'.format(search_str))
experiments = list(
list_flatten(
[glob.glob(os.path.join(s, search_str))[:] for s in sessions]))
experiments.sort()
return experiments
def __init__(self, parent=None):
super(BoutSearch, self).__init__(parent)
self.setupUi(self)
self.bird = None
self.sess = None
self.file = None
self.s_f = None
self.line_in_plot_span.setText(str(10000))
self.fn = et.file_names('')
selection_functions = {
'select_bird': self.bird_change,
'select_sess': self.sess_change,
'select_file': self.file_change
}
for select_str in ['select_bird', 'select_sess', 'select_file']:
q_box = getattr(self, select_str)
q_box.currentIndexChanged.connect(selection_functions[select_str])
self.update_birds_list()
# Init plot widgets connections
self.plot_bar.sliderMoved.connect(self.update_pos_slider)
self.line_in_plot_start.returnPressed.connect(self.update_pos_text)
self.line_in_plot_span.returnPressed.connect(self.update_plot_span)
def search_pattern(bird_id, sess, rec, pattern_chunk):
logger.info('Configuring search for rec {}'.format(rec))
fn = et.file_names(bird_id, sess, rec)
mic_file_path = et.file_path(fn, 'ss', 'mic')
logger.info('Loading the data from rec {}'.format(rec))
chan_sound = st.WavData2(mic_file_path)
chan_chunk = st.Chunk(chan_sound)
logger.info('{} samples loaded'.format(chan_sound.n_samples))
logger.info('Calling find_the_bouts')
exp_par = et.get_parameters(bird_id, sess, 0)
search_pars = exp_par['search_motiff']
search_pars['s_f'] = chan_sound.s_f
cand_file_path_orig = et.file_path(fn, 'ss', 'cand')
cand_file_parts = os.path.split(cand_file_path_orig)
cand_file_path = os.path.join(cand_file_parts[0], cand_file_parts[1] + '.new')
cand_grp = '/pattern_{0}/{1:03d}'.format(pattern, rec)
candidates = fb.find_the_bouts(chan_chunk.data.flatten(), pattern_chunk.data.flatten(),
search_pars,
cand_file_path=cand_file_path,
cand_grp=cand_grp)
logger.info('Found {} candidates'.format(candidates.index.size))
return candidates
def process_awake_recording(bird_id,
sess_day_id,
depth,
cond='',
raw_location='raw'):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
cond_string = cond if cond in ['', '*'] else '{}_'.format(cond)
sess_day_str = cond_string + sess_day_id
sessions = glob.glob(
os.path.join(raw_data_folder, sess_day_str + '*' + str(depth)))
sessions.sort()
print(sessions)
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
data_processor = sess_par['rec_config']['processors']['data']
print([
glob.glob(os.path.join(s, '*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
])
experiments = list(
list_flatten([
glob.glob(os.path.join(s,
'*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
]))
experiments.sort()
super_sess_name = ('day-' + sess_day_str + '_' + depth).replace(
'*', 'all_')
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders']['ss']
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
module_logger.info('Super session path {}'.format(super_file_path))
sess_list = make_super_session(bird_id,
sess_day_str,
depth,
raw_location=raw_location,
super_sess_name=super_sess_name)
make_raw_bkp(bird_id, sess_list, raw_location=raw_location)
extract_wav_chans(bird_id, super_sess_name)
module_logger.info('Done making supersession')
return super_sess_name
def extract_wav_chans(bird_id, sess, ch_name='mic', location='ss'):
fn = et.file_names(bird_id, sess, 0)
super_file_path = os.path.join(fn['folders'][location],
fn['structure']['ss_raw'])
exp_par = et.get_parameters(bird_id, sess)
super_file = h5py.File(super_file_path, 'r')
rec_list = super_file['/recordings'].keys()
module_logger.info('Extract {0} chan to wav for recs {1}'.format(
ch_name, rec_list))
for rec, rec_grp in super_file['/recordings'].items():
fn = et.file_names(bird_id, sess, int(rec))
data_set = rec_grp['data']
s_f = rec_grp.attrs['sample_rate']
wav_file_path = os.path.join(fn['folders']['ss'],
fn['structure'][ch_name])
module_logger.info('Rec {0}: {1}'.format(rec, wav_file_path))
export_audio(data_set,
exp_par['channel_config'][ch_name],
wav_file_path,
filter_func=filter_bp,
args=band_pass_filter_pars(s_f, exp_par))
module_logger.info('Done extracting channels')
def kilo_to_kwik(bird, sess, file_names=None, location='ss', chan_group=0):
module_logger.info('Creating kwik file for bird: {} sess: {}'.format(
bird, sess))
if file_names is None:
file_names = dict(clu='spike_clusters.npy',
spk='spike_times.npy',
grp='cluster_groups.csv',
par='params.py',
temp='spike_templates.npy',
kwd='experiment.raw.kwd',
kwk='experiment.kwik')
fn = et.file_names(bird, sess)
for key, value in file_names.items():
file_names[key] = os.path.join(fn['folders'][location], value)
# Check whether there is manual sort or not:
if not os.path.isfile(file_names['clu']):
module_logger.info(
'Clu not found, will assume no manual sorting was done')
file_names['clu'] = None
file_names['grp'] = None
module_logger.debug(file_names)
else:
module_logger.info(
'Found clu file, will attempt to unpack manual sorted data from kilosort'
)
file_names['temp'] = None
module_logger.debug(file_names)
k = KwikFile(file_names, chan_group=chan_group)
module_logger.info('Making spike tables')
k.make_spk_tables()
module_logger.info('Making rec tables (make_rec_groups)')
k.make_rec_groups()
module_logger.info('Making cluster group tables')
k.make_clu_groups()
module_logger.info('Moving files to their sort folder')
sort_kilo_dir = os.path.join(fn['folders'][location],
'kilo_{:02d}'.format(chan_group))
et.mkdir_p(sort_kilo_dir)
py_files = glob.glob(os.path.join(fn['folders'][location], '*.py'))
npy_files = glob.glob(os.path.join(fn['folders'][location], '*.npy'))
for src in py_files + npy_files:
shutil.move(src, sort_kilo_dir)
module_logger.info('Removing temporary .dat file')
dat_file = os.path.join(fn['folders'][location], 'experiment.dat')
os.remove(dat_file)
module_logger.info('Done')
def make_binary(bird, sess, port=0):
print('Making binary')
fn = et.file_names(bird, sess)
params = et.get_parameters(bird, sess)
kwd_path = et.file_path(fn, 'ss', 'ss_raw')
bin_path = et.file_path(fn, 'ss', 'ss_bin')
neural_key = 'neural_{}'.format(port)
logger.info('port is ' + neural_key)
try:
chan_list = params['channel_config'][neural_key]
except KeyError:
logger.info('not found chan group ' + neural_key)
if port == 0:
logger.info('Using default (neural) chan list')
chan_list = params['channel_config']['neural']
logger.info('chan_list: {}'.format(chan_list))
else:
raise
h5f.kwd_to_binary(kwd_path, bin_path, chan_list=chan_list)
print('done')
def make_moutainsort_config(bird, sess,
detect_sign=-1,
adjacency_radius=-1,
):
fn = et.file_names(bird, sess)
exp_par = et.get_parameters(bird, sess) # load the yml parameter file
sort_dir = fn['folders']['ss']
logger.debug('local sort dir: {}'.format(local_sort_dir))
s_f = h5f.get_record_sampling_frequency(et.open_kwd(bird, sess))
params = {
'detect_sign': detect_sign,
'samplerate': s_f,
}
logger.debug(params)
et.mkdir_p(sort_dir)
with open(os.path.join(dest, 'params.json'), 'w') as output:
json.dump(output_args, output)
def save_syn_streams(streams, bird, syn_sess=1, s_f=30000, new_s_f=30000, file_base='synth_bos'):
stim_folder = et.file_names(bird)['folders']['stim']
syn_file = os.path.join(stim_folder, str(syn_sess).zfill(3), file_base + '.dat')
np.savetxt(stream_resample(streams, s_f, new_s_f))
def load_syn_stream(bird, syn_sess=1, s_f=44100, new_s_f=30000, file_base='synth_bos'):
stim_folder = et.file_names(bird)['folders']['stim']
syn_file = os.path.join(stim_folder, str(syn_sess).zfill(3), file_base + '.dat')
return stream_resample(np.loadtxt(syn_file), s_f, new_s_f)
def save_ss_par(par, bird, sess, ss_location='ss'):
fn = et.file_names(bird, sess)
par_path = et.file_path(fn, ss_location, 'par')
with open(par_path, 'w') as f:
written = yaml.dump(par, f)
return written
def update_files_list(self):
self.fn = et.file_names(self.bird, self.sess)
bout_folder = self.fn['folders']['rw']
all_files = glob.glob(os.path.join(bout_folder, '*.wav'))
all_files.sort()
self.select_file.addItems(all_files)
def update_sess_list(self):
self.fn = et.file_names(self.bird)
sess_list = et.list_raw_sessions(self.bird, location='rw')[0]
sess_list.sort()
self.select_sess.addItems(sess_list)