def make_super_session(bird_id,
sess_str,
depth='',
raw_location='rw',
ss_location='ss',
super_sess_name=None):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
sessions = glob.glob(
os.path.join(raw_data_folder, sess_str + '*' + str(depth)))
exp_files = list_experiment_files(bird_id,
sess_str,
depth=depth,
raw_location=raw_location,
file_type='data')
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
super_sess_par = sess_par.copy()
if super_sess_name is None:
super_sess_name = 'day-' + sess_str
if depth != '':
super_sess_name += '_{0}'.format(depth)
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders'][ss_location]
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
lfp_file_path = os.path.join(super_sess_path, fn['structure']['ss_lfp'])
module_logger.info("Making supersession {}".format(super_sess_name))
module_logger.info('super file path: {}'.format(super_file_path))
module_logger.info('Found {} experiment files'.format(len(exp_files)))
et.mkdir_p(super_sess_path)
make_super_file(super_file_path)
make_super_file(lfp_file_path)
with h5py.File(super_file_path,
'a') as super_file, h5py.File(lfp_file_path,
'a') as lfp_file:
for experiment_path in exp_files:
module_logger.info('Inserting file {0}'.format(experiment_path))
sess_fold = os.path.split(os.path.split(experiment_path)[0])[1]
sess_par = et.get_parameters(bird_id,
sess_fold,
location=raw_location)
kwd_chan_list, new_par_chan_config = new_channel_config(
sess_par['channel_config'])
with h5py.File(experiment_path, 'r') as raw_file:
insert_experiment_groups(super_file, raw_file, kwd_chan_list)
insert_experiment_groups(lfp_file,
raw_file,
kwd_chan_list,
stream_type='lfp')
super_file.flush()
lfp_file.flush()
super_sess_par['channel_config'] = new_par_chan_config.copy()
save_ss_par(super_sess_par,
bird_id,
super_sess_name,
ss_location=ss_location)
return sessions
def __init__(self, bird, sess, fit_pars):
logger.info('Init bird {0}, session {1}'.format(bird, sess))
self.bird = bird
self.sess = sess
self.fp = fit_pars
self.s_f = None
self.all_syl = None
self.X = None
self.X_t = None
self.Y = None
self.Y_t = None
self.Z = None
self.Z_t = None
# load metadatas and files
self.fn = et.file_names(self.bird, self.sess)
self.exp_par = et.get_parameters(bird, sess)
self.kwik_file = et.open_kwik(bird, sess)
self.kwd_file = et.open_kwd(bird, sess)
self.get_s_f()
self.load_syl()
self.load_units()
def process_recording_realtime(bird_id,
sess_day_id,
depth,
raw_location='raw',
ss_location='ss'):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
sessions = glob.glob(
os.path.join(raw_data_folder, sess_day_id + '*' + str(depth)))
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
data_processor = sess_par['rec_config']['processors']['data']
experiments = list(
list_flatten([
glob.glob(os.path.join(s,
'*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
]))
experiments.sort()
super_sess_name = 'day-' + sess_day_id + '_' + depth
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders'][ss_location]
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
module_logger.info('Super session path {}'.format(super_file_path))
sess_list = make_super_session(bird_id,
sess_day_id,
depth,
raw_location=raw_location,
ss_location=ss_location)
return sess_list
def list_experiment_files(bird_id,
sess_str,
depth=None,
raw_location='rw',
file_type='data'):
"""
List experiment files for a bird and a session base string (usually a day) and post string
(usually a depth)
:param bird_id: str, bird id
:param sess_str: str, base identifier for a group of sessions
:param depth: str, post_identifier
:param raw_location: what is the location of the raw data (for the file_names returned structure)
:return:
"""
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
str_depth = '' if depth is None else '_{}'.format(depth)
sessions = list(
glob.glob(os.path.join(raw_data_folder, sess_str + '*' + str_depth)))
module_logger.debug('Sessions: {}'.format(sessions))
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
processor = sess_par['rec_config']['processors'][file_type]
search_str = '*_{}.raw.kwd'.format(processor)
module_logger.info('searching {}'.format(search_str))
experiments = list(
list_flatten(
[glob.glob(os.path.join(s, search_str))[:] for s in sessions]))
experiments.sort()
return experiments
def search_pattern(bird_id, sess, rec, pattern_chunk):
logger.info('Configuring search for rec {}'.format(rec))
fn = et.file_names(bird_id, sess, rec)
mic_file_path = et.file_path(fn, 'ss', 'mic')
logger.info('Loading the data from rec {}'.format(rec))
chan_sound = st.WavData2(mic_file_path)
chan_chunk = st.Chunk(chan_sound)
logger.info('{} samples loaded'.format(chan_sound.n_samples))
logger.info('Calling find_the_bouts')
exp_par = et.get_parameters(bird_id, sess, 0)
search_pars = exp_par['search_motiff']
search_pars['s_f'] = chan_sound.s_f
cand_file_path_orig = et.file_path(fn, 'ss', 'cand')
cand_file_parts = os.path.split(cand_file_path_orig)
cand_file_path = os.path.join(cand_file_parts[0], cand_file_parts[1] + '.new')
cand_grp = '/pattern_{0}/{1:03d}'.format(pattern, rec)
candidates = fb.find_the_bouts(chan_chunk.data.flatten(), pattern_chunk.data.flatten(),
search_pars,
cand_file_path=cand_file_path,
cand_grp=cand_grp)
logger.info('Found {} candidates'.format(candidates.index.size))
return candidates
def get_unit_chans(self):
# load parameters
sess_par = et.get_parameters(self.bird, self.sess, location='ss')
# get the chan list fo this shank
try:
sess_chans = sess_par['channel_config']['neural_{}'.format(
self.group)]
except KeyError:
if self.group == 0:
sess_chans = sess_par['channel_config']['neural']
else:
raise
return sess_chans
def process_awake_recording(bird_id,
sess_day_id,
depth,
cond='',
raw_location='raw'):
raw_data_folder = et.file_names(bird_id)['folders'][raw_location]
cond_string = cond if cond in ['', '*'] else '{}_'.format(cond)
sess_day_str = cond_string + sess_day_id
sessions = glob.glob(
os.path.join(raw_data_folder, sess_day_str + '*' + str(depth)))
sessions.sort()
print(sessions)
sess_par = et.get_parameters(bird_id,
os.path.split(sessions[0])[-1],
location=raw_location)
data_processor = sess_par['rec_config']['processors']['data']
print([
glob.glob(os.path.join(s, '*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
])
experiments = list(
list_flatten([
glob.glob(os.path.join(s,
'*_{}.raw.kwd'.format(data_processor)))[:]
for s in sessions
]))
experiments.sort()
super_sess_name = ('day-' + sess_day_str + '_' + depth).replace(
'*', 'all_')
fn = et.file_names(bird_id, super_sess_name, 0)
super_sess_path = fn['folders']['ss']
super_file_path = os.path.join(super_sess_path, fn['structure']['ss_raw'])
module_logger.info('Super session path {}'.format(super_file_path))
sess_list = make_super_session(bird_id,
sess_day_str,
depth,
raw_location=raw_location,
super_sess_name=super_sess_name)
make_raw_bkp(bird_id, sess_list, raw_location=raw_location)
extract_wav_chans(bird_id, super_sess_name)
module_logger.info('Done making supersession')
return super_sess_name
def make_binary(bird, sess, port=0):
print('Making binary')
fn = et.file_names(bird, sess)
params = et.get_parameters(bird, sess)
kwd_path = et.file_path(fn, 'ss', 'ss_raw')
bin_path = et.file_path(fn, 'ss', 'ss_bin')
neural_key = 'neural_{}'.format(port)
logger.info('port is ' + neural_key)
try:
chan_list = params['channel_config'][neural_key]
except KeyError:
logger.info('not found chan group ' + neural_key)
if port == 0:
logger.info('Using default (neural) chan list')
chan_list = params['channel_config']['neural']
logger.info('chan_list: {}'.format(chan_list))
else:
raise
h5f.kwd_to_binary(kwd_path, bin_path, chan_list=chan_list)
print('done')
def make_moutainsort_config(bird, sess,
detect_sign=-1,
adjacency_radius=-1,
):
fn = et.file_names(bird, sess)
exp_par = et.get_parameters(bird, sess) # load the yml parameter file
sort_dir = fn['folders']['ss']
logger.debug('local sort dir: {}'.format(local_sort_dir))
s_f = h5f.get_record_sampling_frequency(et.open_kwd(bird, sess))
params = {
'detect_sign': detect_sign,
'samplerate': s_f,
}
logger.debug(params)
et.mkdir_p(sort_dir)
with open(os.path.join(dest, 'params.json'), 'w') as output:
json.dump(output_args, output)
def extract_wav_chans(bird_id, sess, ch_name='mic', location='ss'):
fn = et.file_names(bird_id, sess, 0)
super_file_path = os.path.join(fn['folders'][location],
fn['structure']['ss_raw'])
exp_par = et.get_parameters(bird_id, sess)
super_file = h5py.File(super_file_path, 'r')
rec_list = super_file['/recordings'].keys()
module_logger.info('Extract {0} chan to wav for recs {1}'.format(
ch_name, rec_list))
for rec, rec_grp in super_file['/recordings'].items():
fn = et.file_names(bird_id, sess, int(rec))
data_set = rec_grp['data']
s_f = rec_grp.attrs['sample_rate']
wav_file_path = os.path.join(fn['folders']['ss'],
fn['structure'][ch_name])
module_logger.info('Rec {0}: {1}'.format(rec, wav_file_path))
export_audio(data_set,
exp_par['channel_config'][ch_name],
wav_file_path,
filter_func=filter_bp,
args=band_pass_filter_pars(s_f, exp_par))
module_logger.info('Done extracting channels')
def get_exp_pars(self):
file_parts = ff.split_path(self.h5_file.filename)
self.sess_id = file_parts[-2]
self.bird_id = file_parts[-3]
self.exp_pars = et.get_parameters(self.bird_id, self.sess_id)