def copy_data(bird, sess, orig, dest, only_files=[], exclude_files=[]):
fn = et.file_names(bird, sess)
source_folder = fn['folders'][orig]
dest_folder = fn['folders'][dest]
et.mkdir_p(dest_folder)
if len(only_files) > 0:
logger.debug('Grabbing all files in list {}'.format(only_files))
try:
copied = map(
lambda f: sh.copy2(et.file_path(fn, orig, f), dest_folder),
only_files)
except:
logger.error('Missing files')
raise ValueError('Missing kilosort data', 'missingFiles')
else:
copied = [
sh.copy2(et.file_path(fn, orig, f), dest_folder)
for f in fn['files'] if not f in exclude_files
and os.path.isfile(et.file_path(fn, orig, f))
]
return copied
def run_kilosort(bird, sess, no_copy=False):
print "will run bci_pipeline on bird {0} - session {1}".format(bird, sess)
fn = et.file_names(bird, sess)
log_file = os.path.join(fn['folders']['ss'], 'kilosort_py.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh = logging.FileHandler(log_file)
fh.setFormatter(formatter)
logger.info('Saving output in log file: {}'.format(log_file))
logger.addHandler(fh)
logger.info("will run bci_pipeline on bird {0} - session {1}".format(
bird, sess))
if no_copy:
logger.info('Will not pull data from cube')
else:
copyed = fetch_kilo_data(bird, sess)
logger.info('Copied {}'.format(copyed))
logger.info('Will create the scripts')
make_kilo_scripts(bird, sess)
logger.info('Will do the chanMap for matlab')
make_kilo_chanmap(bird, sess)
logger.info('Will do the sort')
sort_out = do_the_sort(bird, sess)
logger.info('Sorted {}'.format(sort_out))
pushed = push_kilo_data(bird, sess)
logger.info('Pushed back to cube {}'.format(pushed))
print 'Finished '
return
def fetch_kilo_data(bird, sess):
logger.info('Fetching data for bci_pipeline')
files = ['ss_bin']
fn = et.file_names(bird, sess)
#files += [f for f in fn['files'].iterkeys() if f.split('_')[0] == 'ks']
logger.info(files)
copied = copy_data(bird, sess, 'ss', 'tmp', only_files=files)
logger.debug(copied)
return copied
def do_the_sort(bird, sess):
logger.info('Running kilosort on matlab')
fn = et.file_names(bird, sess)
sort_folder = fn['folders']['tmp']
log_file = os.path.join(fn['folders']['ss'], 'kilosort_mat.log')
mlb_cmd = '-r "cd(\'{}\'); dir; master; exit();"'.format(sort_folder)
log_cmd = '-logfile {}'.format(log_file)
logger.info('output to {}'.format(log_file))
sorter = subprocess.check_output([
'matlab', '-nodesktop', '-nosplash', '-noawt', '-nojvm', mlb_cmd,
log_cmd
])
return sorter
def push_kilo_data(bird, sess):
logger.info('Pushing back bci_pipeline data')
fn = et.file_names(bird, sess)
source_folder = fn['folders']['tmp']
dest_folder = fn['folders']['ss']
extensions = ['npy', 'csv', 'py']
#print extensions
copied = []
for ext in extensions:
files = glob.glob(os.path.join(source_folder, '*.{}'.format(ext)))
copied += [
sh.copy2(os.path.join(source_folder, f), dest_folder)
for f in files
]
return copied
def make_kilo_scripts(
bird,
sess,
n_filt=None,
kilodir=os.path.abspath('/home/mthielk/github/KiloSort'),
npy_matdir=os.path.abspath('/home/mthielk/github/npy-matlab')):
fn = et.file_names(bird, sess)
exp_par = et.get_parameters(bird, sess) # load the yml parameter file
local_sort_dir = fn['folders']['tmp']
logger.debug('local sort dir: {}'.format(local_sort_dir))
block_name = fn['files']['base']
fs = h5f.get_record_sampling_frequency(et.open_kwd(bird, sess))
n_chan = len(exp_par['channel_config']['neural'])
logger.debug('n_chan: {}'.format(n_chan))
if n_filt is None:
n_filt = np.int(np.ceil(16 * 3. / 32.) * 32)
params = {
'kilodir': kilodir,
'npy_matdir': npy_matdir,
'datadir': local_sort_dir,
'blockname': block_name,
'fs': fs,
'Nchan': n_chan,
'Nfilt': n_filt
}
print params
template_dir = fn['folders']['templ']
print template_dir
with open(os.path.join(template_dir, 'master.template'), 'r') as src:
master_template = Template(src.read())
with open(os.path.join(template_dir, 'config.template'), 'r') as src:
config_template = Template(src.read())
et.mkdir_p(local_sort_dir)
with open(et.file_path(fn, 'tmp', 'ks_mas'), 'w') as f:
f.write(master_template.substitute(params))
with open(et.file_path(fn, 'tmp', 'ks_cfg'), 'w') as f:
f.write(config_template.substitute(params))
def make_kilo_chanmap(bird, sess):
fn = et.file_names(bird, sess)
logger.debug('Making ChanMap file')
par = et.get_parameters(bird, sess)
try:
probe_serial = par['probe']['serial']
logger.debug('Probe serial specified: {}'.format(probe_serial))
try:
probe_rev = par['probe']['rev']
logger.debug('Probe rev specified: {}'.format(probe_rev))
except KeyError:
probe_rev = '0'
logger.debug(
'Probe rev not specified: default is {}'.format(probe_rev))
prb_file = os.path.join(fn['folders']['prb'],
'{0}_{1}.prb'.format(probe_serial, probe_rev))
logger.debug('Probe should be {}'.format(prb_file))
except KeyError:
logger.debug(
'probe not specified in par file, going for default in-folder .prb file'
)
prb_file_path = et.file_path(fn, 'ss', 'kk_prb')
assert len(
glob.glob(prb_file_path)
) == 1, "Error finding .prb file in {}".format(prb_file_path)
prb_file = glob.glob(prb_file_path)[0]
logger.info('probe file: {}'.format(prb_file))
with open(prb_file, 'r') as f:
contents = f.read()
metadata = {}
exec(contents, {}, metadata)
n_channels = 0
for group in metadata['channel_groups']:
n_channels = max(n_channels,
np.max(metadata['channel_groups'][group]['channels']))
n_channels += 1
connected = np.array([True] * n_channels).reshape((n_channels, 1))
chan_map = np.arange(n_channels) + 1
chan_map_zero_ind = np.arange(n_channels)
x_coords = np.ones((n_channels, 1)) * -1
y_coords = np.ones((n_channels, 1)) * -1
k_coords = np.ones((n_channels, 1)) * -1
for group in metadata['channel_groups']:
for channel in metadata['channel_groups'][group]['geometry']:
x_coords[channel], y_coords[channel] = metadata['channel_groups'][
group]['geometry'][channel]
k_coords[channel] = group + 1
chan_map_dict = {
'Nchannels': n_channels,
'connected': connected,
'chanMap': chan_map,
'chanMap0ind': chan_map_zero_ind,
'xcoords': x_coords,
'ycoords': y_coords,
'kcoords': k_coords
}
sio.savemat(et.file_path(fn, 'tmp', 'ks_map'), chan_map_dict)