def install_ironclust(commit):
spikeforest_alg_install_path = get_install_path()
repo = 'https://github.com/jamesjun/ironclust'
key = dict(
alg='ironclust',
repo=repo,
commit=commit
)
source_path = spikeforest_alg_install_path + '/ironclust_' + commit
with FileLock(source_path + '.lock', exclusive=True):
if not os.path.exists(source_path + '/spikeforest.json'):
if os.path.exists(source_path):
shutil.rmtree(source_path)
script = """
#!/bin/bash
set -e
git clone {repo} {source_path}
cd {source_path}
git checkout {commit}
""".format(repo=repo, commit=commit, source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Install script returned a non-zero exit code/')
with open(source_path + '/spikeforest.json', 'w') as f:
json.dump(key, f)
return source_path
def install_waveclus(repo, commit):
spikeforest_alg_install_path = get_install_path()
key = dict(
alg='waveclus',
repo=repo,
commit=commit
)
source_path = spikeforest_alg_install_path + '/waveclus_' + commit
if os.path.exists(source_path):
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# if hash0 == mt.getValue(key=key):
# print('waveclus is already auto-installed.')
# return source_path
a = mt.loadObject(path=source_path + '/spikeforest.json')
if a:
if mt.sha1OfObject(a) == mt.sha1OfObject(key):
print('waveclus is already auto-installed.')
return source_path
print('Removing directory: {}'.format(source_path))
shutil.rmtree(source_path)
script = """
#!/bin/bash
set -e
git clone {repo} {source_path}
cd {source_path}
git checkout {commit}
""".format(repo=repo, commit=commit, source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Install script returned a non-zero exit code/')
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# mt.setValue(key=key, value=hash0)
mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json')
return source_path
def jrclust_helper(
*,
recording, # Recording object
tmpdir, # Temporary working directory
params=dict(),
**kwargs):
jrclust_path = os.environ.get('JRCLUST_PATH_DEV', None)
if jrclust_path:
print('Using jrclust from JRCLUST_PATH_DEV directory: {}'.format(
jrclust_path))
else:
try:
print('Auto-installing jrclust.')
jrclust_path = install_jrclust(
repo='https://github.com/JaneliaSciComp/JRCLUST.git',
commit='3d2e75c0041dca2a9f273598750c6a14dbc4c1b8')
except:
traceback.print_exc()
raise Exception(
'Problem installing jrclust. You can set the JRCLUST_PATH_DEV to force to use a particular path.'
)
print('Using jrclust from: {}'.format(jrclust_path))
dataset_dir = os.path.join(tmpdir, 'jrclust_dataset')
# Generate three files in the dataset directory: raw.mda, geom.csv, params.json
SFMdaRecordingExtractor.write_recording(recording=recording,
save_path=dataset_dir,
params=params,
_preserve_dtype=True)
samplerate = recording.get_sampling_frequency()
print('Reading timeseries header...')
raw_mda = os.path.join(dataset_dir, 'raw.mda')
HH = mdaio.readmda_header(raw_mda)
num_channels = HH.dims[0]
num_timepoints = HH.dims[1]
duration_minutes = num_timepoints / samplerate / 60
print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.
format(num_channels, num_timepoints, duration_minutes))
print('Creating argfile.txt...')
txt = ''
for key0, val0 in kwargs.items():
txt += '{}={}\n'.format(key0, val0)
if 'scale_factor' in params:
txt += 'bitScaling={}\n'.format(params["scale_factor"])
txt += 'sampleRate={}\n'.format(samplerate)
_write_text_file(dataset_dir + '/argfile.txt', txt)
# new method
source_path = os.path.dirname(os.path.realpath(__file__))
print('Running jrclust in {tmpdir}...'.format(tmpdir=tmpdir))
cmd = '''
addpath('{jrclust_path}', '{source_path}', '{source_path}/mdaio');
try
p_jrclust('{tmpdir}', '{dataset_dir}/raw.mda', '{dataset_dir}/geom.csv', '{tmpdir}/firings.mda', '{dataset_dir}/argfile.txt');
catch
fprintf('----------------------------------------');
fprintf(lasterr());
quit(1);
end
quit(0);
'''
cmd = cmd.format(jrclust_path=jrclust_path,
tmpdir=tmpdir,
dataset_dir=dataset_dir,
source_path=source_path)
matlab_cmd = mlpr.ShellScript(cmd,
script_path=tmpdir + '/run_jrclust.m',
keep_temp_files=True)
matlab_cmd.write()
shell_cmd = '''
#!/bin/bash
cd {tmpdir}
matlab -nosplash -nodisplay -r run_jrclust
'''.format(tmpdir=tmpdir)
shell_cmd = mlpr.ShellScript(shell_cmd,
script_path=tmpdir + '/run_jrclust.sh',
keep_temp_files=True)
shell_cmd.write(tmpdir + '/run_jrclust.sh')
time_ = time.time()
shell_cmd.start()
retcode = shell_cmd.wait()
print('#SF-SORTER-RUNTIME#{:.3f}#'.format(time_ - time.time()))
if retcode != 0:
raise Exception('jrclust returned a non-zero exit code')
# parse output
result_fname = tmpdir + '/firings.mda'
if not os.path.exists(result_fname):
raise Exception('Result file does not exist: ' + result_fname)
firings = mdaio.readmda(result_fname)
sorting = se.NumpySortingExtractor()
sorting.set_times_labels(firings[1, :], firings[2, :])
return sorting
def kilosort2_helper(
*,
recording, # Recording object
tmpdir, # Temporary working directory
detect_sign=-1, # Polarity of the spikes, -1, 0, or 1
adjacency_radius=-1, # Channel neighborhood adjacency radius corresponding to geom file
detect_threshold=6, # Threshold for detection
merge_thresh=.98, # Cluster merging threhold 0..1
freq_min=150, # Lower frequency limit for band-pass filter
freq_max=6000, # Upper frequency limit for band-pass filter
pc_per_chan=3, # number of PC per chan
minFR=1 / 50):
# # TODO: do not require ks2 to depend on irc -- rather, put all necessary .m code in the spikeforest repo
# ironclust_path = os.environ.get('IRONCLUST_PATH_DEV', None)
# if ironclust_path:
# print('Using ironclust from IRONCLUST_PATH_DEV directory: {}'.format(ironclust_path))
# else:
# try:
# print('Auto-installing ironclust.')
# ironclust_path = install_ironclust(commit='042b600b014de13f6d11d3b4e50e849caafb4709')
# except:
# traceback.print_exc()
# raise Exception('Problem installing ironclust. You can set the IRONCLUST_PATH_DEV to force to use a particular path.')
# print('For kilosort2, using ironclust utility functions from: {}'.format(ironclust_path))
kilosort2_path = os.environ.get('KILOSORT2_PATH_DEV', None)
if kilosort2_path:
print('Using kilosort2 from KILOSORT2_PATH_DEV directory: {}'.format(
kilosort2_path))
else:
try:
print('Auto-installing kilosort2.')
kilosort2_path = KiloSort2.install()
except:
traceback.print_exc()
raise Exception(
'Problem installing kilosort2. You can set the KILOSORT2_PATH_DEV to force to use a particular path.'
)
print('Using kilosort2 from: {}'.format(kilosort2_path))
source_dir = os.path.dirname(os.path.realpath(__file__))
dataset_dir = tmpdir + '/kilosort2_dataset'
# Generate three files in the dataset directory: raw.mda, geom.csv, params.json
SFMdaRecordingExtractor.write_recording(recording=recording,
save_path=dataset_dir,
_preserve_dtype=True)
samplerate = recording.get_sampling_frequency()
print('Reading timeseries header...')
HH = mdaio.readmda_header(dataset_dir + '/raw.mda')
num_channels = HH.dims[0]
num_timepoints = HH.dims[1]
duration_minutes = num_timepoints / samplerate / 60
print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.
format(num_channels, num_timepoints, duration_minutes))
print('Creating argfile.txt file...')
txt = ''
txt += 'samplerate={}\n'.format(samplerate)
txt += 'detect_sign={}\n'.format(detect_sign)
txt += 'adjacency_radius={}\n'.format(adjacency_radius)
txt += 'detect_threshold={}\n'.format(detect_threshold)
txt += 'merge_thresh={}\n'.format(merge_thresh)
txt += 'freq_min={}\n'.format(freq_min)
txt += 'freq_max={}\n'.format(freq_max)
txt += 'pc_per_chan={}\n'.format(pc_per_chan)
txt += 'minFR={}\n'.format(minFR)
_write_text_file(dataset_dir + '/argfile.txt', txt)
print('Running Kilosort2 in {tmpdir}...'.format(tmpdir=tmpdir))
cmd = '''
addpath('{source_dir}');
addpath('{source_dir}/mdaio')
try
p_kilosort2('{ksort}', '{tmpdir}', '{raw}', '{geom}', '{firings}', '{arg}');
catch
quit(1);
end
quit(0);
'''
cmd = cmd.format(source_dir=source_dir,
ksort=kilosort2_path,
tmpdir=tmpdir,
raw=dataset_dir + '/raw.mda',
geom=dataset_dir + '/geom.csv',
firings=tmpdir + '/firings.mda',
arg=dataset_dir + '/argfile.txt')
matlab_cmd = mlpr.ShellScript(cmd,
script_path=tmpdir + '/run_kilosort2.m',
keep_temp_files=True)
matlab_cmd.write()
shell_cmd = '''
#!/bin/bash
cd {tmpdir}
echo '=====================' `date` '====================='
matlab -nosplash -nodisplay -r run_kilosort2
'''.format(tmpdir=tmpdir)
shell_cmd = mlpr.ShellScript(shell_cmd,
script_path=tmpdir + '/run_kilosort2.sh',
keep_temp_files=True)
shell_cmd.write(tmpdir + '/run_kilosort2.sh')
shell_cmd.start()
retcode = shell_cmd.wait()
if retcode != 0:
raise Exception('kilosort2 returned a non-zero exit code')
# parse output
result_fname = tmpdir + '/firings.mda'
if not os.path.exists(result_fname):
raise Exception('Result file does not exist: ' + result_fname)
firings = mdaio.readmda(result_fname)
sorting = se.NumpySortingExtractor()
sorting.set_times_labels(firings[1, :], firings[2, :])
return sorting
def ironclust_helper(
*,
recording, # Recording object
tmpdir, # Temporary working directory
params=dict(),
ironclust_path,
**kwargs):
source_dir = os.path.dirname(os.path.realpath(__file__))
dataset_dir = tmpdir + '/ironclust_dataset'
# Generate three files in the dataset directory: raw.mda, geom.csv, params.json
SFMdaRecordingExtractor.write_recording(
recording=recording, save_path=dataset_dir, params=params, _preserve_dtype=True)
samplerate = recording.get_sampling_frequency()
print('Reading timeseries header...')
HH = mdaio.readmda_header(dataset_dir + '/raw.mda')
num_channels = HH.dims[0]
num_timepoints = HH.dims[1]
duration_minutes = num_timepoints / samplerate / 60
print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.format(
num_channels, num_timepoints, duration_minutes))
print('Creating argfile.txt...')
txt = ''
for key0, val0 in kwargs.items():
txt += '{}={}\n'.format(key0, val0)
txt += 'samplerate={}\n'.format(samplerate)
if 'scale_factor' in params:
txt += 'scale_factor={}\n'.format(params["scale_factor"])
_write_text_file(dataset_dir + '/argfile.txt', txt)
# new method
print('Running ironclust in {tmpdir}...'.format(tmpdir=tmpdir))
cmd = '''
addpath('{source_dir}');
addpath('{ironclust_path}', '{ironclust_path}/matlab', '{ironclust_path}/matlab/mdaio');
try
p_ironclust('{tmpdir}', '{dataset_dir}/raw.mda', '{dataset_dir}/geom.csv', '', '', '{tmpdir}/firings.mda', '{dataset_dir}/argfile.txt');
catch
fprintf('----------------------------------------');
fprintf(lasterr());
quit(1);
end
quit(0);
'''
cmd = cmd.format(ironclust_path=ironclust_path, tmpdir=tmpdir, dataset_dir=dataset_dir, source_dir=source_dir)
matlab_cmd = mlpr.ShellScript(cmd, script_path=tmpdir + '/run_ironclust.m', keep_temp_files=True)
matlab_cmd.write()
shell_cmd = '''
#!/bin/bash
cd {tmpdir}
matlab -nosplash -nodisplay -r run_ironclust
'''.format(tmpdir=tmpdir)
shell_cmd = mlpr.ShellScript(shell_cmd, script_path=tmpdir + '/run_ironclust.sh', keep_temp_files=True)
shell_cmd.write(tmpdir + '/run_ironclust.sh')
shell_cmd.start()
retcode = shell_cmd.wait()
if retcode != 0:
raise Exception('ironclust returned a non-zero exit code')
# parse output
result_fname = tmpdir + '/firings.mda'
if not os.path.exists(result_fname):
raise Exception('Result file does not exist: ' + result_fname)
firings = mdaio.readmda(result_fname)
sorting = se.NumpySortingExtractor()
sorting.set_times_labels(firings[1, :], firings[2, :])
return sorting
def waveclus_helper(
*,
recording, # Recording object
tmpdir, # Temporary working directory
params=dict(),
**kwargs):
waveclus_path = os.environ.get('WAVECLUS_PATH_DEV', None)
if waveclus_path:
print('Using waveclus from WAVECLUS_PATH_DEV directory: {}'.format(
waveclus_path))
else:
try:
print('Auto-installing waveclus.')
waveclus_path = install_waveclus(
repo='https://github.com/csn-le/wave_clus.git',
commit='248d15c7eaa2b45b15e4488dfb9b09bfe39f5341')
except:
traceback.print_exc()
raise Exception(
'Problem installing waveclus. You can set the WAVECLUS_PATH_DEV to force to use a particular path.'
)
print('Using waveclus from: {}'.format(waveclus_path))
dataset_dir = os.path.join(tmpdir, 'waveclus_dataset')
# Generate three files in the dataset directory: raw.mda, geom.csv, params.json
SFMdaRecordingExtractor.write_recording(recording=recording,
save_path=dataset_dir,
params=params,
_preserve_dtype=True)
samplerate = recording.get_sampling_frequency()
print('Reading timeseries header...')
raw_mda = os.path.join(dataset_dir, 'raw.mda')
HH = mdaio.readmda_header(raw_mda)
num_channels = HH.dims[0]
num_timepoints = HH.dims[1]
duration_minutes = num_timepoints / samplerate / 60
print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.
format(num_channels, num_timepoints, duration_minutes))
# new method
source_path = os.path.dirname(os.path.realpath(__file__))
print('Running waveclus in {tmpdir}...'.format(tmpdir=tmpdir))
cmd = '''
addpath(genpath('{waveclus_path}'), '{source_path}', '{source_path}/mdaio');
try
p_waveclus('{tmpdir}', '{dataset_dir}/raw.mda', '{tmpdir}/firings.mda', {samplerate});
catch
fprintf('----------------------------------------');
fprintf(lasterr());
quit(1);
end
quit(0);
'''
cmd = cmd.format(waveclus_path=waveclus_path,
tmpdir=tmpdir,
dataset_dir=dataset_dir,
source_path=source_path,
samplerate=samplerate)
matlab_cmd = mlpr.ShellScript(cmd,
script_path=tmpdir + '/run_waveclus.m',
keep_temp_files=True)
matlab_cmd.write()
shell_cmd = '''
#!/bin/bash
cd {tmpdir}
matlab -nosplash -nodisplay -r run_waveclus
'''.format(tmpdir=tmpdir)
shell_cmd = mlpr.ShellScript(shell_cmd,
script_path=tmpdir + '/run_waveclus.sh',
keep_temp_files=True)
shell_cmd.write(tmpdir + '/run_waveclus.sh')
time_ = time.time()
shell_cmd.start()
retcode = shell_cmd.wait()
print('#SF-SORTER-RUNTIME#{:.3f}#'.format(time_ - time.time()))
if retcode != 0:
raise Exception('waveclus returned a non-zero exit code')
# parse output
result_fname = tmpdir + '/firings.mda'
if not os.path.exists(result_fname):
raise Exception('Result file does not exist: ' + result_fname)
firings = mdaio.readmda(result_fname)
sorting = se.NumpySortingExtractor()
sorting.set_times_labels(firings[1, :], firings[2, :])
return sorting
def install_jrclust(repo, commit):
spikeforest_alg_install_path = get_install_path()
key = dict(alg='jrclust', repo=repo, commit=commit)
source_path = spikeforest_alg_install_path + '/jrclust_' + commit
if os.path.exists(source_path):
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# if hash0 == mt.getValue(key=key):
# print('jrclust is already auto-installed.')
# return source_path
a = mt.loadObject(path=source_path + '/spikeforest.json')
if a:
if mt.sha1OfObject(a) == mt.sha1OfObject(key):
print('jrclust is already auto-installed.')
return source_path
print('Removing directory: {}'.format(source_path))
shutil.rmtree(source_path)
script = """
#!/bin/bash
set -e
git clone {repo} {source_path}
cd {source_path}
git checkout {commit}
""".format(repo=repo, commit=commit, source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Install script returned a non-zero exit code/')
compile_gpu = mlpr.ShellScript(script="""
function compile_gpu
try
jrc compile
catch
disp('Problem running `jrc compile`');
disp(lasterr());
exit(-1)
end;
exit(0)
""")
compile_gpu.write(script_path=source_path + '/compile_gpu.m')
script = """
#!/bin/bash
set -e
cd {source_path}
matlab -nodisplay -nosplash -r "compile_gpu"
""".format(source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Compute gpu script returned a non-zero exit code.')
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# mt.setValue(key=key, value=hash0)
mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json')
return source_path
def install_kilosort2(repo, commit):
spikeforest_alg_install_path = get_install_path()
key = dict(alg='kilosort2', repo=repo, commit=commit)
source_path = spikeforest_alg_install_path + '/kilosort2_' + commit
if os.path.exists(source_path):
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# if hash0 == mt.getValue(key=key):
# print('Kilosort2 is already auto-installed.')
# return source_path
a = mt.loadObject(path=source_path + '/spikeforest.json')
if a:
if mt.sha1OfObject(a) == mt.sha1OfObject(key):
print('Kilosort2 is already auto-installed.')
return source_path
print('Removing directory: {}'.format(source_path))
shutil.rmtree(source_path)
script = """
#!/bin/bash
set -e
git clone {repo} {source_path}
cd {source_path}
git checkout {commit}
""".format(repo=repo, commit=commit, source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Install script returned a non-zero exit code/')
# make sure module unload gcc/7.4.0
compile_gpu = mlpr.ShellScript(script="""
function compile_gpu
try
[~,path_nvcc_] = system('which nvcc');
path_nvcc_ = strrep(path_nvcc_, 'nvcc', '');
disp(['path_nvcc_: ', path_nvcc_]);
setenv('MW_NVCC_PATH', path_nvcc_);
run('mexGPUall.m');
catch
disp('Problem running mexGPUall.');
disp(lasterr());
exit(-1)
end;
exit(0)
""")
compile_gpu.write(script_path=source_path + '/CUDA/compile_gpu.m')
script = """
#!/bin/bash
set -e
cd {source_path}/CUDA
matlab -nodisplay -nosplash -r "compile_gpu"
""".format(source_path=source_path)
ss = mlpr.ShellScript(script=script)
ss.start()
retcode = ss.wait()
if retcode != 0:
raise Exception('Compute gpu script returned a non-zero exit code.')
# The dir hash method does not seem to be working for some reason here
# hash0 = mt.computeDirHash(source_path)
# mt.setValue(key=key, value=hash0)
mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json')
return source_path
