def install_ironclust(commit): spikeforest_alg_install_path = get_install_path() repo = 'https://github.com/jamesjun/ironclust' key = dict( alg='ironclust', repo=repo, commit=commit ) source_path = spikeforest_alg_install_path + '/ironclust_' + commit with FileLock(source_path + '.lock', exclusive=True): if not os.path.exists(source_path + '/spikeforest.json'): if os.path.exists(source_path): shutil.rmtree(source_path) script = """ #!/bin/bash set -e git clone {repo} {source_path} cd {source_path} git checkout {commit} """.format(repo=repo, commit=commit, source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Install script returned a non-zero exit code/') with open(source_path + '/spikeforest.json', 'w') as f: json.dump(key, f) return source_path
def install_waveclus(repo, commit): spikeforest_alg_install_path = get_install_path() key = dict( alg='waveclus', repo=repo, commit=commit ) source_path = spikeforest_alg_install_path + '/waveclus_' + commit if os.path.exists(source_path): # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # if hash0 == mt.getValue(key=key): # print('waveclus is already auto-installed.') # return source_path a = mt.loadObject(path=source_path + '/spikeforest.json') if a: if mt.sha1OfObject(a) == mt.sha1OfObject(key): print('waveclus is already auto-installed.') return source_path print('Removing directory: {}'.format(source_path)) shutil.rmtree(source_path) script = """ #!/bin/bash set -e git clone {repo} {source_path} cd {source_path} git checkout {commit} """.format(repo=repo, commit=commit, source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Install script returned a non-zero exit code/') # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # mt.setValue(key=key, value=hash0) mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json') return source_path
def jrclust_helper( *, recording, # Recording object tmpdir, # Temporary working directory params=dict(), **kwargs): jrclust_path = os.environ.get('JRCLUST_PATH_DEV', None) if jrclust_path: print('Using jrclust from JRCLUST_PATH_DEV directory: {}'.format( jrclust_path)) else: try: print('Auto-installing jrclust.') jrclust_path = install_jrclust( repo='https://github.com/JaneliaSciComp/JRCLUST.git', commit='3d2e75c0041dca2a9f273598750c6a14dbc4c1b8') except: traceback.print_exc() raise Exception( 'Problem installing jrclust. You can set the JRCLUST_PATH_DEV to force to use a particular path.' ) print('Using jrclust from: {}'.format(jrclust_path)) dataset_dir = os.path.join(tmpdir, 'jrclust_dataset') # Generate three files in the dataset directory: raw.mda, geom.csv, params.json SFMdaRecordingExtractor.write_recording(recording=recording, save_path=dataset_dir, params=params, _preserve_dtype=True) samplerate = recording.get_sampling_frequency() print('Reading timeseries header...') raw_mda = os.path.join(dataset_dir, 'raw.mda') HH = mdaio.readmda_header(raw_mda) num_channels = HH.dims[0] num_timepoints = HH.dims[1] duration_minutes = num_timepoints / samplerate / 60 print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'. format(num_channels, num_timepoints, duration_minutes)) print('Creating argfile.txt...') txt = '' for key0, val0 in kwargs.items(): txt += '{}={}\n'.format(key0, val0) if 'scale_factor' in params: txt += 'bitScaling={}\n'.format(params["scale_factor"]) txt += 'sampleRate={}\n'.format(samplerate) _write_text_file(dataset_dir + '/argfile.txt', txt) # new method source_path = os.path.dirname(os.path.realpath(__file__)) print('Running jrclust in {tmpdir}...'.format(tmpdir=tmpdir)) cmd = ''' addpath('{jrclust_path}', '{source_path}', '{source_path}/mdaio'); try p_jrclust('{tmpdir}', '{dataset_dir}/raw.mda', '{dataset_dir}/geom.csv', '{tmpdir}/firings.mda', '{dataset_dir}/argfile.txt'); catch fprintf('----------------------------------------'); fprintf(lasterr()); quit(1); end quit(0); ''' cmd = cmd.format(jrclust_path=jrclust_path, tmpdir=tmpdir, dataset_dir=dataset_dir, source_path=source_path) matlab_cmd = mlpr.ShellScript(cmd, script_path=tmpdir + '/run_jrclust.m', keep_temp_files=True) matlab_cmd.write() shell_cmd = ''' #!/bin/bash cd {tmpdir} matlab -nosplash -nodisplay -r run_jrclust '''.format(tmpdir=tmpdir) shell_cmd = mlpr.ShellScript(shell_cmd, script_path=tmpdir + '/run_jrclust.sh', keep_temp_files=True) shell_cmd.write(tmpdir + '/run_jrclust.sh') time_ = time.time() shell_cmd.start() retcode = shell_cmd.wait() print('#SF-SORTER-RUNTIME#{:.3f}#'.format(time_ - time.time())) if retcode != 0: raise Exception('jrclust returned a non-zero exit code') # parse output result_fname = tmpdir + '/firings.mda' if not os.path.exists(result_fname): raise Exception('Result file does not exist: ' + result_fname) firings = mdaio.readmda(result_fname) sorting = se.NumpySortingExtractor() sorting.set_times_labels(firings[1, :], firings[2, :]) return sorting
def kilosort2_helper( *, recording, # Recording object tmpdir, # Temporary working directory detect_sign=-1, # Polarity of the spikes, -1, 0, or 1 adjacency_radius=-1, # Channel neighborhood adjacency radius corresponding to geom file detect_threshold=6, # Threshold for detection merge_thresh=.98, # Cluster merging threhold 0..1 freq_min=150, # Lower frequency limit for band-pass filter freq_max=6000, # Upper frequency limit for band-pass filter pc_per_chan=3, # number of PC per chan minFR=1 / 50): # # TODO: do not require ks2 to depend on irc -- rather, put all necessary .m code in the spikeforest repo # ironclust_path = os.environ.get('IRONCLUST_PATH_DEV', None) # if ironclust_path: # print('Using ironclust from IRONCLUST_PATH_DEV directory: {}'.format(ironclust_path)) # else: # try: # print('Auto-installing ironclust.') # ironclust_path = install_ironclust(commit='042b600b014de13f6d11d3b4e50e849caafb4709') # except: # traceback.print_exc() # raise Exception('Problem installing ironclust. You can set the IRONCLUST_PATH_DEV to force to use a particular path.') # print('For kilosort2, using ironclust utility functions from: {}'.format(ironclust_path)) kilosort2_path = os.environ.get('KILOSORT2_PATH_DEV', None) if kilosort2_path: print('Using kilosort2 from KILOSORT2_PATH_DEV directory: {}'.format( kilosort2_path)) else: try: print('Auto-installing kilosort2.') kilosort2_path = KiloSort2.install() except: traceback.print_exc() raise Exception( 'Problem installing kilosort2. You can set the KILOSORT2_PATH_DEV to force to use a particular path.' ) print('Using kilosort2 from: {}'.format(kilosort2_path)) source_dir = os.path.dirname(os.path.realpath(__file__)) dataset_dir = tmpdir + '/kilosort2_dataset' # Generate three files in the dataset directory: raw.mda, geom.csv, params.json SFMdaRecordingExtractor.write_recording(recording=recording, save_path=dataset_dir, _preserve_dtype=True) samplerate = recording.get_sampling_frequency() print('Reading timeseries header...') HH = mdaio.readmda_header(dataset_dir + '/raw.mda') num_channels = HH.dims[0] num_timepoints = HH.dims[1] duration_minutes = num_timepoints / samplerate / 60 print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'. format(num_channels, num_timepoints, duration_minutes)) print('Creating argfile.txt file...') txt = '' txt += 'samplerate={}\n'.format(samplerate) txt += 'detect_sign={}\n'.format(detect_sign) txt += 'adjacency_radius={}\n'.format(adjacency_radius) txt += 'detect_threshold={}\n'.format(detect_threshold) txt += 'merge_thresh={}\n'.format(merge_thresh) txt += 'freq_min={}\n'.format(freq_min) txt += 'freq_max={}\n'.format(freq_max) txt += 'pc_per_chan={}\n'.format(pc_per_chan) txt += 'minFR={}\n'.format(minFR) _write_text_file(dataset_dir + '/argfile.txt', txt) print('Running Kilosort2 in {tmpdir}...'.format(tmpdir=tmpdir)) cmd = ''' addpath('{source_dir}'); addpath('{source_dir}/mdaio') try p_kilosort2('{ksort}', '{tmpdir}', '{raw}', '{geom}', '{firings}', '{arg}'); catch quit(1); end quit(0); ''' cmd = cmd.format(source_dir=source_dir, ksort=kilosort2_path, tmpdir=tmpdir, raw=dataset_dir + '/raw.mda', geom=dataset_dir + '/geom.csv', firings=tmpdir + '/firings.mda', arg=dataset_dir + '/argfile.txt') matlab_cmd = mlpr.ShellScript(cmd, script_path=tmpdir + '/run_kilosort2.m', keep_temp_files=True) matlab_cmd.write() shell_cmd = ''' #!/bin/bash cd {tmpdir} echo '=====================' `date` '=====================' matlab -nosplash -nodisplay -r run_kilosort2 '''.format(tmpdir=tmpdir) shell_cmd = mlpr.ShellScript(shell_cmd, script_path=tmpdir + '/run_kilosort2.sh', keep_temp_files=True) shell_cmd.write(tmpdir + '/run_kilosort2.sh') shell_cmd.start() retcode = shell_cmd.wait() if retcode != 0: raise Exception('kilosort2 returned a non-zero exit code') # parse output result_fname = tmpdir + '/firings.mda' if not os.path.exists(result_fname): raise Exception('Result file does not exist: ' + result_fname) firings = mdaio.readmda(result_fname) sorting = se.NumpySortingExtractor() sorting.set_times_labels(firings[1, :], firings[2, :]) return sorting
def ironclust_helper( *, recording, # Recording object tmpdir, # Temporary working directory params=dict(), ironclust_path, **kwargs): source_dir = os.path.dirname(os.path.realpath(__file__)) dataset_dir = tmpdir + '/ironclust_dataset' # Generate three files in the dataset directory: raw.mda, geom.csv, params.json SFMdaRecordingExtractor.write_recording( recording=recording, save_path=dataset_dir, params=params, _preserve_dtype=True) samplerate = recording.get_sampling_frequency() print('Reading timeseries header...') HH = mdaio.readmda_header(dataset_dir + '/raw.mda') num_channels = HH.dims[0] num_timepoints = HH.dims[1] duration_minutes = num_timepoints / samplerate / 60 print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.format( num_channels, num_timepoints, duration_minutes)) print('Creating argfile.txt...') txt = '' for key0, val0 in kwargs.items(): txt += '{}={}\n'.format(key0, val0) txt += 'samplerate={}\n'.format(samplerate) if 'scale_factor' in params: txt += 'scale_factor={}\n'.format(params["scale_factor"]) _write_text_file(dataset_dir + '/argfile.txt', txt) # new method print('Running ironclust in {tmpdir}...'.format(tmpdir=tmpdir)) cmd = ''' addpath('{source_dir}'); addpath('{ironclust_path}', '{ironclust_path}/matlab', '{ironclust_path}/matlab/mdaio'); try p_ironclust('{tmpdir}', '{dataset_dir}/raw.mda', '{dataset_dir}/geom.csv', '', '', '{tmpdir}/firings.mda', '{dataset_dir}/argfile.txt'); catch fprintf('----------------------------------------'); fprintf(lasterr()); quit(1); end quit(0); ''' cmd = cmd.format(ironclust_path=ironclust_path, tmpdir=tmpdir, dataset_dir=dataset_dir, source_dir=source_dir) matlab_cmd = mlpr.ShellScript(cmd, script_path=tmpdir + '/run_ironclust.m', keep_temp_files=True) matlab_cmd.write() shell_cmd = ''' #!/bin/bash cd {tmpdir} matlab -nosplash -nodisplay -r run_ironclust '''.format(tmpdir=tmpdir) shell_cmd = mlpr.ShellScript(shell_cmd, script_path=tmpdir + '/run_ironclust.sh', keep_temp_files=True) shell_cmd.write(tmpdir + '/run_ironclust.sh') shell_cmd.start() retcode = shell_cmd.wait() if retcode != 0: raise Exception('ironclust returned a non-zero exit code') # parse output result_fname = tmpdir + '/firings.mda' if not os.path.exists(result_fname): raise Exception('Result file does not exist: ' + result_fname) firings = mdaio.readmda(result_fname) sorting = se.NumpySortingExtractor() sorting.set_times_labels(firings[1, :], firings[2, :]) return sorting
def waveclus_helper( *, recording, # Recording object tmpdir, # Temporary working directory params=dict(), **kwargs): waveclus_path = os.environ.get('WAVECLUS_PATH_DEV', None) if waveclus_path: print('Using waveclus from WAVECLUS_PATH_DEV directory: {}'.format( waveclus_path)) else: try: print('Auto-installing waveclus.') waveclus_path = install_waveclus( repo='https://github.com/csn-le/wave_clus.git', commit='248d15c7eaa2b45b15e4488dfb9b09bfe39f5341') except: traceback.print_exc() raise Exception( 'Problem installing waveclus. You can set the WAVECLUS_PATH_DEV to force to use a particular path.' ) print('Using waveclus from: {}'.format(waveclus_path)) dataset_dir = os.path.join(tmpdir, 'waveclus_dataset') # Generate three files in the dataset directory: raw.mda, geom.csv, params.json SFMdaRecordingExtractor.write_recording(recording=recording, save_path=dataset_dir, params=params, _preserve_dtype=True) samplerate = recording.get_sampling_frequency() print('Reading timeseries header...') raw_mda = os.path.join(dataset_dir, 'raw.mda') HH = mdaio.readmda_header(raw_mda) num_channels = HH.dims[0] num_timepoints = HH.dims[1] duration_minutes = num_timepoints / samplerate / 60 print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'. format(num_channels, num_timepoints, duration_minutes)) # new method source_path = os.path.dirname(os.path.realpath(__file__)) print('Running waveclus in {tmpdir}...'.format(tmpdir=tmpdir)) cmd = ''' addpath(genpath('{waveclus_path}'), '{source_path}', '{source_path}/mdaio'); try p_waveclus('{tmpdir}', '{dataset_dir}/raw.mda', '{tmpdir}/firings.mda', {samplerate}); catch fprintf('----------------------------------------'); fprintf(lasterr()); quit(1); end quit(0); ''' cmd = cmd.format(waveclus_path=waveclus_path, tmpdir=tmpdir, dataset_dir=dataset_dir, source_path=source_path, samplerate=samplerate) matlab_cmd = mlpr.ShellScript(cmd, script_path=tmpdir + '/run_waveclus.m', keep_temp_files=True) matlab_cmd.write() shell_cmd = ''' #!/bin/bash cd {tmpdir} matlab -nosplash -nodisplay -r run_waveclus '''.format(tmpdir=tmpdir) shell_cmd = mlpr.ShellScript(shell_cmd, script_path=tmpdir + '/run_waveclus.sh', keep_temp_files=True) shell_cmd.write(tmpdir + '/run_waveclus.sh') time_ = time.time() shell_cmd.start() retcode = shell_cmd.wait() print('#SF-SORTER-RUNTIME#{:.3f}#'.format(time_ - time.time())) if retcode != 0: raise Exception('waveclus returned a non-zero exit code') # parse output result_fname = tmpdir + '/firings.mda' if not os.path.exists(result_fname): raise Exception('Result file does not exist: ' + result_fname) firings = mdaio.readmda(result_fname) sorting = se.NumpySortingExtractor() sorting.set_times_labels(firings[1, :], firings[2, :]) return sorting
def install_jrclust(repo, commit): spikeforest_alg_install_path = get_install_path() key = dict(alg='jrclust', repo=repo, commit=commit) source_path = spikeforest_alg_install_path + '/jrclust_' + commit if os.path.exists(source_path): # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # if hash0 == mt.getValue(key=key): # print('jrclust is already auto-installed.') # return source_path a = mt.loadObject(path=source_path + '/spikeforest.json') if a: if mt.sha1OfObject(a) == mt.sha1OfObject(key): print('jrclust is already auto-installed.') return source_path print('Removing directory: {}'.format(source_path)) shutil.rmtree(source_path) script = """ #!/bin/bash set -e git clone {repo} {source_path} cd {source_path} git checkout {commit} """.format(repo=repo, commit=commit, source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Install script returned a non-zero exit code/') compile_gpu = mlpr.ShellScript(script=""" function compile_gpu try jrc compile catch disp('Problem running `jrc compile`'); disp(lasterr()); exit(-1) end; exit(0) """) compile_gpu.write(script_path=source_path + '/compile_gpu.m') script = """ #!/bin/bash set -e cd {source_path} matlab -nodisplay -nosplash -r "compile_gpu" """.format(source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Compute gpu script returned a non-zero exit code.') # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # mt.setValue(key=key, value=hash0) mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json') return source_path
def install_kilosort2(repo, commit): spikeforest_alg_install_path = get_install_path() key = dict(alg='kilosort2', repo=repo, commit=commit) source_path = spikeforest_alg_install_path + '/kilosort2_' + commit if os.path.exists(source_path): # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # if hash0 == mt.getValue(key=key): # print('Kilosort2 is already auto-installed.') # return source_path a = mt.loadObject(path=source_path + '/spikeforest.json') if a: if mt.sha1OfObject(a) == mt.sha1OfObject(key): print('Kilosort2 is already auto-installed.') return source_path print('Removing directory: {}'.format(source_path)) shutil.rmtree(source_path) script = """ #!/bin/bash set -e git clone {repo} {source_path} cd {source_path} git checkout {commit} """.format(repo=repo, commit=commit, source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Install script returned a non-zero exit code/') # make sure module unload gcc/7.4.0 compile_gpu = mlpr.ShellScript(script=""" function compile_gpu try [~,path_nvcc_] = system('which nvcc'); path_nvcc_ = strrep(path_nvcc_, 'nvcc', ''); disp(['path_nvcc_: ', path_nvcc_]); setenv('MW_NVCC_PATH', path_nvcc_); run('mexGPUall.m'); catch disp('Problem running mexGPUall.'); disp(lasterr()); exit(-1) end; exit(0) """) compile_gpu.write(script_path=source_path + '/CUDA/compile_gpu.m') script = """ #!/bin/bash set -e cd {source_path}/CUDA matlab -nodisplay -nosplash -r "compile_gpu" """.format(source_path=source_path) ss = mlpr.ShellScript(script=script) ss.start() retcode = ss.wait() if retcode != 0: raise Exception('Compute gpu script returned a non-zero exit code.') # The dir hash method does not seem to be working for some reason here # hash0 = mt.computeDirHash(source_path) # mt.setValue(key=key, value=hash0) mt.saveObject(object=key, dest_path=source_path + '/spikeforest.json') return source_path