def main():
parser = argparse.ArgumentParser(
description='update spike-front analysis history')
parser.add_argument(
'--delete-and-init',
help='Use this only the first time. Will wipe out the history.',
action='store_true')
parser.add_argument('--show',
help='Show the analysis history.',
action='store_true')
args = parser.parse_args()
history_path = 'key://pairio/spikeforest/spike-front-analysis-history.json'
results_path = 'key://pairio/spikeforest/spike-front-results.json'
mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public'])
if args.delete_and_init:
print('Initializing analysis history...')
mt.createSnapshot(mt.saveObject(object=dict(analyses=[])),
dest_path=history_path,
upload_to='spikeforest.public')
print('Done.')
return
print('Loading history...')
history = mt.loadObject(path=history_path)
assert history, 'Unable to load history'
if args.show:
for aa in history['analyses']:
print(
'==============================================================='
)
print('ANALYSIS: {}'.format(aa['General']['dateUpdated']))
print('PATH: {}'.format(aa['path']))
print(json.dumps(aa['General'], indent=4))
print('')
return
print('Loading current results...')
spike_front_results = mt.loadObject(path=results_path)
assert spike_front_results, 'Unable to load current results'
sha1_url = mt.saveObject(object=spike_front_results,
basename='analysis.json')
for aa in history['analyses']:
if aa['path'] == sha1_url:
print('Analysis already stored in history.')
return
history['analyses'].append(
dict(path=sha1_url, General=spike_front_results['General'][0]))
print('Saving updated history to {}'.format(history_path))
mt.createSnapshot(mt.saveObject(object=history),
dest_path=history_path,
upload_to='spikeforest.public')
print('Done.')
def do_sorting_test(sorting_processor,
params,
recording_dir,
assert_avg_accuracy,
container='default'):
mt.configDownloadFrom('spikeforest.kbucket')
recdir = recording_dir
mt.createSnapshot(path=recdir, download_recursive=True)
sorting = sorting_processor.execute(recording_dir=recdir,
firings_out={'ext': '.mda'},
**params,
_container=container,
_force_run=True)
comparison = sa.GenSortingComparisonTable.execute(
firings=sorting.outputs['firings_out'],
firings_true=recdir + '/firings_true.mda',
units_true=[],
json_out={'ext': '.json'},
html_out={'ext': '.html'},
_container=None,
_force_run=True)
X = mt.loadObject(path=comparison.outputs['json_out'])
accuracies = [float(a['accuracy']) for a in X.values()]
avg_accuracy = np.mean(accuracies)
print('Average accuracy: {}'.format(avg_accuracy))
assert (avg_accuracy >= assert_avg_accuracy)
def main():
path = mt.createSnapshot(path='recordings_out')
mt.configDownloadFrom('spikeforest.public')
X = mt.readDir(path)
for study_set_name, d in X['dirs'].items():
study_sets = []
studies = []
recordings = []
study_sets.append(dict(
name=study_set_name + '_b',
info=dict(),
description=''
))
for study_name, d2 in d['dirs'].items():
study_dir = path + '/' + study_set_name + '/' + study_name
study0 = dict(
name=study_name,
study_set=study_set_name + '_b',
directory=study_dir,
description=''
)
studies.append(study0)
index_within_study = 0
for recording_name, d3 in d2['dirs'].items():
recdir = study_dir + '/' + recording_name
recordings.append(dict(
name=recording_name,
study=study_name,
directory=recdir,
firings_true=recdir + '/firings_true.mda',
index_within_study=index_within_study,
description='One of the recordings in the {} study'.format(study_name)
))
index_within_study = index_within_study + 1
print('Saving object...')
group_name = study_set_name + '_b'
address = mt.saveObject(
object=dict(
studies=studies,
recordings=recordings,
study_sets=study_sets
),
key=dict(name='spikeforest_recording_group', group_name=group_name),
upload_to='spikeforest.public'
)
if not address:
raise Exception('Problem uploading object to kachery')
output_fname = 'key://pairio/spikeforest/spikeforest_recording_group.{}.json'.format(group_name)
print('Saving output to {}'.format(output_fname))
mt.createSnapshot(path=address, dest_path=output_fname)
def mem_profile_test(sorting_processor, params, recording_dir, container='default', _keep_temp_files=True):
mt.configDownloadFrom('spikeforest.public')
params['fMemProfile'] = True
recdir = recording_dir
mt.createSnapshot(path=recdir, download_recursive=True)
sorting = sorting_processor.execute(
recording_dir=recdir,
firings_out={'ext': '.mda'},
**params,
_container=container,
_force_run=True,
_keep_temp_files=_keep_temp_files
)
def prepare_synth_magland_studies(*, basedir):
study_sets = [load_study_set_from_md('descriptions/spf_synth_magland.md')]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/magland_synth'
print('Creating snapshot of study set directory...')
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
mt.createSnapshot(study_set_dir0,
upload_to=upload_public_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
study_set_dir = study_set_dir + '.synth_magland'
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
names = []
names = names + ['noise10_K10_C4', 'noise10_K10_C8']
names = names + ['noise10_K20_C4', 'noise10_K20_C8']
names = names + ['noise20_K10_C4', 'noise20_K10_C8']
names = names + ['noise20_K20_C4', 'noise20_K20_C8']
description = mt.loadText(path=study_set_dir + '/readme.txt')
for name in names:
print('PREPARING: ' + name)
study_name = 'synth_magland_' + name
study_dir = study_set_dir + '/datasets_' + name
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description=description)
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
recordings.append(
dict(
name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def prepare_synth_monotrode_studies(*, basedir):
study_sets = [
load_study_set_from_md('descriptions/spf_synth_monotrode.md')
]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/waveclus_synth'
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
mt.createSnapshot(study_set_dir0,
upload_to=upload_public_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
study_set_dir = study_set_dir + '.synth_monotrode'
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
names = [
'sim2_2K10', 'sim2_11K20', 'quiroga_easy1', 'quiroga_easy2',
'quiroga_difficult1', 'quiroga_difficult2'
]
for name in names:
study_name = 'neurocube_' + name
print('PREPARING: ' + study_name)
study_dir = '{}/{}'.format(study_set_dir, name)
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description='')
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
recordings.append(
dict(
name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def prepare_hybrid_janelia_studies(*, basedir):
study_sets = [load_study_set_from_md('descriptions/spf_hybrid_janelia.md')]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/hybrid_synth'
print('Creating snapshot of {}'.format(study_set_dir0))
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
mt.createSnapshot(study_set_dir0,
upload_to=upload_public_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
study_set_dir = study_set_dir + '.hybrid_janelia'
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
names = [
'drift_siprobe', 'static_siprobe', 'drift_tetrode', 'static_tetrode'
]
for name in names:
study_name = 'hybrid_' + name
print('PREPARING: ' + study_name)
study_dir = '{}/{}'.format(study_set_dir, name)
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description='')
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
print(dsdir)
recordings.append(
dict(
name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def prepare_paired_monotrode_studies(*, basedir):
study_sets = [
load_study_set_from_md('descriptions/spf_paired_monotrode.md')
]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/paired_monotrode'
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
mt.createSnapshot(study_set_dir0,
upload_to=upload_public_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
names = ['boyden32c', 'kampff', 'crcns', 'mea64c']
for name in names:
study_name = 'paired_monotrode_' + name
print('PREPARING: ' + study_name)
study_dir = study_set_dir + '/' + name
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description='')
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
recordings.append(
dict(
name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def do_prepare(recording_group, study_name):
print(recording_group, study_name)
X = mt.loadObject(
path="key://pairio/spikeforest/spikeforest_recording_group.{}.json".
format(recording_group))
studies = [y for y in X['studies'] if (y['name'] == study_name)]
recordings = [y for y in X['recordings'] if y['study'] == study_name]
recordings = recordings[0:1]
study_sets = X['study_sets']
Y = dict(studies=studies, recordings=recordings, study_sets=study_sets)
address = mt.saveObject(object=Y)
assert address is not None
dest_path = 'key://pairio/spikeforest/spikeforest_recording_group.test_{}.json'.format(
recording_group)
print(dest_path)
mt.createSnapshot(path=address,
upload_to='spikeforest.kbucket',
dest_path=dest_path)
def main():
parser = argparse.ArgumentParser(description=help_txt, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('path', help='Path to the assembled website data in .json format')
args = parser.parse_args()
print('Loading spike-front results object...')
with open(args.path, 'r') as f:
obj = json.load(f)
SortingResults = obj['SortingResults']
for sr in SortingResults:
print(sr['studyName'], sr['recordingName'])
console_out_fname = ka.load_file(sr['consoleOut'])
mt.createSnapshot(path=console_out_fname, upload_to='spikeforest.public')
if sr.get('firings', None) is not None:
firings_fname = ka.load_file(sr['firings'])
mt.createSnapshot(path=firings_fname, upload_to='spikeforest.public')
def do_sorting_test(
sorting_processor,
params,
recording_dir,
container='default',
force_run=True,
_keep_temp_files=False
):
mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public'])
recdir = recording_dir
mt.createSnapshot(path=recdir, download_recursive=True)
timer = time.time()
sorting = sorting_processor.execute(
recording_dir=recdir,
firings_out={'ext': '.mda'},
**params,
_container=container,
_force_run=force_run,
_keep_temp_files=_keep_temp_files
)
elapsed = time.time() - timer
print('################ ELAPSED for sorting (sec): {}'.format(elapsed))
timer = time.time()
comparison = sa.GenSortingComparisonTable.execute(
firings=sorting.outputs['firings_out'],
firings_true=recdir + '/firings_true.mda',
units_true=[],
json_out={'ext': '.json'},
html_out={'ext': '.html'},
_container='default',
_force_run=True
)
elapsed = time.time() - timer
print('################ ELAPSED for comparison (sec): {}'.format(elapsed))
X = mt.loadObject(path=comparison.outputs['json_out'])
accuracies = [float(a['accuracy']) for a in X.values()]
avg_accuracy = np.mean(accuracies)
print('Average accuracy: {}'.format(avg_accuracy))
def _np_snapshot(X):
_, fname = tempfile.mkstemp(suffix='.npy')
try:
np.save(fname, X)
ret = mt.createSnapshot(fname)
except:
raise
finally:
if os.path.exists(fname):
os.unlink(fname)
return ret
def prepare_test_reliability_studies(*, basedir):
study_sets = [
load_study_set_from_md('descriptions/spf_test_reliability.md')
]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/test_reliability'
print('Creating snapshot of study set directory...')
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
# study_set_dir = study_set_dir + '.test_reliability'
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
names = [
'001_synth', '002_synth', '003_synth', '004_synth', '005_synth',
'006_synth', '007_synth', '008_synth', '009_synth', '010_synth'
]
# description = mt.loadText(path=study_set_dir + '/readme.txt')
for name in names:
print('PREPARING: ' + name)
study_name = 'test_reliability_' + name
study_dir = study_set_dir + '/' + name
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description='')
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
recordings.append(
dict(
name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def prepare_manual_buzsaki_studies(*, basedir):
study_sets = [
load_study_set_from_md('descriptions/spf_manual_buzsakilab.md')
]
study_set_name = study_sets[0]['name']
study_set_dir0 = basedir + '/manual_sortings/buzsaki_petersen'
study_set_dir = mt.createSnapshot(study_set_dir0,
upload_to=upload_to,
upload_recursive=False,
download_recursive=False)
if not study_set_dir:
raise Exception('Failed to create snapshot of study set directory: ' +
study_set_dir0)
study_set_dir = study_set_dir + '.manual_buzsaki'
print('Using study set dir: ' + study_set_dir)
studies = []
recordings = []
study_name = 'manual_siprobe'
print('PREPARING: ' + study_name)
study_dir = study_set_dir
study0 = dict(name=study_name,
study_set=study_set_name,
directory=study_dir,
description='')
studies.append(study0)
dd = mt.readDir(study_dir)
dirnames = sorted(list(dd['dirs'].keys()))
for i, dsname in enumerate(dirnames):
dsdir = '{}/{}'.format(study_dir, dsname)
recordings.append(
dict(name=dsname,
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
def _handle_ndarray(x, *, opts, name, snapshot=False):
if np.issubdtype(x.dtype, np.number):
if snapshot:
with TemporaryDirectory() as f:
fname = '{}/{}.npy'.format(f, name)
writenpy(x, fname, dtype=npy_dtype_to_string(x.dtype))
return mt.createSnapshot(fname,
upload_to=opts.get('upload_to', None))
else:
if (x.size > 10000):
raise Exception(
'Array is too large to include in file (need to use snapshot). name={}, shape={}'
.format(name, x.shape))
list0 = [
_handle_list_or_val(val, opts=opts, name=name)
for val in x.tolist()
]
return list0
else:
list0 = [
_handle_list_or_val(val, opts=opts, name='{}.{}'.format(name, ind))
for ind, val in enumerate(x.tolist())
]
return list0
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
# Prepare the studies
studies, recordings, study_sets = prepare_manual_buzsaki_studies(
basedir=basedir)
print('Uploading files to kachery...')
for rec in recordings:
mt.createSnapshot(rec['directory'],
upload_to=upload_to,
upload_recursive=True)
print('Saving object...')
for ut in [upload_to]:
address = mt.saveObject(object=dict(studies=studies,
recordings=recordings,
study_sets=study_sets),
key=dict(name='spikeforest_recording_group',
group_name=group_name),
upload_to=ut)
if not address:
raise Exception('Problem uploading object to {}'.format(ut))
output_fname = 'key://pairio/spikeforest/spikeforest_recording_group.{}.json'.format(
group_name)
study=study_name,
directory=dsdir,
firings_true=dsdir + '/firings_true.mda',
index_within_study=i,
description='One of the recordings in the {} study'.format(
study_name)))
return studies, recordings, study_sets
# Prepare the studies
studies, recordings, study_sets = prepare_synth_visapy_studies(basedir=basedir)
print('Uploading files to kachery...')
for rec in recordings:
mt.createSnapshot(rec['directory'],
upload_to=upload_to,
upload_recursive=True)
if rec['index_within_study'] == 0:
mt.createSnapshot(rec['directory'],
upload_to=upload_public_to,
upload_recursive=True)
rec['public'] = True
mt.createSnapshot(rec['directory'] + '/geom.csv',
upload_to=upload_public_to)
mt.createSnapshot(rec['directory'] + '/params.json',
upload_to=upload_public_to)
print('Saving object...')
for ut in [upload_to, upload_public_to]:
address = mt.saveObject(object=dict(studies=studies,
recordings=recordings,
def apply_sorters_to_recordings(*, label, sorters, recordings, studies, study_sets, output_id=None, output_path=None, job_timeout=60 * 20, upload_to=None, skip_failing=None):
# Summarize the recordings
mtlogging.sublog('summarize-recordings')
recordings = sa.summarize_recordings(
recordings=recordings,
compute_resource='default',
label='Summarize recordings ({})'.format(label),
upload_to=upload_to
)
# Run the spike sorting
mtlogging.sublog('sorting')
sorting_results = sa.multi_sort_recordings(
sorters=sorters,
recordings=recordings,
label='Sort recordings ({})'.format(label),
job_timeout=job_timeout,
upload_to=upload_to,
skip_failing=skip_failing
)
# Summarize the sortings
mtlogging.sublog('summarize-sortings')
sorting_results = sa.summarize_sortings(
sortings=sorting_results,
compute_resource='default',
label='Summarize sortings ({})'.format(label)
)
# Compare with ground truth
mtlogging.sublog('compare-with-truth')
sorting_results = sa.compare_sortings_with_truth(
sortings=sorting_results,
compute_resource='default',
label='Compare with truth ({})'.format(label),
upload_to=upload_to
)
# Aggregate the results
mtlogging.sublog('aggregate')
aggregated_sorting_results = sa.aggregate_sorting_results(
studies, recordings, sorting_results)
output_object = dict(
studies=studies,
recordings=recordings,
study_sets=study_sets,
sorting_results=sorting_results,
aggregated_sorting_results=mt.saveObject(
object=aggregated_sorting_results, upload_to=upload_to)
)
# Save the output
if output_id:
print('Saving the output')
mtlogging.sublog('save-output')
mt.saveObject(
key=dict(
name='spikeforest_results'
),
subkey=output_id,
object=output_object,
upload_to=upload_to
)
if output_path:
print('Saving the output to {}'.format(output_path))
mtlogging.sublog('save-output-path')
address = mt.saveObject(output_object, upload_to=upload_to)
if not address:
raise Exception('Problem saving output object.')
if not mt.createSnapshot(path=address, dest_path=output_path):
raise Exception('Problem saving output to {}'.format(output_path))
mtlogging.sublog('show-output-summary')
for sr in aggregated_sorting_results['study_sorting_results']:
study_name = sr['study']
sorter_name = sr['sorter']
n1 = np.array(sr['num_matches'])
n2 = np.array(sr['num_false_positives'])
n3 = np.array(sr['num_false_negatives'])
accuracies = n1 / (n1 + n2 + n3)
avg_accuracy = np.mean(accuracies)
txt = 'STUDY: {}, SORTER: {}, AVG ACCURACY: {}'.format(
study_name, sorter_name, avg_accuracy)
print(txt)
def main():
mt.configDownloadFrom('spikeforest.public')
templates_path = 'sha1dir://95dba567b5168bacb480411ca334ffceb96b8c19.2019-06-11.templates'
recordings_path = 'recordings_out'
tempgen_tetrode = templates_path + '/templates_tetrode.h5'
tempgen_neuronexus = templates_path + '/templates_neuronexus.h5'
tempgen_neuropixels = templates_path + '/templates_neuropixels.h5'
tempgen_neuronexus_drift = templates_path + '/templates_neuronexus_drift.h5'
noise_level = [10, 20]
duration = 600
bursting = [False, True]
nrec = 2 # change this to 10
ei_ratio = 0.8
rec_dict = {
'tetrode': {
'ncells': [10, 20],
'tempgen': tempgen_tetrode,
'drifting': False
},
'neuronexus': {
'ncells': [10, 20, 40],
'tempgen': tempgen_neuronexus,
'drifting': False
},
'neuropixels': {
'ncells': [20, 40, 60],
'tempgen': tempgen_neuropixels,
'drifting': False
},
'neuronexus_drift': {
'ncells': [10, 20, 40],
'tempgen': tempgen_neuronexus_drift,
'drifting': True
}
}
# optional: if drifting change drift velocity
# recording_params['recordings']['drift_velocity] = ...
# Generate and save recordings
if os.path.exists(recordings_path):
shutil.rmtree(recordings_path)
os.mkdir(recordings_path)
# Set up slurm configuration
slurm_working_dir = 'tmp_slurm_job_handler_' + _random_string(5)
job_handler = mlpr.SlurmJobHandler(working_dir=slurm_working_dir)
use_slurm = True
job_timeout = 3600 * 4
if use_slurm:
job_handler.addBatchType(name='default',
num_workers_per_batch=4,
num_cores_per_job=6,
time_limit_per_batch=job_timeout * 3,
use_slurm=True,
max_simultaneous_batches=20,
additional_srun_opts=['-p ccm'])
else:
job_handler.addBatchType(
name='default',
num_workers_per_batch=multiprocessing.cpu_count(),
num_cores_per_job=2,
max_simultaneous_batches=1,
use_slurm=False)
with mlpr.JobQueue(job_handler=job_handler) as JQ:
results_to_write = []
for rec_type in rec_dict.keys():
study_set_name = 'SYNTH_MEAREC_{}'.format(rec_type.upper())
os.mkdir(recordings_path + '/' + study_set_name)
params = dict()
params['duration'] = duration
params['drifting'] = rec_dict[rec_type]['drifting']
# reduce minimum distance for dense recordings
params['min_dist'] = 15
for ncells in rec_dict[rec_type]['ncells']:
# changing number of cells
n_exc = int(ei_ratio *
10) # intentionally replaced nrec by 10 here
params['n_exc'] = n_exc
params['n_inh'] = ncells - n_exc
for n in noise_level:
# changing noise level
params['noise_level'] = n
for b in bursting:
bursting_str = ''
if b:
bursting_str = '_bursting'
study_name = 'synth_mearec_{}_noise{}_K{}{}'.format(
rec_type, n, ncells, bursting_str)
os.mkdir(recordings_path + '/' + study_set_name + '/' +
study_name)
for i in range(nrec):
# set random seeds
params[
'seed'] = i # intentionally doing it this way
# changing bursting and shape modulation
print('Generating', rec_type, 'recording with',
ncells, 'noise level', n, 'bursting', b)
params['bursting'] = b
params['shape_mod'] = b
templates0 = mt.realizeFile(
path=rec_dict[rec_type]['tempgen'])
result0 = GenerateMearecRecording.execute(
**params,
templates_in=templates0,
recording_out=dict(ext='.h5'))
mda_output_folder = recordings_path + '/' + study_set_name + '/' + study_name + '/' + '{}'.format(
i)
results_to_write.append(
dict(result=result0,
mda_output_folder=mda_output_folder))
JQ.wait()
for x in results_to_write:
result0: mlpr.MountainJobResult = x['result']
mda_output_folder = x['mda_output_folder']
path = mt.realizeFile(path=result0.outputs['recording_out'])
recording = se.MEArecRecordingExtractor(recording_path=path)
sorting_true = se.MEArecSortingExtractor(recording_path=path)
se.MdaRecordingExtractor.write_recording(
recording=recording, save_path=mda_output_folder)
se.MdaSortingExtractor.write_sorting(sorting=sorting_true,
save_path=mda_output_folder +
'/firings_true.mda')
if result0.console_out:
mt.realizeFile(path=result0.console_out,
dest_path=mda_output_folder +
'.console_out.txt')
if result0.runtime_info:
mt.saveObject(object=result0.runtime_info,
dest_path=mda_output_folder +
'.runtime_info.json')
print('Creating and uploading snapshot...')
sha1dir_path = mt.createSnapshot(path=recordings_path,
upload_to='spikeforest.public',
upload_recursive=False)
# sha1dir_path = mt.createSnapshot(path=recordings_path, upload_to='spikeforest.kbucket', upload_recursive=True)
print(sha1dir_path)
def main():
parser = argparse.ArgumentParser(description=help_txt, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--output_ids', help='Comma-separated list of IDs of the analysis outputs to include in the website.', required=False, default=None)
parser.add_argument('--upload_to', help='Optional kachery to upload to', required=False, default=None)
parser.add_argument('--dest_key_path', help='Optional destination key path', required=False, default=None)
args = parser.parse_args()
if args.upload_to:
upload_to = args.upload_to.split(',')
else:
upload_to = None
mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public'])
if args.output_ids is not None:
output_ids = args.output_ids.split(',')
else:
output_ids = [
'paired_boyden32c',
'paired_crcns',
'paired_mea64c',
'paired_kampff',
'paired_monotrode',
'synth_monotrode',
'synth_bionet',
'synth_magland',
'manual_franklab',
'synth_mearec_neuronexus',
'synth_mearec_tetrode',
# 'SYNTH_MEAREC_TETRODE_b',
# 'SYNTH_MEAREC_NEURONEXUS_b',
# 'SYNTH_MEAREC_NEUROPIXELS_b',
'synth_visapy',
'hybrid_janelia'
]
print('Using output ids: ', output_ids)
sorters_to_include = set([
'HerdingSpikes2',
'IronClust',
'IronClust1',
'IronClust2',
'IronClust3',
'IronClust4',
'IronClust5',
'IronClust6',
'IronClust7',
'IronClust8',
'JRClust',
'KiloSort',
'KiloSort2',
'Klusta',
'MountainSort4',
'SpykingCircus',
'Tridesclous',
'Waveclus',
# 'Yass'
])
print('******************************** LOADING ANALYSIS OUTPUT OBJECTS...')
studies = []
study_sets = []
recordings = []
sorting_results = []
for output_id in output_ids:
print('Loading output object: {}'.format(output_id))
output_path = ('key://pairio/spikeforest/spikeforest_analysis_results.{}.json').format(output_id)
obj = mt.loadObject(path=output_path)
if obj is not None:
studies = studies + obj['studies']
print(obj.keys())
study_sets = study_sets + obj.get('study_sets', [])
recordings = recordings + obj['recordings']
sorting_results = sorting_results + obj['sorting_results']
else:
raise Exception('Unable to load: {}'.format(output_path))
# ALGORITHMS
print('******************************** ASSEMBLING ALGORITHMS...')
algorithms_by_processor_name = dict()
Algorithms = []
basepath = '../../spikeforest/spikeforestsorters/descriptions'
repo_base_url = 'https://github.com/flatironinstitute/spikeforest/blob/master'
for item in os.listdir(basepath):
if item.endswith('.md'):
alg = frontmatter.load(basepath + '/' + item).to_dict()
alg['markdown_link'] = repo_base_url + '/spikeforest/spikeforestsorters/descriptions/' + item
alg['markdown'] = alg['content']
del alg['content']
if 'processor_name' in alg:
algorithms_by_processor_name[alg['processor_name']] = alg
Algorithms.append(alg)
print([alg['label'] for alg in Algorithms])
# # STUDIES
# print('******************************** ASSEMBLING STUDIES...')
# studies_by_name = dict()
# for study in studies:
# studies_by_name[study['name']] = study
# study['recordings'] = []
# for recording in recordings:
# studies_by_name[recording['studyName']]['recordings'].append(dict(
# name=recording['name'],
# studyName=recording['study_name'],
# directory=recording['directory'],
# ))
Studies = []
for study in studies:
Studies.append(dict(
name=study['name'],
studySet=study['study_set'],
description=study['description'],
recordings=[]
# the following can be obtained from the other collections
# numRecordings, sorters, etc...
))
print([S['name'] for S in Studies])
print('******************************** ASSEMBLING STUDY SETS...')
study_sets_by_name = dict()
for study_set in study_sets:
study_sets_by_name[study_set['name']] = study_set
study_set['studies'] = []
studies_by_name = dict()
for study in studies:
study0 = dict(
name=study['name'],
studySetName=study['study_set'],
recordings=[]
)
study_sets_by_name[study['study_set']]['studies'].append(study0)
studies_by_name[study0['name']] = study0
for recording in recordings:
true_units_info = mt.loadObject(path=recording['summary']['true_units_info'])
if not true_units_info:
print(recording['summary']['true_units_info'])
raise Exception('Unable to load true_units_info for recording {}'.format(recording['name']))
recording0 = dict(
name=recording['name'],
studyName=recording['study'],
studySetName=studies_by_name[recording['study']]['studySetName'],
directory=recording['directory'],
firingsTrue=recording['firings_true'],
sampleRateHz=recording['summary']['computed_info']['samplerate'],
numChannels=recording['summary']['computed_info']['num_channels'],
durationSec=recording['summary']['computed_info']['duration_sec'],
numTrueUnits=len(true_units_info),
spikeSign=-1 # TODO: set this properly
)
studies_by_name[recording0['studyName']]['recordings'].append(recording0)
StudySets = []
for study_set in study_sets:
StudySets.append(study_set)
print(StudySets)
# SORTING RESULTS
print('******************************** SORTING RESULTS...')
SortingResults = []
for sr in sorting_results:
if sr['sorter']['name'] in sorters_to_include:
SR = dict(
recordingName=sr['recording']['name'],
studyName=sr['recording']['study'],
sorterName=sr['sorter']['name'],
recordingDirectory=sr['recording']['directory'],
firingsTrue=sr['recording']['firings_true'],
consoleOut=sr['console_out'],
container=sr['container'],
cpuTimeSec=sr['execution_stats'].get('elapsed_sec', None),
returnCode=sr['execution_stats'].get('retcode', 0), # TODO: in future, the default should not be 0 -- rather it should be a required field of execution_stats
timedOut=sr['execution_stats'].get('timed_out', False),
startTime=datetime.fromtimestamp(sr['execution_stats'].get('start_time')).isoformat(),
endTime=datetime.fromtimestamp(sr['execution_stats'].get('end_time')).isoformat()
)
if sr.get('firings', None):
SR['firings'] = sr['firings']
if not sr.get('comparison_with_truth', None):
print('Warning: comparison with truth not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name']))
print('Console output is here: ' + sr['console_out'])
else:
print('Warning: firings not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name']))
print('Console output is here: ' + sr['console_out'])
SortingResults.append(SR)
# print('Num unit results:', len(UnitResults))
# SORTERS
print('******************************** ASSEMBLING SORTERS...')
sorters_by_name = dict()
for sr in sorting_results:
sorters_by_name[sr['sorter']['name']] = sr['sorter']
Sorters = []
sorter_names = sorted(list(sorters_by_name.keys()))
sorter_names = [sorter_name for sorter_name in sorter_names if sorter_name in sorters_to_include]
for sorter_name in sorter_names:
sorter = sorters_by_name[sorter_name]
alg = algorithms_by_processor_name.get(sorter['processor_name'], dict())
alg_label = alg.get('label', sorter['processor_name'])
if sorter['name'] in sorters_to_include:
Sorters.append(dict(
name=sorter['name'],
algorithmName=alg_label,
processorName=sorter['processor_name'],
processorVersion='0', # jfm to provide this
sortingParameters=sorter['params']
))
print([S['name'] + ':' + S['algorithmName'] for S in Sorters])
# STUDY ANALYSIS RESULTS
print('******************************** ASSEMBLING STUDY ANALYSIS RESULTS...')
StudyAnalysisResults = [
_assemble_study_analysis_result(
study_name=study['name'],
study_set_name=study['study_set'],
recordings=recordings,
sorting_results=sorting_results,
sorter_names=sorter_names
)
for study in studies
]
# GENERAL
print('******************************** ASSEMBLING GENERAL INFO...')
General = [dict(
dateUpdated=datetime.now().isoformat(),
packageVersions=dict(
mountaintools=pkg_resources.get_distribution("mountaintools").version,
spikeforest=pkg_resources.get_distribution("spikeforest").version
)
)]
obj = dict(
mode='spike-front',
StudySets=StudySets,
# TrueUnits=TrueUnits,
# UnitResults=UnitResults,
SortingResults=SortingResults,
Sorters=Sorters,
Algorithms=Algorithms,
StudyAnalysisResults=StudyAnalysisResults,
General=General
)
address = mt.saveObject(object=obj)
mt.createSnapshot(path=address, upload_to=upload_to, dest_path=args.dest_key_path)
