def main(): parser = argparse.ArgumentParser( description='update spike-front analysis history') parser.add_argument( '--delete-and-init', help='Use this only the first time. Will wipe out the history.', action='store_true') parser.add_argument('--show', help='Show the analysis history.', action='store_true') args = parser.parse_args() history_path = 'key://pairio/spikeforest/spike-front-analysis-history.json' results_path = 'key://pairio/spikeforest/spike-front-results.json' mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) if args.delete_and_init: print('Initializing analysis history...') mt.createSnapshot(mt.saveObject(object=dict(analyses=[])), dest_path=history_path, upload_to='spikeforest.public') print('Done.') return print('Loading history...') history = mt.loadObject(path=history_path) assert history, 'Unable to load history' if args.show: for aa in history['analyses']: print( '===============================================================' ) print('ANALYSIS: {}'.format(aa['General']['dateUpdated'])) print('PATH: {}'.format(aa['path'])) print(json.dumps(aa['General'], indent=4)) print('') return print('Loading current results...') spike_front_results = mt.loadObject(path=results_path) assert spike_front_results, 'Unable to load current results' sha1_url = mt.saveObject(object=spike_front_results, basename='analysis.json') for aa in history['analyses']: if aa['path'] == sha1_url: print('Analysis already stored in history.') return history['analyses'].append( dict(path=sha1_url, General=spike_front_results['General'][0])) print('Saving updated history to {}'.format(history_path)) mt.createSnapshot(mt.saveObject(object=history), dest_path=history_path, upload_to='spikeforest.public') print('Done.')
def do_sorting_test(sorting_processor, params, recording_dir, assert_avg_accuracy, container='default'): mt.configDownloadFrom('spikeforest.kbucket') recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) sorting = sorting_processor.execute(recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=True) comparison = sa.GenSortingComparisonTable.execute( firings=sorting.outputs['firings_out'], firings_true=recdir + '/firings_true.mda', units_true=[], json_out={'ext': '.json'}, html_out={'ext': '.html'}, _container=None, _force_run=True) X = mt.loadObject(path=comparison.outputs['json_out']) accuracies = [float(a['accuracy']) for a in X.values()] avg_accuracy = np.mean(accuracies) print('Average accuracy: {}'.format(avg_accuracy)) assert (avg_accuracy >= assert_avg_accuracy)
def main(): path = mt.createSnapshot(path='recordings_out') mt.configDownloadFrom('spikeforest.public') X = mt.readDir(path) for study_set_name, d in X['dirs'].items(): study_sets = [] studies = [] recordings = [] study_sets.append(dict( name=study_set_name + '_b', info=dict(), description='' )) for study_name, d2 in d['dirs'].items(): study_dir = path + '/' + study_set_name + '/' + study_name study0 = dict( name=study_name, study_set=study_set_name + '_b', directory=study_dir, description='' ) studies.append(study0) index_within_study = 0 for recording_name, d3 in d2['dirs'].items(): recdir = study_dir + '/' + recording_name recordings.append(dict( name=recording_name, study=study_name, directory=recdir, firings_true=recdir + '/firings_true.mda', index_within_study=index_within_study, description='One of the recordings in the {} study'.format(study_name) )) index_within_study = index_within_study + 1 print('Saving object...') group_name = study_set_name + '_b' address = mt.saveObject( object=dict( studies=studies, recordings=recordings, study_sets=study_sets ), key=dict(name='spikeforest_recording_group', group_name=group_name), upload_to='spikeforest.public' ) if not address: raise Exception('Problem uploading object to kachery') output_fname = 'key://pairio/spikeforest/spikeforest_recording_group.{}.json'.format(group_name) print('Saving output to {}'.format(output_fname)) mt.createSnapshot(path=address, dest_path=output_fname)
def mem_profile_test(sorting_processor, params, recording_dir, container='default', _keep_temp_files=True): mt.configDownloadFrom('spikeforest.public') params['fMemProfile'] = True recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) sorting = sorting_processor.execute( recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=True, _keep_temp_files=_keep_temp_files )
def prepare_synth_magland_studies(*, basedir): study_sets = [load_study_set_from_md('descriptions/spf_synth_magland.md')] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/magland_synth' print('Creating snapshot of study set directory...') study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) mt.createSnapshot(study_set_dir0, upload_to=upload_public_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) study_set_dir = study_set_dir + '.synth_magland' print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] names = [] names = names + ['noise10_K10_C4', 'noise10_K10_C8'] names = names + ['noise10_K20_C4', 'noise10_K20_C8'] names = names + ['noise20_K10_C4', 'noise20_K10_C8'] names = names + ['noise20_K20_C4', 'noise20_K20_C8'] description = mt.loadText(path=study_set_dir + '/readme.txt') for name in names: print('PREPARING: ' + name) study_name = 'synth_magland_' + name study_dir = study_set_dir + '/datasets_' + name study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description=description) studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) recordings.append( dict( name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def prepare_synth_monotrode_studies(*, basedir): study_sets = [ load_study_set_from_md('descriptions/spf_synth_monotrode.md') ] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/waveclus_synth' study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) mt.createSnapshot(study_set_dir0, upload_to=upload_public_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) study_set_dir = study_set_dir + '.synth_monotrode' print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] names = [ 'sim2_2K10', 'sim2_11K20', 'quiroga_easy1', 'quiroga_easy2', 'quiroga_difficult1', 'quiroga_difficult2' ] for name in names: study_name = 'neurocube_' + name print('PREPARING: ' + study_name) study_dir = '{}/{}'.format(study_set_dir, name) study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description='') studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) recordings.append( dict( name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def prepare_hybrid_janelia_studies(*, basedir): study_sets = [load_study_set_from_md('descriptions/spf_hybrid_janelia.md')] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/hybrid_synth' print('Creating snapshot of {}'.format(study_set_dir0)) study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) mt.createSnapshot(study_set_dir0, upload_to=upload_public_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) study_set_dir = study_set_dir + '.hybrid_janelia' print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] names = [ 'drift_siprobe', 'static_siprobe', 'drift_tetrode', 'static_tetrode' ] for name in names: study_name = 'hybrid_' + name print('PREPARING: ' + study_name) study_dir = '{}/{}'.format(study_set_dir, name) study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description='') studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) print(dsdir) recordings.append( dict( name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def prepare_paired_monotrode_studies(*, basedir): study_sets = [ load_study_set_from_md('descriptions/spf_paired_monotrode.md') ] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/paired_monotrode' study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) mt.createSnapshot(study_set_dir0, upload_to=upload_public_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] names = ['boyden32c', 'kampff', 'crcns', 'mea64c'] for name in names: study_name = 'paired_monotrode_' + name print('PREPARING: ' + study_name) study_dir = study_set_dir + '/' + name study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description='') studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) recordings.append( dict( name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def do_prepare(recording_group, study_name): print(recording_group, study_name) X = mt.loadObject( path="key://pairio/spikeforest/spikeforest_recording_group.{}.json". format(recording_group)) studies = [y for y in X['studies'] if (y['name'] == study_name)] recordings = [y for y in X['recordings'] if y['study'] == study_name] recordings = recordings[0:1] study_sets = X['study_sets'] Y = dict(studies=studies, recordings=recordings, study_sets=study_sets) address = mt.saveObject(object=Y) assert address is not None dest_path = 'key://pairio/spikeforest/spikeforest_recording_group.test_{}.json'.format( recording_group) print(dest_path) mt.createSnapshot(path=address, upload_to='spikeforest.kbucket', dest_path=dest_path)
def main(): parser = argparse.ArgumentParser(description=help_txt, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('path', help='Path to the assembled website data in .json format') args = parser.parse_args() print('Loading spike-front results object...') with open(args.path, 'r') as f: obj = json.load(f) SortingResults = obj['SortingResults'] for sr in SortingResults: print(sr['studyName'], sr['recordingName']) console_out_fname = ka.load_file(sr['consoleOut']) mt.createSnapshot(path=console_out_fname, upload_to='spikeforest.public') if sr.get('firings', None) is not None: firings_fname = ka.load_file(sr['firings']) mt.createSnapshot(path=firings_fname, upload_to='spikeforest.public')
def do_sorting_test( sorting_processor, params, recording_dir, container='default', force_run=True, _keep_temp_files=False ): mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) timer = time.time() sorting = sorting_processor.execute( recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=force_run, _keep_temp_files=_keep_temp_files ) elapsed = time.time() - timer print('################ ELAPSED for sorting (sec): {}'.format(elapsed)) timer = time.time() comparison = sa.GenSortingComparisonTable.execute( firings=sorting.outputs['firings_out'], firings_true=recdir + '/firings_true.mda', units_true=[], json_out={'ext': '.json'}, html_out={'ext': '.html'}, _container='default', _force_run=True ) elapsed = time.time() - timer print('################ ELAPSED for comparison (sec): {}'.format(elapsed)) X = mt.loadObject(path=comparison.outputs['json_out']) accuracies = [float(a['accuracy']) for a in X.values()] avg_accuracy = np.mean(accuracies) print('Average accuracy: {}'.format(avg_accuracy))
def _np_snapshot(X): _, fname = tempfile.mkstemp(suffix='.npy') try: np.save(fname, X) ret = mt.createSnapshot(fname) except: raise finally: if os.path.exists(fname): os.unlink(fname) return ret
def prepare_test_reliability_studies(*, basedir): study_sets = [ load_study_set_from_md('descriptions/spf_test_reliability.md') ] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/test_reliability' print('Creating snapshot of study set directory...') study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) # study_set_dir = study_set_dir + '.test_reliability' print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] names = [ '001_synth', '002_synth', '003_synth', '004_synth', '005_synth', '006_synth', '007_synth', '008_synth', '009_synth', '010_synth' ] # description = mt.loadText(path=study_set_dir + '/readme.txt') for name in names: print('PREPARING: ' + name) study_name = 'test_reliability_' + name study_dir = study_set_dir + '/' + name study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description='') studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) recordings.append( dict( name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def prepare_manual_buzsaki_studies(*, basedir): study_sets = [ load_study_set_from_md('descriptions/spf_manual_buzsakilab.md') ] study_set_name = study_sets[0]['name'] study_set_dir0 = basedir + '/manual_sortings/buzsaki_petersen' study_set_dir = mt.createSnapshot(study_set_dir0, upload_to=upload_to, upload_recursive=False, download_recursive=False) if not study_set_dir: raise Exception('Failed to create snapshot of study set directory: ' + study_set_dir0) study_set_dir = study_set_dir + '.manual_buzsaki' print('Using study set dir: ' + study_set_dir) studies = [] recordings = [] study_name = 'manual_siprobe' print('PREPARING: ' + study_name) study_dir = study_set_dir study0 = dict(name=study_name, study_set=study_set_name, directory=study_dir, description='') studies.append(study0) dd = mt.readDir(study_dir) dirnames = sorted(list(dd['dirs'].keys())) for i, dsname in enumerate(dirnames): dsdir = '{}/{}'.format(study_dir, dsname) recordings.append( dict(name=dsname, study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets
def _handle_ndarray(x, *, opts, name, snapshot=False): if np.issubdtype(x.dtype, np.number): if snapshot: with TemporaryDirectory() as f: fname = '{}/{}.npy'.format(f, name) writenpy(x, fname, dtype=npy_dtype_to_string(x.dtype)) return mt.createSnapshot(fname, upload_to=opts.get('upload_to', None)) else: if (x.size > 10000): raise Exception( 'Array is too large to include in file (need to use snapshot). name={}, shape={}' .format(name, x.shape)) list0 = [ _handle_list_or_val(val, opts=opts, name=name) for val in x.tolist() ] return list0 else: list0 = [ _handle_list_or_val(val, opts=opts, name='{}.{}'.format(name, ind)) for ind, val in enumerate(x.tolist()) ] return list0
directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets # Prepare the studies studies, recordings, study_sets = prepare_manual_buzsaki_studies( basedir=basedir) print('Uploading files to kachery...') for rec in recordings: mt.createSnapshot(rec['directory'], upload_to=upload_to, upload_recursive=True) print('Saving object...') for ut in [upload_to]: address = mt.saveObject(object=dict(studies=studies, recordings=recordings, study_sets=study_sets), key=dict(name='spikeforest_recording_group', group_name=group_name), upload_to=ut) if not address: raise Exception('Problem uploading object to {}'.format(ut)) output_fname = 'key://pairio/spikeforest/spikeforest_recording_group.{}.json'.format( group_name)
study=study_name, directory=dsdir, firings_true=dsdir + '/firings_true.mda', index_within_study=i, description='One of the recordings in the {} study'.format( study_name))) return studies, recordings, study_sets # Prepare the studies studies, recordings, study_sets = prepare_synth_visapy_studies(basedir=basedir) print('Uploading files to kachery...') for rec in recordings: mt.createSnapshot(rec['directory'], upload_to=upload_to, upload_recursive=True) if rec['index_within_study'] == 0: mt.createSnapshot(rec['directory'], upload_to=upload_public_to, upload_recursive=True) rec['public'] = True mt.createSnapshot(rec['directory'] + '/geom.csv', upload_to=upload_public_to) mt.createSnapshot(rec['directory'] + '/params.json', upload_to=upload_public_to) print('Saving object...') for ut in [upload_to, upload_public_to]: address = mt.saveObject(object=dict(studies=studies, recordings=recordings,
def apply_sorters_to_recordings(*, label, sorters, recordings, studies, study_sets, output_id=None, output_path=None, job_timeout=60 * 20, upload_to=None, skip_failing=None): # Summarize the recordings mtlogging.sublog('summarize-recordings') recordings = sa.summarize_recordings( recordings=recordings, compute_resource='default', label='Summarize recordings ({})'.format(label), upload_to=upload_to ) # Run the spike sorting mtlogging.sublog('sorting') sorting_results = sa.multi_sort_recordings( sorters=sorters, recordings=recordings, label='Sort recordings ({})'.format(label), job_timeout=job_timeout, upload_to=upload_to, skip_failing=skip_failing ) # Summarize the sortings mtlogging.sublog('summarize-sortings') sorting_results = sa.summarize_sortings( sortings=sorting_results, compute_resource='default', label='Summarize sortings ({})'.format(label) ) # Compare with ground truth mtlogging.sublog('compare-with-truth') sorting_results = sa.compare_sortings_with_truth( sortings=sorting_results, compute_resource='default', label='Compare with truth ({})'.format(label), upload_to=upload_to ) # Aggregate the results mtlogging.sublog('aggregate') aggregated_sorting_results = sa.aggregate_sorting_results( studies, recordings, sorting_results) output_object = dict( studies=studies, recordings=recordings, study_sets=study_sets, sorting_results=sorting_results, aggregated_sorting_results=mt.saveObject( object=aggregated_sorting_results, upload_to=upload_to) ) # Save the output if output_id: print('Saving the output') mtlogging.sublog('save-output') mt.saveObject( key=dict( name='spikeforest_results' ), subkey=output_id, object=output_object, upload_to=upload_to ) if output_path: print('Saving the output to {}'.format(output_path)) mtlogging.sublog('save-output-path') address = mt.saveObject(output_object, upload_to=upload_to) if not address: raise Exception('Problem saving output object.') if not mt.createSnapshot(path=address, dest_path=output_path): raise Exception('Problem saving output to {}'.format(output_path)) mtlogging.sublog('show-output-summary') for sr in aggregated_sorting_results['study_sorting_results']: study_name = sr['study'] sorter_name = sr['sorter'] n1 = np.array(sr['num_matches']) n2 = np.array(sr['num_false_positives']) n3 = np.array(sr['num_false_negatives']) accuracies = n1 / (n1 + n2 + n3) avg_accuracy = np.mean(accuracies) txt = 'STUDY: {}, SORTER: {}, AVG ACCURACY: {}'.format( study_name, sorter_name, avg_accuracy) print(txt)
def main(): mt.configDownloadFrom('spikeforest.public') templates_path = 'sha1dir://95dba567b5168bacb480411ca334ffceb96b8c19.2019-06-11.templates' recordings_path = 'recordings_out' tempgen_tetrode = templates_path + '/templates_tetrode.h5' tempgen_neuronexus = templates_path + '/templates_neuronexus.h5' tempgen_neuropixels = templates_path + '/templates_neuropixels.h5' tempgen_neuronexus_drift = templates_path + '/templates_neuronexus_drift.h5' noise_level = [10, 20] duration = 600 bursting = [False, True] nrec = 2 # change this to 10 ei_ratio = 0.8 rec_dict = { 'tetrode': { 'ncells': [10, 20], 'tempgen': tempgen_tetrode, 'drifting': False }, 'neuronexus': { 'ncells': [10, 20, 40], 'tempgen': tempgen_neuronexus, 'drifting': False }, 'neuropixels': { 'ncells': [20, 40, 60], 'tempgen': tempgen_neuropixels, 'drifting': False }, 'neuronexus_drift': { 'ncells': [10, 20, 40], 'tempgen': tempgen_neuronexus_drift, 'drifting': True } } # optional: if drifting change drift velocity # recording_params['recordings']['drift_velocity] = ... # Generate and save recordings if os.path.exists(recordings_path): shutil.rmtree(recordings_path) os.mkdir(recordings_path) # Set up slurm configuration slurm_working_dir = 'tmp_slurm_job_handler_' + _random_string(5) job_handler = mlpr.SlurmJobHandler(working_dir=slurm_working_dir) use_slurm = True job_timeout = 3600 * 4 if use_slurm: job_handler.addBatchType(name='default', num_workers_per_batch=4, num_cores_per_job=6, time_limit_per_batch=job_timeout * 3, use_slurm=True, max_simultaneous_batches=20, additional_srun_opts=['-p ccm']) else: job_handler.addBatchType( name='default', num_workers_per_batch=multiprocessing.cpu_count(), num_cores_per_job=2, max_simultaneous_batches=1, use_slurm=False) with mlpr.JobQueue(job_handler=job_handler) as JQ: results_to_write = [] for rec_type in rec_dict.keys(): study_set_name = 'SYNTH_MEAREC_{}'.format(rec_type.upper()) os.mkdir(recordings_path + '/' + study_set_name) params = dict() params['duration'] = duration params['drifting'] = rec_dict[rec_type]['drifting'] # reduce minimum distance for dense recordings params['min_dist'] = 15 for ncells in rec_dict[rec_type]['ncells']: # changing number of cells n_exc = int(ei_ratio * 10) # intentionally replaced nrec by 10 here params['n_exc'] = n_exc params['n_inh'] = ncells - n_exc for n in noise_level: # changing noise level params['noise_level'] = n for b in bursting: bursting_str = '' if b: bursting_str = '_bursting' study_name = 'synth_mearec_{}_noise{}_K{}{}'.format( rec_type, n, ncells, bursting_str) os.mkdir(recordings_path + '/' + study_set_name + '/' + study_name) for i in range(nrec): # set random seeds params[ 'seed'] = i # intentionally doing it this way # changing bursting and shape modulation print('Generating', rec_type, 'recording with', ncells, 'noise level', n, 'bursting', b) params['bursting'] = b params['shape_mod'] = b templates0 = mt.realizeFile( path=rec_dict[rec_type]['tempgen']) result0 = GenerateMearecRecording.execute( **params, templates_in=templates0, recording_out=dict(ext='.h5')) mda_output_folder = recordings_path + '/' + study_set_name + '/' + study_name + '/' + '{}'.format( i) results_to_write.append( dict(result=result0, mda_output_folder=mda_output_folder)) JQ.wait() for x in results_to_write: result0: mlpr.MountainJobResult = x['result'] mda_output_folder = x['mda_output_folder'] path = mt.realizeFile(path=result0.outputs['recording_out']) recording = se.MEArecRecordingExtractor(recording_path=path) sorting_true = se.MEArecSortingExtractor(recording_path=path) se.MdaRecordingExtractor.write_recording( recording=recording, save_path=mda_output_folder) se.MdaSortingExtractor.write_sorting(sorting=sorting_true, save_path=mda_output_folder + '/firings_true.mda') if result0.console_out: mt.realizeFile(path=result0.console_out, dest_path=mda_output_folder + '.console_out.txt') if result0.runtime_info: mt.saveObject(object=result0.runtime_info, dest_path=mda_output_folder + '.runtime_info.json') print('Creating and uploading snapshot...') sha1dir_path = mt.createSnapshot(path=recordings_path, upload_to='spikeforest.public', upload_recursive=False) # sha1dir_path = mt.createSnapshot(path=recordings_path, upload_to='spikeforest.kbucket', upload_recursive=True) print(sha1dir_path)
def main(): parser = argparse.ArgumentParser(description=help_txt, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--output_ids', help='Comma-separated list of IDs of the analysis outputs to include in the website.', required=False, default=None) parser.add_argument('--upload_to', help='Optional kachery to upload to', required=False, default=None) parser.add_argument('--dest_key_path', help='Optional destination key path', required=False, default=None) args = parser.parse_args() if args.upload_to: upload_to = args.upload_to.split(',') else: upload_to = None mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) if args.output_ids is not None: output_ids = args.output_ids.split(',') else: output_ids = [ 'paired_boyden32c', 'paired_crcns', 'paired_mea64c', 'paired_kampff', 'paired_monotrode', 'synth_monotrode', 'synth_bionet', 'synth_magland', 'manual_franklab', 'synth_mearec_neuronexus', 'synth_mearec_tetrode', # 'SYNTH_MEAREC_TETRODE_b', # 'SYNTH_MEAREC_NEURONEXUS_b', # 'SYNTH_MEAREC_NEUROPIXELS_b', 'synth_visapy', 'hybrid_janelia' ] print('Using output ids: ', output_ids) sorters_to_include = set([ 'HerdingSpikes2', 'IronClust', 'IronClust1', 'IronClust2', 'IronClust3', 'IronClust4', 'IronClust5', 'IronClust6', 'IronClust7', 'IronClust8', 'JRClust', 'KiloSort', 'KiloSort2', 'Klusta', 'MountainSort4', 'SpykingCircus', 'Tridesclous', 'Waveclus', # 'Yass' ]) print('******************************** LOADING ANALYSIS OUTPUT OBJECTS...') studies = [] study_sets = [] recordings = [] sorting_results = [] for output_id in output_ids: print('Loading output object: {}'.format(output_id)) output_path = ('key://pairio/spikeforest/spikeforest_analysis_results.{}.json').format(output_id) obj = mt.loadObject(path=output_path) if obj is not None: studies = studies + obj['studies'] print(obj.keys()) study_sets = study_sets + obj.get('study_sets', []) recordings = recordings + obj['recordings'] sorting_results = sorting_results + obj['sorting_results'] else: raise Exception('Unable to load: {}'.format(output_path)) # ALGORITHMS print('******************************** ASSEMBLING ALGORITHMS...') algorithms_by_processor_name = dict() Algorithms = [] basepath = '../../spikeforest/spikeforestsorters/descriptions' repo_base_url = 'https://github.com/flatironinstitute/spikeforest/blob/master' for item in os.listdir(basepath): if item.endswith('.md'): alg = frontmatter.load(basepath + '/' + item).to_dict() alg['markdown_link'] = repo_base_url + '/spikeforest/spikeforestsorters/descriptions/' + item alg['markdown'] = alg['content'] del alg['content'] if 'processor_name' in alg: algorithms_by_processor_name[alg['processor_name']] = alg Algorithms.append(alg) print([alg['label'] for alg in Algorithms]) # # STUDIES # print('******************************** ASSEMBLING STUDIES...') # studies_by_name = dict() # for study in studies: # studies_by_name[study['name']] = study # study['recordings'] = [] # for recording in recordings: # studies_by_name[recording['studyName']]['recordings'].append(dict( # name=recording['name'], # studyName=recording['study_name'], # directory=recording['directory'], # )) Studies = [] for study in studies: Studies.append(dict( name=study['name'], studySet=study['study_set'], description=study['description'], recordings=[] # the following can be obtained from the other collections # numRecordings, sorters, etc... )) print([S['name'] for S in Studies]) print('******************************** ASSEMBLING STUDY SETS...') study_sets_by_name = dict() for study_set in study_sets: study_sets_by_name[study_set['name']] = study_set study_set['studies'] = [] studies_by_name = dict() for study in studies: study0 = dict( name=study['name'], studySetName=study['study_set'], recordings=[] ) study_sets_by_name[study['study_set']]['studies'].append(study0) studies_by_name[study0['name']] = study0 for recording in recordings: true_units_info = mt.loadObject(path=recording['summary']['true_units_info']) if not true_units_info: print(recording['summary']['true_units_info']) raise Exception('Unable to load true_units_info for recording {}'.format(recording['name'])) recording0 = dict( name=recording['name'], studyName=recording['study'], studySetName=studies_by_name[recording['study']]['studySetName'], directory=recording['directory'], firingsTrue=recording['firings_true'], sampleRateHz=recording['summary']['computed_info']['samplerate'], numChannels=recording['summary']['computed_info']['num_channels'], durationSec=recording['summary']['computed_info']['duration_sec'], numTrueUnits=len(true_units_info), spikeSign=-1 # TODO: set this properly ) studies_by_name[recording0['studyName']]['recordings'].append(recording0) StudySets = [] for study_set in study_sets: StudySets.append(study_set) print(StudySets) # SORTING RESULTS print('******************************** SORTING RESULTS...') SortingResults = [] for sr in sorting_results: if sr['sorter']['name'] in sorters_to_include: SR = dict( recordingName=sr['recording']['name'], studyName=sr['recording']['study'], sorterName=sr['sorter']['name'], recordingDirectory=sr['recording']['directory'], firingsTrue=sr['recording']['firings_true'], consoleOut=sr['console_out'], container=sr['container'], cpuTimeSec=sr['execution_stats'].get('elapsed_sec', None), returnCode=sr['execution_stats'].get('retcode', 0), # TODO: in future, the default should not be 0 -- rather it should be a required field of execution_stats timedOut=sr['execution_stats'].get('timed_out', False), startTime=datetime.fromtimestamp(sr['execution_stats'].get('start_time')).isoformat(), endTime=datetime.fromtimestamp(sr['execution_stats'].get('end_time')).isoformat() ) if sr.get('firings', None): SR['firings'] = sr['firings'] if not sr.get('comparison_with_truth', None): print('Warning: comparison with truth not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name'])) print('Console output is here: ' + sr['console_out']) else: print('Warning: firings not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name'])) print('Console output is here: ' + sr['console_out']) SortingResults.append(SR) # print('Num unit results:', len(UnitResults)) # SORTERS print('******************************** ASSEMBLING SORTERS...') sorters_by_name = dict() for sr in sorting_results: sorters_by_name[sr['sorter']['name']] = sr['sorter'] Sorters = [] sorter_names = sorted(list(sorters_by_name.keys())) sorter_names = [sorter_name for sorter_name in sorter_names if sorter_name in sorters_to_include] for sorter_name in sorter_names: sorter = sorters_by_name[sorter_name] alg = algorithms_by_processor_name.get(sorter['processor_name'], dict()) alg_label = alg.get('label', sorter['processor_name']) if sorter['name'] in sorters_to_include: Sorters.append(dict( name=sorter['name'], algorithmName=alg_label, processorName=sorter['processor_name'], processorVersion='0', # jfm to provide this sortingParameters=sorter['params'] )) print([S['name'] + ':' + S['algorithmName'] for S in Sorters]) # STUDY ANALYSIS RESULTS print('******************************** ASSEMBLING STUDY ANALYSIS RESULTS...') StudyAnalysisResults = [ _assemble_study_analysis_result( study_name=study['name'], study_set_name=study['study_set'], recordings=recordings, sorting_results=sorting_results, sorter_names=sorter_names ) for study in studies ] # GENERAL print('******************************** ASSEMBLING GENERAL INFO...') General = [dict( dateUpdated=datetime.now().isoformat(), packageVersions=dict( mountaintools=pkg_resources.get_distribution("mountaintools").version, spikeforest=pkg_resources.get_distribution("spikeforest").version ) )] obj = dict( mode='spike-front', StudySets=StudySets, # TrueUnits=TrueUnits, # UnitResults=UnitResults, SortingResults=SortingResults, Sorters=Sorters, Algorithms=Algorithms, StudyAnalysisResults=StudyAnalysisResults, General=General ) address = mt.saveObject(object=obj) mt.createSnapshot(path=address, upload_to=upload_to, dest_path=args.dest_key_path)