def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running TwoPhotonSeries') nwb_path = state.get('nwb_path', None) download_from = state.get('download_from', []) if not nwb_path: self._set_error('Missing nwb_path') return mt.configDownloadFrom(download_from) nwb_path2 = mt.realizeFile(nwb_path) if not nwb_path2: self._set_error('Unable to realize nwb file: {}'.format(nwb_path)) return self._set_status('running', 'Extracting .mp4 data') outputs = ExtractTwoPhotonSeriesMp4.execute(nwb_in=nwb_path2, mp4_out={'ext': '.mp4'}).outputs self._set_status('running', 'Reading .mp4 data') mp4_fname = mt.realizeFile(outputs['mp4_out']) with open(mp4_fname, 'rb') as f: video_data = f.read() self._set_status('running', 'Encoding .mp4 data') video_data_b64 = base64.b64encode(video_data).decode() video_url = 'data:video/mp4;base64,{}'.format(video_data_b64) self._set_status('running', 'Setting .mp4 data to python state') self.set_python_state(dict( video_url=video_url, status='finished', status_message='' ))
def main(): parser = argparse.ArgumentParser( description='update spike-front analysis history') parser.add_argument( '--delete-and-init', help='Use this only the first time. Will wipe out the history.', action='store_true') parser.add_argument('--show', help='Show the analysis history.', action='store_true') args = parser.parse_args() history_path = 'key://pairio/spikeforest/spike-front-analysis-history.json' results_path = 'key://pairio/spikeforest/spike-front-results.json' mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) if args.delete_and_init: print('Initializing analysis history...') mt.createSnapshot(mt.saveObject(object=dict(analyses=[])), dest_path=history_path, upload_to='spikeforest.public') print('Done.') return print('Loading history...') history = mt.loadObject(path=history_path) assert history, 'Unable to load history' if args.show: for aa in history['analyses']: print( '===============================================================' ) print('ANALYSIS: {}'.format(aa['General']['dateUpdated'])) print('PATH: {}'.format(aa['path'])) print(json.dumps(aa['General'], indent=4)) print('') return print('Loading current results...') spike_front_results = mt.loadObject(path=results_path) assert spike_front_results, 'Unable to load current results' sha1_url = mt.saveObject(object=spike_front_results, basename='analysis.json') for aa in history['analyses']: if aa['path'] == sha1_url: print('Analysis already stored in history.') return history['analyses'].append( dict(path=sha1_url, General=spike_front_results['General'][0])) print('Saving updated history to {}'.format(history_path)) mt.createSnapshot(mt.saveObject(object=history), dest_path=history_path, upload_to='spikeforest.public') print('Done.')
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) path = state.get('path', None) if path: self.set_python_state( dict(status_message='Realizing file: {}'.format(path))) if path.endswith('.csv'): path2 = mt.realizeFile(path) if not path2: self.set_python_state( dict( status='error', status_message='Unable to realize file: {}'.format( path))) return self.set_python_state(dict(status_message='Loading locatoins')) x = np.genfromtxt(path2, delimiter=',') locations = x.T num_elec = x.shape[0] labels = ['{}'.format(a) for a in range(1, num_elec + 1)] else: raise Exception('Unexpected file type for {}'.format(path)) else: locations = [[0, 0], [1, 0], [1, 1], [2, 1]] labels = ['1', '2', '3', '4'] state = dict() state['locations'] = locations state['labels'] = labels state['status'] = 'finished' self.set_python_state(state)
def do_sorting_test(sorting_processor, params, recording_dir, assert_avg_accuracy, container='default'): mt.configDownloadFrom('spikeforest.kbucket') recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) sorting = sorting_processor.execute(recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=True) comparison = sa.GenSortingComparisonTable.execute( firings=sorting.outputs['firings_out'], firings_true=recdir + '/firings_true.mda', units_true=[], json_out={'ext': '.json'}, html_out={'ext': '.html'}, _container=None, _force_run=True) X = mt.loadObject(path=comparison.outputs['json_out']) accuracies = [float(a['accuracy']) for a in X.values()] avg_accuracy = np.mean(accuracies) print('Average accuracy: {}'.format(avg_accuracy)) assert (avg_accuracy >= assert_avg_accuracy)
def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running NWBFile') mt.configDownloadFrom(state.get('download_from', [])) path = state.get('path', None) if path: if path.endswith('.nwb'): self._set_status('running', 'Reading nwb file: {}'.format(path)) obj = nwb_to_dict(path, use_cache=False, exclude_data=True, verbose=False) self._set_status('running', 'Finished nwb file: {}'.format(path)) else: self._set_status('running', 'Realizing object: {}'.format(path)) obj = mt.loadObject(path=path) if not obj: self._set_error('Unable to realize object: {}'.format(path)) return self.set_python_state( dict(status='finished', status_message='finished loading: {}'.format(path), object=obj)) else: self._set_error('Missing path')
def javascript_state_changed(self, prev_state, state): if not self._recording: recordingPath = state.get('recordingPath', None) if not recordingPath: return self.set_python_state(dict(status_message='Loading recording')) mt.configDownloadFrom(state.get('download_from')) X = SFMdaRecordingExtractor(dataset_directory=recordingPath, download=True) self.set_python_state( dict(numChannels=X.get_num_channels(), numTimepoints=X.get_num_frames(), samplerate=X.get_sampling_frequency(), status_message='Loaded recording.')) self._recording = X else: X = self._recording SR = state.get('segmentsRequested', {}) for key in SR.keys(): aa = SR[key] if not self.get_python_state(key, None): self.set_python_state( dict(status_message='Loading segment {}'.format(key))) data0 = self._load_data(aa['ds'], aa['ss']) data0_base64 = _mda32_to_base64(data0) state0 = {} state0[key] = dict(data=data0_base64, ds=aa['ds'], ss=aa['ss']) self.set_python_state(state0) self.set_python_state( dict(status_message='Loaded segment {}'.format(key)))
def main(): parser = argparse.ArgumentParser( description=help_txt, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--output_dir', help='The output directory for saving the files.') parser.add_argument('--download-from', help='The output directory for saving the files.', required=False, default='spikeforest.kbucket') parser.add_argument('--key_path', help='Key path to retrieve data from') args = parser.parse_args() output_dir = args.output_dir if os.path.exists(output_dir): raise Exception( 'Output directory already exists: {}'.format(output_dir)) os.mkdir(output_dir) mt.configDownloadFrom(args.download_from) print('Loading spike-front results object...') obj = mt.loadObject(path=args.key_path) StudySets = obj['StudySets'] SortingResults = obj['SortingResults'] Sorters = obj['Sorters'] Algorithms = obj['Algorithms'] StudyAnalysisResults = obj['StudyAnalysisResults'] General = obj['General'] print('Saving {} study sets to {}/StudySets.json'.format( len(StudySets), output_dir)) mt.saveObject(object=StudySets, dest_path=output_dir + '/StudySets.json') print('Saving {} sorting results to {}/SortingResults.json'.format( len(SortingResults), output_dir)) mt.saveObject(object=SortingResults, dest_path=output_dir + '/SortingResults.json') print('Saving {} sorters to {}/Sorters.json'.format( len(Sorters), output_dir)) mt.saveObject(object=Sorters, dest_path=output_dir + '/Sorters.json') print('Saving {} algorithms to {}/Algorithms.json'.format( len(Algorithms), output_dir)) mt.saveObject(object=Algorithms, dest_path=output_dir + '/Algorithms.json') print('Saving {} study analysis results to {}/StudyAnalysisResults.json'. format(len(StudySets), output_dir)) mt.saveObject(object=StudyAnalysisResults, dest_path=output_dir + '/StudyAnalysisResults.json') print('Saving general info to {}/General.json'.format(output_dir)) mt.saveObject(object=General, dest_path=output_dir + '/General.json')
def main(): parser = argparse.ArgumentParser(description='Show the publicly downloadable studies of SpikeForest') parser.add_argument('--group_names', help='Comma-separated list of recording group names.', required=False, default=None) args = parser.parse_args() mt.configDownloadFrom(['spikeforest.public']) if args.group_names is not None: group_names = args.output_ids.split(',') else: group_names = [ 'paired_boyden32c', 'paired_crcns', 'paired_mea64c', 'paired_kampff', 'synth_bionet', 'synth_magland', 'manual_franklab', 'synth_mearec_neuronexus', 'synth_mearec_tetrode', 'synth_visapy', 'hybrid_janelia' ] print('Using group names: ', group_names) studies = [] study_sets = [] recordings = [] for group_name in group_names: print('RECORDING GROUP: {}'.format(group_name)) output_path = ('key://pairio/spikeforest/spikeforest_recording_group.{}.json').format(group_name) obj = mt.loadObject(path=output_path) if obj: studies = studies + obj['studies'] study_sets = study_sets + obj.get('study_sets', []) recordings = recordings + obj['recordings'] study_sets_by_study = dict() for study in obj['studies']: study_sets_by_study[study['name']] = study['study_set'] for rec in obj['recordings']: if rec.get('public', False): study_set = study_sets_by_study.get(rec.get('study', ''), '') print('{}/{}/{}: {}'.format(study_set, rec.get('study', ''), rec.get('name', ''), rec.get('directory', ''))) else: print('WARNING: unable to load object: ' + output_path) print('') print('ALL GROUPS') study_sets_by_study = dict() for study in studies: study_sets_by_study[study['name']] = study['study_set'] for rec in recordings: if rec.get('public', False): study_set = study_sets_by_study.get(rec.get('study', ''), '') print('- {}/{}/{}: `{}`'.format(study_set, rec.get('study', ''), rec.get('name', ''), rec.get('directory', '')))
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) max_samples = state.get('max_samples') max_dt_msec = state.get('max_dt_msec') bin_size_msec = state.get('bin_size_msec') if not max_dt_msec: return firings_path = state.get('firingsPath', None) if not firings_path: self.set_python_state(dict( status='error', status_message='No firingsPath provided' )) return samplerate = state.get('samplerate', None) if not samplerate: self.set_python_state(dict( status='error', status_message='No samplerate provided' )) return self.set_python_state(dict(status_message='Realizing file: {}'.format(firings_path))) firings_path2 = mt.realizeFile(firings_path) if not firings_path2: self.set_python_state(dict( status='error', status_message='Unable to realize file: {}'.format(firings_path) )) return result = ComputeAutocorrelograms.execute( firings_path=firings_path2, samplerate=samplerate, max_samples=max_samples, bin_size_msec=bin_size_msec, max_dt_msec=max_dt_msec, json_out=dict(ext='.json') ) if result.retcode != 0: self.set_python_state(dict( status='error', status_message='Error computing autocorrelogram.' )) return output = mt.loadObject(path=result.outputs['json_out']) self.set_python_state(dict( status='finished', output=output ))
def main(): path = mt.createSnapshot(path='recordings_out') mt.configDownloadFrom('spikeforest.public') X = mt.readDir(path) for study_set_name, d in X['dirs'].items(): study_sets = [] studies = [] recordings = [] study_sets.append(dict( name=study_set_name + '_b', info=dict(), description='' )) for study_name, d2 in d['dirs'].items(): study_dir = path + '/' + study_set_name + '/' + study_name study0 = dict( name=study_name, study_set=study_set_name + '_b', directory=study_dir, description='' ) studies.append(study0) index_within_study = 0 for recording_name, d3 in d2['dirs'].items(): recdir = study_dir + '/' + recording_name recordings.append(dict( name=recording_name, study=study_name, directory=recdir, firings_true=recdir + '/firings_true.mda', index_within_study=index_within_study, description='One of the recordings in the {} study'.format(study_name) )) index_within_study = index_within_study + 1 print('Saving object...') group_name = study_set_name + '_b' address = mt.saveObject( object=dict( studies=studies, recordings=recordings, study_sets=study_sets ), key=dict(name='spikeforest_recording_group', group_name=group_name), upload_to='spikeforest.public' ) if not address: raise Exception('Problem uploading object to kachery') output_fname = 'key://pairio/spikeforest/spikeforest_recording_group.{}.json'.format(group_name) print('Saving output to {}'.format(output_fname)) mt.createSnapshot(path=address, dest_path=output_fname)
def mem_profile_test(sorting_processor, params, recording_dir, container='default', _keep_temp_files=True): mt.configDownloadFrom('spikeforest.public') params['fMemProfile'] = True recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) sorting = sorting_processor.execute( recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=True, _keep_temp_files=_keep_temp_files )
def main(): path = 'sha1dir://8516cc54587e0c5ddd0709154e7f609b9b7884b4' mt.configDownloadFrom('spikeforest.public') X = mt.readDir(path) for study_set_name, d in X['dirs'].items(): for study_name, d2 in d['dirs'].items(): for recording_name, d3 in d2['dirs'].items(): x = mt.loadObject(path=path + '/' + study_set_name + '/' + study_name + '/' + recording_name + '.runtime_info.json') print('{}/{}/{}\n{} sec\n'.format(study_set_name, study_name, recording_name, x['elapsed_sec']))
def javascript_state_changed(self, prev_state, state): self.set_state(dict(status='running', status_message='Running')) vtp_path = state.get('vtp_path', None) download_from = state.get('download_from', None) scalar_info = state.get('scalar_info', None) vector_field_info = state.get('vector_field_info', None) arrow_subsample_factor = state.get('arrow_subsample_factor', None) if not vtp_path: self.set_state(dict(status='error', status_message='No vtp_path')) return if download_from: mt.configDownloadFrom(download_from) fname = mt.realizeFile(path=vtp_path) reader = vtkXMLPolyDataReader() reader.SetFileName(fname) reader.Update() X = reader.GetOutput() vertices = vtk_to_numpy(X.GetPoints().GetData()).T faces = vtk_to_numpy(X.GetPolys().GetData()) if scalar_info: scalars = vtk_to_numpy(X.GetPointData().GetArray( scalar_info['name'])) scalars = scalars[:, scalar_info['component']] else: scalars = None if vector_field_info: vector_field = vtk_to_numpy(X.GetPointData().GetArray( vector_field_info['name'])) vector_field = vector_field[:, vector_field_info['components']] arrows = [ dict(start=vertices[:, j] - vector_field[j, :].T / 2, end=vertices[:, j] + vector_field[j, :].T / 2) for j in range(0, vector_field.shape[0], arrow_subsample_factor) ] else: arrows = None self.set_state( dict(status='finished', vertices=vertices, faces=faces, scalars=scalars, arrows=arrows))
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) # get javascript state download_from = state.get('download_from', []) path = state.get('path', None) name = state.get('name', None) if path and name: mt.configDownloadFrom(download_from) if path.endswith('.nwb'): self.set_python_state( dict(status_message='Realizing object from nwb file: {}'. format(path))) obj = nwb_to_dict(path, use_cache=True) else: self.set_python_state( dict(status_message='Realizing object: {}'.format(path))) obj = mt.loadObject(path=path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( path))) return datasets = obj['general']['subject']['cortical_surfaces'][name][ '_datasets'] faces0 = np.load(mt.realizeFile(datasets['faces']['_data'])) vertices = np.load(mt.realizeFile(datasets['vertices']['_data'])).T # there's a better way to do the following # (need to get it into a single vector format) faces = [] for j in range(faces0.shape[0]): # 3 = #vertices in polygon (assume a triangulation) faces.extend([3, faces0[j, 0], faces0[j, 1], faces0[j, 2]]) faces = np.array(faces) # return this python state to the javascript self.set_python_state( dict(faces=faces, vertices=vertices, status='finished', status_message='Done.')) else: self.set_python_state( dict(status='error', status_message='Missing path and/or name'))
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) path = state.get('path', None) if path: self.set_python_state( dict(status_message='Realizing object: {}'.format(path))) obj = mt.loadObject(path=path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( path))) return state['object'] = obj state['status'] = 'finished' self.set_python_state(state)
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) nwb_path = state.get('nwb_path', None) downsample_factor = state.get('downsample_factor', 1) if nwb_path: if nwb_path.endswith('.nwb'): self.set_python_state( dict(status_message='Realizing object from nwb file: {}'. format(nwb_path))) obj = h5_to_dict(nwb_path, use_cache=True) else: self.set_python_state( dict(status_message='Realizing object: {}'.format( nwb_path))) obj = mt.loadObject(path=nwb_path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( nwb_path))) return try: positions_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['data']['_data'] except: self.set_python_state( dict(status='error', status_message= 'Problem extracting behavior positions in file: {}'. format(nwb_path))) return positions = np.load(mt.realizeFile(path=positions_path)) positions = positions[::downsample_factor, :] self.set_python_state( dict(status_message='Finished loading positions')) state['positions'] = positions state['status'] = 'finished' self.set_python_state(state) else: self.set_python_state( dict(status='error', status_message='Missing in state: nwb_path'))
def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running surface3D') mt.configDownloadFrom(state.get('download_from', [])) python_state = dict() path0 = state.get('faces_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return faces0 = np.load(x) # there's a better way to do the following # (need to get it into a single vector format) faces = [] for j in range(faces0.shape[0]): # 3 = #vertices in polygon (assume a triangulation) faces.extend([3, faces0[j, 0], faces0[j, 1], faces0[j, 2]]) faces = np.array(faces) python_state['faces'] = faces path0 = state.get('vertices_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return vertices0 = np.load(x) python_state['vertices'] = vertices0.T path0 = state.get('scalars_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return x = np.load(x) python_state['scalars'] = x python_state['status'] = 'finished' python_state['status_message'] = 'finished' self.set_python_state(python_state)
def do_sorting_test( sorting_processor, params, recording_dir, container='default', force_run=True, _keep_temp_files=False ): mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) recdir = recording_dir mt.createSnapshot(path=recdir, download_recursive=True) timer = time.time() sorting = sorting_processor.execute( recording_dir=recdir, firings_out={'ext': '.mda'}, **params, _container=container, _force_run=force_run, _keep_temp_files=_keep_temp_files ) elapsed = time.time() - timer print('################ ELAPSED for sorting (sec): {}'.format(elapsed)) timer = time.time() comparison = sa.GenSortingComparisonTable.execute( firings=sorting.outputs['firings_out'], firings_true=recdir + '/firings_true.mda', units_true=[], json_out={'ext': '.json'}, html_out={'ext': '.html'}, _container='default', _force_run=True ) elapsed = time.time() - timer print('################ ELAPSED for comparison (sec): {}'.format(elapsed)) X = mt.loadObject(path=comparison.outputs['json_out']) accuracies = [float(a['accuracy']) for a in X.values()] avg_accuracy = np.mean(accuracies) print('Average accuracy: {}'.format(avg_accuracy))
def main(): parser = argparse.ArgumentParser( description='Upload public files, e.g., console outputs') parser.add_argument( '--output_ids', help='Comma-separated list of IDs of the analysis outputs to include.', required=False, default=None) args = parser.parse_args() mt.configDownloadFrom(['spikeforest.kbucket']) if args.output_ids is not None: output_ids = args.output_ids.split(',') else: output_ids = [ 'paired_boyden32c', 'paired_crcns', 'paired_mea64c', 'paired_kampff', 'paired_monotrode', 'synth_monotrode', 'synth_bionet', 'synth_magland', 'manual_franklab', 'synth_mearec_neuronexus', 'synth_mearec_tetrode', 'synth_visapy', 'hybrid_janelia' ] print('Using output ids: ', output_ids) for output_id in output_ids: print('Loading output object: {}'.format(output_id)) output_path = ( 'key://pairio/spikeforest/spikeforest_analysis_results.{}.json' ).format(output_id) obj = mt.loadObject(path=output_path) paths = [ sr['console_out'] for sr in obj['sorting_results'] if 'console_out' in sr ] print('{}: {} sorting results - {} with console_out'.format( output_id, len(obj['sorting_results']), len(paths))) mt.createSnapshots(paths=paths, upload_to='spikeforest.public')
def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running') path = state.get('path', None) download_from = state.get('download_from', []) mt.configDownloadFrom(download_from) if not path: self._set_status('finished') return path = mt.realizeFile(path) if not path: self._set_error('Unable to realize file.') return with open(path, 'rb') as f: video_data = f.read() video_data_b64 = base64.b64encode(video_data).decode() self.set_python_state( dict(video_data_b64=video_data_b64, status='finished', status_message=''))
def javascript_state_changed(self, prev_state, state): path = state.get('path', None) download_from = state.get('download_from', []) mt.configDownloadFrom(download_from) if not path: self.set_python_state( dict(status='error', status_message='No path provided.')) return self.set_python_state( dict(status='running', status_message='Loading: {}'.format(path))) obj = mt.loadObject(path=path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( path))) return obj['StudyAnalysisResults'] = None self.set_python_state(dict(status='finished', object=obj))
def main(): parser = argparse.ArgumentParser( description= 'Generate unit detail data (including spikesprays) for website') parser.add_argument( '--output_ids', help= 'Comma-separated list of IDs of the analysis outputs to include in the website.', required=False, default=None) args = parser.parse_args() use_slurm = True mt.configDownloadFrom(['spikeforest.kbucket']) if args.output_ids is not None: output_ids = args.output_ids.split(',') else: output_ids = [ 'paired_boyden32c', 'paired_crcns', 'paired_mea64c', 'paired_kampff', 'paired_monotrode', 'synth_monotrode', # 'synth_bionet', 'synth_magland', 'manual_franklab', 'synth_mearec_neuronexus', 'synth_mearec_tetrode', 'synth_visapy', 'hybrid_janelia' ] print('Using output ids: ', output_ids) print( '******************************** LOADING ANALYSIS OUTPUT OBJECTS...') for output_id in output_ids: slurm_working_dir = 'tmp_slurm_job_handler_' + _random_string(5) job_handler = mlpr.SlurmJobHandler(working_dir=slurm_working_dir) if use_slurm: job_handler.addBatchType(name='default', num_workers_per_batch=20, num_cores_per_job=1, time_limit_per_batch=1800, use_slurm=True, additional_srun_opts=['-p ccm']) else: job_handler.addBatchType( name='default', num_workers_per_batch=multiprocessing.cpu_count(), num_cores_per_job=1, use_slurm=False) with mlpr.JobQueue(job_handler=job_handler) as JQ: print('=============================================', output_id) print('Loading output object: {}'.format(output_id)) output_path = ( 'key://pairio/spikeforest/spikeforest_analysis_results.{}.json' ).format(output_id) obj = mt.loadObject(path=output_path) # studies = obj['studies'] # study_sets = obj.get('study_sets', []) # recordings = obj['recordings'] sorting_results = obj['sorting_results'] print( 'Determining sorting results to process ({} total)...'.format( len(sorting_results))) sorting_results_to_process = [] for sr in sorting_results: key = dict(name='unit-details-v0.1.0', recording_directory=sr['recording']['directory'], firings_true=sr['firings_true'], firings=sr['firings']) val = mt.getValue(key=key, collection='spikeforest') if not val: sr['key'] = key sorting_results_to_process.append(sr) print('Need to process {} of {} sorting results'.format( len(sorting_results_to_process), len(sorting_results))) recording_directories_to_process = sorted( list( set([ sr['recording']['directory'] for sr in sorting_results_to_process ]))) print('{} recording directories to process'.format( len(recording_directories_to_process))) print('Filtering recordings...') filter_jobs = FilterTimeseries.createJobs([ dict(recording_directory=recdir, timeseries_out={'ext': '.mda'}, _timeout=600) for recdir in recording_directories_to_process ]) filter_results = [job.execute() for job in filter_jobs] JQ.wait() filtered_timeseries_by_recdir = dict() for i, recdir in enumerate(recording_directories_to_process): result0 = filter_results[i] if result0.retcode != 0: raise Exception( 'Problem computing filtered timeseries for recording: {}' .format(recdir)) filtered_timeseries_by_recdir[recdir] = result0.outputs[ 'timeseries_out'] print('Creating spike sprays...') for sr in sorting_results_to_process: rec = sr['recording'] study_name = rec['study'] rec_name = rec['name'] sorter_name = sr['sorter']['name'] print('====== COMPUTING {}/{}/{}'.format( study_name, rec_name, sorter_name)) if sr.get('comparison_with_truth', None) is not None: cwt = mt.loadObject( path=sr['comparison_with_truth']['json']) filtered_timeseries = filtered_timeseries_by_recdir[ rec['directory']] spike_spray_job_objects = [] list0 = list(cwt.values()) for _, unit in enumerate(list0): # print('') # print('=========================== {}/{}/{} unit {} of {}'.format(study_name, rec_name, sorter_name, ii + 1, len(list0))) # ssobj = create_spikesprays(rx=rx, sx_true=sx_true, sx_sorted=sx, neighborhood_size=neighborhood_size, num_spikes=num_spikes, unit_id_true=unit['unit_id'], unit_id_sorted=unit['best_unit']) spike_spray_job_objects.append( dict(args=dict( recording_directory=rec['directory'], filtered_timeseries=filtered_timeseries, firings_true=os.path.join( rec['directory'], 'firings_true.mda'), firings_sorted=sr['firings'], unit_id_true=unit['unit_id'], unit_id_sorted=unit['best_unit'], json_out={'ext': '.json'}, _container='default', _timeout=180), study_name=study_name, rec_name=rec_name, sorter_name=sorter_name, unit=unit)) spike_spray_jobs = CreateSpikeSprays.createJobs( [obj['args'] for obj in spike_spray_job_objects]) spike_spray_results = [ job.execute() for job in spike_spray_jobs ] sr['spike_spray_job_objects'] = spike_spray_job_objects sr['spike_spray_results'] = spike_spray_results JQ.wait() for sr in sorting_results_to_process: rec = sr['recording'] study_name = rec['study'] rec_name = rec['name'] sorter_name = sr['sorter']['name'] print('====== SAVING {}/{}/{}'.format(study_name, rec_name, sorter_name)) if sr.get('comparison_with_truth', None) is not None: spike_spray_job_objects = sr['spike_spray_job_objects'] spike_spray_results = sr['spike_spray_results'] unit_details = [] ok = True for i, result in enumerate(spike_spray_results): obj0 = spike_spray_job_objects[i] if result.retcode != 0: print('WARNING: Error creating spike sprays for job:') print(spike_spray_job_objects[i]) ok = False break ssobj = mt.loadObject(path=result.outputs['json_out']) if ssobj is None: raise Exception( 'Problem loading spikespray object output.') address = mt.saveObject(object=ssobj, upload_to='spikeforest.kbucket') unit = obj0['unit'] unit_details.append( dict(studyName=obj0['study_name'], recordingName=obj0['rec_name'], sorterName=obj0['sorter_name'], trueUnitId=unit['unit_id'], sortedUnitId=unit['best_unit'], spikeSprayUrl=mt.findFile( path=address, remote_only=True, download_from='spikeforest.kbucket'), _container='default')) if ok: mt.saveObject(collection='spikeforest', key=sr['key'], object=unit_details, upload_to='spikeforest.public')
def main(): mt.configDownloadFrom('spikeforest.public') templates_path = 'sha1dir://95dba567b5168bacb480411ca334ffceb96b8c19.2019-06-11.templates' recordings_path = 'recordings_out' tempgen_tetrode = templates_path + '/templates_tetrode.h5' tempgen_neuronexus = templates_path + '/templates_neuronexus.h5' tempgen_neuropixels = templates_path + '/templates_neuropixels.h5' tempgen_neuronexus_drift = templates_path + '/templates_neuronexus_drift.h5' noise_level = [10, 20] duration = 600 bursting = [False, True] nrec = 2 # change this to 10 ei_ratio = 0.8 rec_dict = { 'tetrode': { 'ncells': [10, 20], 'tempgen': tempgen_tetrode, 'drifting': False }, 'neuronexus': { 'ncells': [10, 20, 40], 'tempgen': tempgen_neuronexus, 'drifting': False }, 'neuropixels': { 'ncells': [20, 40, 60], 'tempgen': tempgen_neuropixels, 'drifting': False }, 'neuronexus_drift': { 'ncells': [10, 20, 40], 'tempgen': tempgen_neuronexus_drift, 'drifting': True } } # optional: if drifting change drift velocity # recording_params['recordings']['drift_velocity] = ... # Generate and save recordings if os.path.exists(recordings_path): shutil.rmtree(recordings_path) os.mkdir(recordings_path) # Set up slurm configuration slurm_working_dir = 'tmp_slurm_job_handler_' + _random_string(5) job_handler = mlpr.SlurmJobHandler(working_dir=slurm_working_dir) use_slurm = True job_timeout = 3600 * 4 if use_slurm: job_handler.addBatchType(name='default', num_workers_per_batch=4, num_cores_per_job=6, time_limit_per_batch=job_timeout * 3, use_slurm=True, max_simultaneous_batches=20, additional_srun_opts=['-p ccm']) else: job_handler.addBatchType( name='default', num_workers_per_batch=multiprocessing.cpu_count(), num_cores_per_job=2, max_simultaneous_batches=1, use_slurm=False) with mlpr.JobQueue(job_handler=job_handler) as JQ: results_to_write = [] for rec_type in rec_dict.keys(): study_set_name = 'SYNTH_MEAREC_{}'.format(rec_type.upper()) os.mkdir(recordings_path + '/' + study_set_name) params = dict() params['duration'] = duration params['drifting'] = rec_dict[rec_type]['drifting'] # reduce minimum distance for dense recordings params['min_dist'] = 15 for ncells in rec_dict[rec_type]['ncells']: # changing number of cells n_exc = int(ei_ratio * 10) # intentionally replaced nrec by 10 here params['n_exc'] = n_exc params['n_inh'] = ncells - n_exc for n in noise_level: # changing noise level params['noise_level'] = n for b in bursting: bursting_str = '' if b: bursting_str = '_bursting' study_name = 'synth_mearec_{}_noise{}_K{}{}'.format( rec_type, n, ncells, bursting_str) os.mkdir(recordings_path + '/' + study_set_name + '/' + study_name) for i in range(nrec): # set random seeds params[ 'seed'] = i # intentionally doing it this way # changing bursting and shape modulation print('Generating', rec_type, 'recording with', ncells, 'noise level', n, 'bursting', b) params['bursting'] = b params['shape_mod'] = b templates0 = mt.realizeFile( path=rec_dict[rec_type]['tempgen']) result0 = GenerateMearecRecording.execute( **params, templates_in=templates0, recording_out=dict(ext='.h5')) mda_output_folder = recordings_path + '/' + study_set_name + '/' + study_name + '/' + '{}'.format( i) results_to_write.append( dict(result=result0, mda_output_folder=mda_output_folder)) JQ.wait() for x in results_to_write: result0: mlpr.MountainJobResult = x['result'] mda_output_folder = x['mda_output_folder'] path = mt.realizeFile(path=result0.outputs['recording_out']) recording = se.MEArecRecordingExtractor(recording_path=path) sorting_true = se.MEArecSortingExtractor(recording_path=path) se.MdaRecordingExtractor.write_recording( recording=recording, save_path=mda_output_folder) se.MdaSortingExtractor.write_sorting(sorting=sorting_true, save_path=mda_output_folder + '/firings_true.mda') if result0.console_out: mt.realizeFile(path=result0.console_out, dest_path=mda_output_folder + '.console_out.txt') if result0.runtime_info: mt.saveObject(object=result0.runtime_info, dest_path=mda_output_folder + '.runtime_info.json') print('Creating and uploading snapshot...') sha1dir_path = mt.createSnapshot(path=recordings_path, upload_to='spikeforest.public', upload_recursive=False) # sha1dir_path = mt.createSnapshot(path=recordings_path, upload_to='spikeforest.kbucket', upload_recursive=True) print(sha1dir_path)
#!/usr/bin/env python from mountaintools import client as mt mt.configDownloadFrom('spikeforest.kbucket') def do_prepare(recording_group, study_name): print(recording_group, study_name) X = mt.loadObject( path="key://pairio/spikeforest/spikeforest_recording_group.{}.json". format(recording_group)) studies = [y for y in X['studies'] if (y['name'] == study_name)] recordings = [y for y in X['recordings'] if y['study'] == study_name] recordings = recordings[0:1] study_sets = X['study_sets'] Y = dict(studies=studies, recordings=recordings, study_sets=study_sets) address = mt.saveObject(object=Y) assert address is not None dest_path = 'key://pairio/spikeforest/spikeforest_recording_group.test_{}.json'.format( recording_group) print(dest_path) mt.createSnapshot(path=address, upload_to='spikeforest.kbucket', dest_path=dest_path) do_prepare(recording_group='synth_magland', study_name='synth_magland_noise10_K10_C4') do_prepare(recording_group='paired_mea64c', study_name='paired_mea64c')
#!/usr/bin/env python from spikeforest import SFMdaRecordingExtractor, SFMdaSortingExtractor from mountaintools import client as mt # Configure to download from the public spikeforest kachery node mt.configDownloadFrom('spikeforest.public') # Load an example tetrode recording with its ground truth # You can also substitute any of the other available recordings recdir = 'sha1dir://fb52d510d2543634e247e0d2d1d4390be9ed9e20.synth_magland/datasets_noise10_K10_C4/001_synth' print('loading recording...') recording = SFMdaRecordingExtractor(dataset_directory=recdir, download=True) sorting_true = SFMdaSortingExtractor(firings_file=recdir + '/firings_true.mda') # import a spike sorter from the spikesorters module of spikeforest from spikeforestsorters import MountainSort4 import os import shutil # In place of MountainSort4 you could use any of the following: # # MountainSort4, SpykingCircus, KiloSort, KiloSort2, YASS # IronClust, HerdingSpikes2, JRClust, Tridesclous, Klusta # although the Matlab sorters require further setup. # clear and create an empty output directory (keep things tidy) if os.path.exists('test_outputs'): shutil.rmtree('test_outputs') os.makedirs('test_outputs', exist_ok=True)
def javascript_state_changed(self, prev_state, state): self.set_status('running') self.set_status_message('Running') mt.configDownloadFrom(state.get('download_from', [])) nwb_query = state.get('nwb_query', None) downsample_factor = state.get('downsample_factor', 1) if nwb_query: self.set_status_message('Loading nwb object') obj = _load_nwb_object(nwb_query) if not obj: self.set_error('Unable to load nwb object') return self.set_status_message('Loading positions and timestamps from') try: positions_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['data']['_data'] timestamps_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['timestamps']['_data'] except: self.set_error( 'Problem extracting behavior positions or timestamps') return positions = np.load(mt.realizeFile(path=positions_path)) positions = positions[::downsample_factor, :] timestamps = np.load(mt.realizeFile(path=timestamps_path)) timestamps = timestamps[::downsample_factor] self.set_status_message('Loading spike times') try: spike_times_path = obj['units']['_datasets']['spike_times'][ '_data'] spike_times_index = obj['units']['_datasets'][ 'spike_times_index']['_data'] spike_times_index_id = obj['units']['_datasets']['id']['_data'] if 'cluster_name' in obj['units']['_datasets']: cluster_names = obj['units']['_datasets']['cluster_name'][ '_data'] else: cluster_names = [] except: self.set_error('Problem extracting spike times') return spike_times = np.load(mt.realizeFile(path=spike_times_path)) spike_time_indices = _find_closest(timestamps, spike_times) spike_labels = np.zeros(spike_time_indices.shape) aa = 0 for i, val in enumerate(spike_times_index): spike_labels[aa:val] = spike_times_index_id[i] aa = val all_unit_ids = sorted(list(set(spike_labels))) state['positions'] = positions state['status'] = 'finished' state['spike_time_indices'] = spike_time_indices state['spike_labels'] = spike_labels state['all_unit_ids'] = all_unit_ids state['cluster_names'] = cluster_names self.set_python_state(state) self.set_status('finished') else: self.set_error('Missing in state: nwb_query')
def main(): from mountaintools import client as mt parser = argparse.ArgumentParser( description= 'Generate unit detail data (including spikesprays) for website') parser.add_argument('analysis_path', help='assembled analysis file (output.json)') parser.add_argument( '--studysets', help='Comma-separated list of study set names to include', required=False, default=None) parser.add_argument('--force-run', help='Force rerunning of processing', action='store_true') parser.add_argument( '--force-run-all', help='Force rerunning of processing including filtering', action='store_true') parser.add_argument('--parallel', help='Optional number of parallel jobs', required=False, default='0') parser.add_argument('--slurm', help='Path to slurm config file', required=False, default=None) parser.add_argument('--cache', help='The cache database to use', required=False, default=None) parser.add_argument('--job-timeout', help='Timeout for processing jobs', required=False, default=600) parser.add_argument('--log-file', help='Log file for analysis progress', required=False, default=None) parser.add_argument( '--force-regenerate', help= 'Whether to force regenerating spike sprays (for when code has changed)', action='store_true') parser.add_argument('--test', help='Whether to just test by running only 1', action='store_true') args = parser.parse_args() mt.configDownloadFrom(['spikeforest.kbucket']) with open(args.analysis_path, 'r') as f: analysis = json.load(f) if args.studysets is not None: studyset_names = args.studysets.split(',') print('Using study sets: ', studyset_names) else: studyset_names = None study_sets = analysis['StudySets'] sorting_results = analysis['SortingResults'] studies_to_include = [] for ss in study_sets: if (studyset_names is None) or (ss['name'] in studyset_names): for study in ss['studies']: studies_to_include.append(study['name']) print('Including studies:', studies_to_include) print('Determining sorting results to process ({} total)...'.format( len(sorting_results))) sorting_results_to_process = [] sorting_results_to_consider = [] for sr in sorting_results: study_name = sr['studyName'] if study_name in studies_to_include: if 'firings' in sr: if sr.get('comparisonWithTruth', None) is not None: sorting_results_to_consider.append(sr) key = dict(name='unit-details-v0.1.0', recording_directory=sr['recordingDirectory'], firings_true=sr['firingsTrue'], firings=sr['firings']) val = mt.getValue(key=key, collection='spikeforest') if (not val) or (args.force_regenerate): sr['key'] = key sorting_results_to_process.append(sr) if args.test and len(sorting_results_to_process) > 0: sorting_results_to_process = [sorting_results_to_process[0]] print('Need to process {} of {} sorting results'.format( len(sorting_results_to_process), len(sorting_results_to_consider))) recording_directories_to_process = sorted( list( set([ sr['recordingDirectory'] for sr in sorting_results_to_process ]))) print('{} recording directories to process'.format( len(recording_directories_to_process))) if int(args.parallel) > 0: job_handler = hither.ParallelJobHandler(int(args.parallel)) elif args.slurm: with open(args.slurm, 'r') as f: slurm_config = json.load(f) job_handler = hither.SlurmJobHandler(working_dir='tmp_slurm', **slurm_config['cpu']) else: job_handler = None print('Filtering recordings...') filter_results = [] with hither.config(container='default', cache=args.cache, force_run=args.force_run_all, job_handler=job_handler, log_path=args.log_file, exception_on_fail=True, cache_failing=False, rerun_failing=True, job_timeout=args.job_timeout), hither.job_queue(): for recdir in recording_directories_to_process: result = filter_recording.run(recording_directory=recdir, timeseries_out=hither.File()) filter_results.append(result) filtered_timeseries_by_recdir = dict() for i, recdir in enumerate(recording_directories_to_process): result0 = filter_results[i] if not result0.success: raise Exception( 'Problem computing filtered timeseries for recording: {}'. format(recdir)) filtered_timeseries_by_recdir[ recdir] = result0.outputs.timeseries_out._path print('Creating spike sprays...') with hither.config(container='default', cache=args.cache, force_run=args.force_run or args.force_run_all, job_handler=job_handler, log_path=args.log_file, exception_on_fail=True, cache_failing=False, rerun_failing=True, job_timeout=args.job_timeout), hither.job_queue(): for sr in sorting_results_to_process: recdir = sr['recordingDirectory'] study_name = sr['studyName'] rec_name = sr['recordingName'] sorter_name = sr['sorterName'] print('====== COMPUTING {}/{}/{}'.format(study_name, rec_name, sorter_name)) cwt = ka.load_object(path=sr['comparisonWithTruth']['json']) filtered_timeseries = filtered_timeseries_by_recdir[recdir] spike_spray_results = [] list0 = list(cwt.values()) for _, unit in enumerate(list0): result = create_spike_sprays.run( recording_directory=recdir, filtered_timeseries=filtered_timeseries, firings_true=os.path.join(recdir, 'firings_true.mda'), firings_sorted=sr['firings'], unit_id_true=unit['unit_id'], unit_id_sorted=unit['best_unit'], json_out=hither.File()) setattr(result, 'unit', unit) spike_spray_results.append(result) sr['spike_spray_results'] = spike_spray_results for sr in sorting_results_to_process: recdir = sr['recordingDirectory'] study_name = sr['studyName'] rec_name = sr['recordingName'] sorter_name = sr['sorterName'] print('====== SAVING {}/{}/{}'.format(study_name, rec_name, sorter_name)) spike_spray_results = sr['spike_spray_results'] key = sr['key'] unit_details = [] ok = True for i, result in enumerate(spike_spray_results): if not result.success: print( 'WARNING: Error creating spike sprays for {}/{}/{}'.format( study_name, rec_name, sorter_name)) ok = False break ssobj = ka.load_object(result.outputs.json_out._path) if ssobj is None: raise Exception('Problem loading spikespray object output.') address = mt.saveObject(object=ssobj, upload_to='spikeforest.kbucket') unit = getattr(result, 'unit') unit_details.append( dict(studyName=study_name, recordingName=rec_name, sorterName=sorter_name, trueUnitId=unit['unit_id'], sortedUnitId=unit['best_unit'], spikeSprayUrl=mt.findFile( path=address, remote_only=True, download_from='spikeforest.kbucket'))) if ok: mt.saveObject(collection='spikeforest', key=key, object=unit_details, upload_to='spikeforest.public')
def main(): parser = argparse.ArgumentParser(description=help_txt, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--output_ids', help='Comma-separated list of IDs of the analysis outputs to include in the website.', required=False, default=None) parser.add_argument('--upload_to', help='Optional kachery to upload to', required=False, default=None) parser.add_argument('--dest_key_path', help='Optional destination key path', required=False, default=None) args = parser.parse_args() if args.upload_to: upload_to = args.upload_to.split(',') else: upload_to = None mt.configDownloadFrom(['spikeforest.kbucket', 'spikeforest.public']) if args.output_ids is not None: output_ids = args.output_ids.split(',') else: output_ids = [ 'paired_boyden32c', 'paired_crcns', 'paired_mea64c', 'paired_kampff', 'paired_monotrode', 'synth_monotrode', 'synth_bionet', 'synth_magland', 'manual_franklab', 'synth_mearec_neuronexus', 'synth_mearec_tetrode', # 'SYNTH_MEAREC_TETRODE_b', # 'SYNTH_MEAREC_NEURONEXUS_b', # 'SYNTH_MEAREC_NEUROPIXELS_b', 'synth_visapy', 'hybrid_janelia' ] print('Using output ids: ', output_ids) sorters_to_include = set([ 'HerdingSpikes2', 'IronClust', 'IronClust1', 'IronClust2', 'IronClust3', 'IronClust4', 'IronClust5', 'IronClust6', 'IronClust7', 'IronClust8', 'JRClust', 'KiloSort', 'KiloSort2', 'Klusta', 'MountainSort4', 'SpykingCircus', 'Tridesclous', 'Waveclus', # 'Yass' ]) print('******************************** LOADING ANALYSIS OUTPUT OBJECTS...') studies = [] study_sets = [] recordings = [] sorting_results = [] for output_id in output_ids: print('Loading output object: {}'.format(output_id)) output_path = ('key://pairio/spikeforest/spikeforest_analysis_results.{}.json').format(output_id) obj = mt.loadObject(path=output_path) if obj is not None: studies = studies + obj['studies'] print(obj.keys()) study_sets = study_sets + obj.get('study_sets', []) recordings = recordings + obj['recordings'] sorting_results = sorting_results + obj['sorting_results'] else: raise Exception('Unable to load: {}'.format(output_path)) # ALGORITHMS print('******************************** ASSEMBLING ALGORITHMS...') algorithms_by_processor_name = dict() Algorithms = [] basepath = '../../spikeforest/spikeforestsorters/descriptions' repo_base_url = 'https://github.com/flatironinstitute/spikeforest/blob/master' for item in os.listdir(basepath): if item.endswith('.md'): alg = frontmatter.load(basepath + '/' + item).to_dict() alg['markdown_link'] = repo_base_url + '/spikeforest/spikeforestsorters/descriptions/' + item alg['markdown'] = alg['content'] del alg['content'] if 'processor_name' in alg: algorithms_by_processor_name[alg['processor_name']] = alg Algorithms.append(alg) print([alg['label'] for alg in Algorithms]) # # STUDIES # print('******************************** ASSEMBLING STUDIES...') # studies_by_name = dict() # for study in studies: # studies_by_name[study['name']] = study # study['recordings'] = [] # for recording in recordings: # studies_by_name[recording['studyName']]['recordings'].append(dict( # name=recording['name'], # studyName=recording['study_name'], # directory=recording['directory'], # )) Studies = [] for study in studies: Studies.append(dict( name=study['name'], studySet=study['study_set'], description=study['description'], recordings=[] # the following can be obtained from the other collections # numRecordings, sorters, etc... )) print([S['name'] for S in Studies]) print('******************************** ASSEMBLING STUDY SETS...') study_sets_by_name = dict() for study_set in study_sets: study_sets_by_name[study_set['name']] = study_set study_set['studies'] = [] studies_by_name = dict() for study in studies: study0 = dict( name=study['name'], studySetName=study['study_set'], recordings=[] ) study_sets_by_name[study['study_set']]['studies'].append(study0) studies_by_name[study0['name']] = study0 for recording in recordings: true_units_info = mt.loadObject(path=recording['summary']['true_units_info']) if not true_units_info: print(recording['summary']['true_units_info']) raise Exception('Unable to load true_units_info for recording {}'.format(recording['name'])) recording0 = dict( name=recording['name'], studyName=recording['study'], studySetName=studies_by_name[recording['study']]['studySetName'], directory=recording['directory'], firingsTrue=recording['firings_true'], sampleRateHz=recording['summary']['computed_info']['samplerate'], numChannels=recording['summary']['computed_info']['num_channels'], durationSec=recording['summary']['computed_info']['duration_sec'], numTrueUnits=len(true_units_info), spikeSign=-1 # TODO: set this properly ) studies_by_name[recording0['studyName']]['recordings'].append(recording0) StudySets = [] for study_set in study_sets: StudySets.append(study_set) print(StudySets) # SORTING RESULTS print('******************************** SORTING RESULTS...') SortingResults = [] for sr in sorting_results: if sr['sorter']['name'] in sorters_to_include: SR = dict( recordingName=sr['recording']['name'], studyName=sr['recording']['study'], sorterName=sr['sorter']['name'], recordingDirectory=sr['recording']['directory'], firingsTrue=sr['recording']['firings_true'], consoleOut=sr['console_out'], container=sr['container'], cpuTimeSec=sr['execution_stats'].get('elapsed_sec', None), returnCode=sr['execution_stats'].get('retcode', 0), # TODO: in future, the default should not be 0 -- rather it should be a required field of execution_stats timedOut=sr['execution_stats'].get('timed_out', False), startTime=datetime.fromtimestamp(sr['execution_stats'].get('start_time')).isoformat(), endTime=datetime.fromtimestamp(sr['execution_stats'].get('end_time')).isoformat() ) if sr.get('firings', None): SR['firings'] = sr['firings'] if not sr.get('comparison_with_truth', None): print('Warning: comparison with truth not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name'])) print('Console output is here: ' + sr['console_out']) else: print('Warning: firings not found for sorting result: {} {}/{}'.format(sr['sorter']['name'], sr['recording']['study'], sr['recording']['name'])) print('Console output is here: ' + sr['console_out']) SortingResults.append(SR) # print('Num unit results:', len(UnitResults)) # SORTERS print('******************************** ASSEMBLING SORTERS...') sorters_by_name = dict() for sr in sorting_results: sorters_by_name[sr['sorter']['name']] = sr['sorter'] Sorters = [] sorter_names = sorted(list(sorters_by_name.keys())) sorter_names = [sorter_name for sorter_name in sorter_names if sorter_name in sorters_to_include] for sorter_name in sorter_names: sorter = sorters_by_name[sorter_name] alg = algorithms_by_processor_name.get(sorter['processor_name'], dict()) alg_label = alg.get('label', sorter['processor_name']) if sorter['name'] in sorters_to_include: Sorters.append(dict( name=sorter['name'], algorithmName=alg_label, processorName=sorter['processor_name'], processorVersion='0', # jfm to provide this sortingParameters=sorter['params'] )) print([S['name'] + ':' + S['algorithmName'] for S in Sorters]) # STUDY ANALYSIS RESULTS print('******************************** ASSEMBLING STUDY ANALYSIS RESULTS...') StudyAnalysisResults = [ _assemble_study_analysis_result( study_name=study['name'], study_set_name=study['study_set'], recordings=recordings, sorting_results=sorting_results, sorter_names=sorter_names ) for study in studies ] # GENERAL print('******************************** ASSEMBLING GENERAL INFO...') General = [dict( dateUpdated=datetime.now().isoformat(), packageVersions=dict( mountaintools=pkg_resources.get_distribution("mountaintools").version, spikeforest=pkg_resources.get_distribution("spikeforest").version ) )] obj = dict( mode='spike-front', StudySets=StudySets, # TrueUnits=TrueUnits, # UnitResults=UnitResults, SortingResults=SortingResults, Sorters=Sorters, Algorithms=Algorithms, StudyAnalysisResults=StudyAnalysisResults, General=General ) address = mt.saveObject(object=obj) mt.createSnapshot(path=address, upload_to=upload_to, dest_path=args.dest_key_path)