def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running TwoPhotonSeries') nwb_path = state.get('nwb_path', None) download_from = state.get('download_from', []) if not nwb_path: self._set_error('Missing nwb_path') return mt.configDownloadFrom(download_from) nwb_path2 = mt.realizeFile(nwb_path) if not nwb_path2: self._set_error('Unable to realize nwb file: {}'.format(nwb_path)) return self._set_status('running', 'Extracting .mp4 data') outputs = ExtractTwoPhotonSeriesMp4.execute(nwb_in=nwb_path2, mp4_out={'ext': '.mp4'}).outputs self._set_status('running', 'Reading .mp4 data') mp4_fname = mt.realizeFile(outputs['mp4_out']) with open(mp4_fname, 'rb') as f: video_data = f.read() self._set_status('running', 'Encoding .mp4 data') video_data_b64 = base64.b64encode(video_data).decode() video_url = 'data:video/mp4;base64,{}'.format(video_data_b64) self._set_status('running', 'Setting .mp4 data to python state') self.set_python_state(dict( video_url=video_url, status='finished', status_message='' ))
def _download_recordings(*, jobs): for _, job in enumerate(jobs): val = mt.getValue(key=job) if not val: if 'recording' in job: if 'directory' in job['recording']: dsdir = job['recording']['directory'] fname = dsdir + '/raw.mda' print('REALIZING FILE: ' + fname) mt.realizeFile(path=fname)
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) # get javascript state download_from = state.get('download_from', []) path = state.get('path', None) name = state.get('name', None) if path and name: mt.configDownloadFrom(download_from) if path.endswith('.nwb'): self.set_python_state( dict(status_message='Realizing object from nwb file: {}'. format(path))) obj = nwb_to_dict(path, use_cache=True) else: self.set_python_state( dict(status_message='Realizing object: {}'.format(path))) obj = mt.loadObject(path=path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( path))) return datasets = obj['general']['subject']['cortical_surfaces'][name][ '_datasets'] faces0 = np.load(mt.realizeFile(datasets['faces']['_data'])) vertices = np.load(mt.realizeFile(datasets['vertices']['_data'])).T # there's a better way to do the following # (need to get it into a single vector format) faces = [] for j in range(faces0.shape[0]): # 3 = #vertices in polygon (assume a triangulation) faces.extend([3, faces0[j, 0], faces0[j, 1], faces0[j, 2]]) faces = np.array(faces) # return this python state to the javascript self.set_python_state( dict(faces=faces, vertices=vertices, status='finished', status_message='Done.')) else: self.set_python_state( dict(status='error', status_message='Missing path and/or name'))
def load_ntrode(path, *, name, epoch_name, processed_path=None): # use the .geom.csv if it exists (we assume path ends with .mda) geom_file = path[0:-4] + '.geom.csv' if mt.findFile(geom_file): print('Using geometry file: {}'.format(geom_file)) else: # if doesn't exist, we will create a trivial geom later geom_file = None path2 = mt.realizeFile(path) if not path2: raise Exception('Unable to realize file: ' + path) X = mdaio.DiskReadMda(path2) num_channels = X.N1() num_timepoints = X.N2() processed_info = load_ntrode_processed_info(processed_path, recording_path=path, epoch_name=epoch_name, ntrode_name=name) # here's the structure for representing ntrode information return dict( type='ntrode', name=name, epoch_name=epoch_name, path=path, processed_path=processed_path, recording_file=path, geom_file=geom_file, num_channels=num_channels, num_timepoints=num_timepoints, samplerate=30000, # fix this processed_info=processed_info )
def _load_nwb_json(path): path2 = mt.realizeFile(path=path) if not path2: raise Exception('Unable to realize file: {}'.format(path)) with open(path2, 'r') as f: obj = json.load(f) return obj
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) path = state.get('path', None) if path: self.set_python_state( dict(status_message='Realizing file: {}'.format(path))) if path.endswith('.csv'): path2 = mt.realizeFile(path) if not path2: self.set_python_state( dict( status='error', status_message='Unable to realize file: {}'.format( path))) return self.set_python_state(dict(status_message='Loading locatoins')) x = np.genfromtxt(path2, delimiter=',') locations = x.T num_elec = x.shape[0] labels = ['{}'.format(a) for a in range(1, num_elec + 1)] else: raise Exception('Unexpected file type for {}'.format(path)) else: locations = [[0, 0], [1, 0], [1, 1], [2, 1]] labels = ['1', '2', '3', '4'] state = dict() state['locations'] = locations state['labels'] = labels state['status'] = 'finished' self.set_python_state(state)
def test_mandelbrot(): from mountaintools import client as mt import mlprocessors as mlpr from .mandelbrot import compute_mandelbrot, show_mandelbrot, combine_subsampled_mandelbrot, ComputeMandelbrot, compute_mandelbrot_parallel import numpy as np num_x = 4000 result = ComputeMandelbrot.execute( num_iter=10, num_x=num_x, output_npy=dict(ext='.npy', upload=True) ) X = np.load(mt.realizeFile(result.outputs['output_npy'])) Y = compute_mandelbrot_parallel( num_iter=10, num_x=num_x, num_parallel=3, use_slurm=False, _force_run=True ) Z = compute_mandelbrot_parallel( num_iter=10, num_x=num_x, num_parallel=3, use_slurm=False, _force_run=False ) print(X.shape, Y.shape, Z.shape) assert np.all(np.isclose(X, Y)) assert np.all(np.isclose(X, Z))
def __init__(self, dataset_directory, *, download=True, raw_fname='raw.mda', params_fname='params.json'): RecordingExtractor.__init__(self) self._dataset_directory = dataset_directory self._timeseries_path = dataset_directory + '/' + raw_fname self._dataset_params = read_dataset_params(dataset_directory, params_fname) self._samplerate = self._dataset_params['samplerate'] * 1.0 if download: path0 = mt.realizeFile(path=self._timeseries_path) if not path0: raise Exception('Unable to realize file: ' + self._timeseries_path) self._timeseries_path = path0 geom0 = dataset_directory + '/geom.csv' self._geom_fname = mt.realizeFile(path=geom0) self._geom = np.genfromtxt(self._geom_fname, delimiter=',') timeseries_path_or_url = self._timeseries_path if not mt.isLocalPath(timeseries_path_or_url): a = mt.findFile(timeseries_path_or_url) if not a: raise Exception('Cannot find timeseries file: ' + timeseries_path_or_url) timeseries_path_or_url = a # if is_kbucket_url(timeseries0): # download_needed = is_url(ca.findFile(path=timeseries0)) # else: # download_needed = is_url(timeseries0) # if download and download_needed: # print('Downloading file: ' + timeseries0) # self._timeseries_path = ca.realizeFile(path=timeseries0) # print('Done.') # else: # self._timeseries_path = ca.findFile(path=timeseries0) X = DiskReadMda(timeseries_path_or_url) if self._geom.shape[0] != X.N1(): # raise Exception( # 'Incompatible dimensions between geom.csv and timeseries file {} <> {}'.format(self._geom.shape[0], X.N1())) print('WARNING: Incompatible dimensions between geom.csv and timeseries file {} <> {}'.format(self._geom.shape[0], X.N1())) self._geom = np.zeros((X.N1(), 2)) self._num_channels = X.N1() self._num_timepoints = X.N2() for m in range(self._num_channels): self.set_channel_property(m, 'location', self._geom[m, :])
def read_dataset_params(dsdir, params_fname): fname1 = dsdir + '/' + params_fname fname2 = mt.realizeFile(path=fname1) if not fname2: raise Exception('Unable to find file: ' + fname1) if not os.path.exists(fname2): raise Exception('Dataset parameter file does not exist: ' + fname2) with open(fname2) as f: return json.load(f)
def run(self): nwb_obj = nwb_to_dict(self.nwb_in, use_cache=True) npy_path = nwb_obj['acquisition']['TwoPhotonSeries']['_datasets']['data']['_data'] npy_path2 = mt.realizeFile(npy_path) if not npy_path2: nwb_obj = nwb_to_dict(self.nwb_in, use_cache=False) npy_path = nwb_obj['acquisition']['TwoPhotonSeries']['_datasets']['data']['_data'] npy_path2 = mt.realizeFile(npy_path) if not npy_path2: self._set_error('Unable to realize npy file: {}'.format(npy_path)) return X = np.load(npy_path2) # Note that there is a bug in imageio.mimwrite that prevents us to # write to a memory buffer. # See: https://github.com/imageio/imageio/issues/157 imageio.mimwrite(self.mp4_out, X, format='mp4', fps=10)
def test_mandelbrot_errors(): from mountaintools import client as mt import mlprocessors as mlpr from .mandelbrot import ComputeMandelbrotWithError, combine_subsampled_mandelbrot import numpy as np num_iter = 10 num_x = 50 num_parallel = 1 subsampling_factor = num_parallel job_args = [ dict( num_x=num_x, num_iter=num_iter, subsampling_factor=subsampling_factor, subsampling_offset=offset, output_npy=dict(ext='.npy', upload=True), throw_error=(offset == 0), _force_run=False # _container='sha1://87319c2856f312ccc3187927ae899d1d67b066f9/03-20-2019/mountaintools_basic.simg' ) for offset in range(subsampling_factor) ] jobs = ComputeMandelbrotWithError.createJobs(job_args) results = [] for job in jobs: results.append(job.execute()) X_list = [] for result0 in results: if result0.retcode == 0: X0 = np.load(mt.realizeFile(result0.outputs['output_npy'])) X_list.append(X0) else: print('Warning: retcode is non-zero for job.') print('============================================= BEGIN CONSOLE OUT ==========================================') print(mt.realizeFile(result0.console_out)) print('============================================= END CONSOLE OUT ==========================================') if len(X_list) > 0: _ = combine_subsampled_mandelbrot(X_list)
def read_dataset_params(dsdir): # ca = _load_required_modules() fname1 = dsdir + '/params.json' fname2 = mt.realizeFile(path=fname1) if not fname2: raise Exception('Unable to find file: ' + fname1) if not os.path.exists(fname2): raise Exception('Dataset parameter file does not exist: ' + fname2) with open(fname2) as f: return json.load(f)
def __init__(self, firings_file): SortingExtractor.__init__(self) if is_kbucket_url(firings_file): download_needed = is_url(mt.findFile(path=firings_file)) else: download_needed = is_url(firings_file) if download_needed: print('Downloading file: ' + firings_file) self._firings_path = mt.realizeFile(path=firings_file) print('Done.') else: self._firings_path = mt.realizeFile(path=firings_file) if not self._firings_path: raise Exception('Unable to realize firings file: ' + firings_file) self._firings = readmda(self._firings_path) self._times = self._firings[1, :] self._labels = self._firings[2, :] self._unit_ids = np.unique(self._labels).astype(int)
def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running surface3D') mt.configDownloadFrom(state.get('download_from', [])) python_state = dict() path0 = state.get('faces_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return faces0 = np.load(x) # there's a better way to do the following # (need to get it into a single vector format) faces = [] for j in range(faces0.shape[0]): # 3 = #vertices in polygon (assume a triangulation) faces.extend([3, faces0[j, 0], faces0[j, 1], faces0[j, 2]]) faces = np.array(faces) python_state['faces'] = faces path0 = state.get('vertices_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return vertices0 = np.load(x) python_state['vertices'] = vertices0.T path0 = state.get('scalars_path', None) if path0: x = mt.realizeFile(path0) if not x: self._set_error('Unable to load file: {}'.format(path0)) return x = np.load(x) python_state['scalars'] = x python_state['status'] = 'finished' python_state['status_message'] = 'finished' self.set_python_state(python_state)
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) max_samples = state.get('max_samples') max_dt_msec = state.get('max_dt_msec') bin_size_msec = state.get('bin_size_msec') if not max_dt_msec: return firings_path = state.get('firingsPath', None) if not firings_path: self.set_python_state(dict( status='error', status_message='No firingsPath provided' )) return samplerate = state.get('samplerate', None) if not samplerate: self.set_python_state(dict( status='error', status_message='No samplerate provided' )) return self.set_python_state(dict(status_message='Realizing file: {}'.format(firings_path))) firings_path2 = mt.realizeFile(firings_path) if not firings_path2: self.set_python_state(dict( status='error', status_message='Unable to realize file: {}'.format(firings_path) )) return result = ComputeAutocorrelograms.execute( firings_path=firings_path2, samplerate=samplerate, max_samples=max_samples, bin_size_msec=bin_size_msec, max_dt_msec=max_dt_msec, json_out=dict(ext='.json') ) if result.retcode != 0: self.set_python_state(dict( status='error', status_message='Error computing autocorrelogram.' )) return output = mt.loadObject(path=result.outputs['json_out']) self.set_python_state(dict( status='finished', output=output ))
def plot(self, name: str, format: str = 'image'): plots = self._obj['summary'].get('plots', dict()) url = plots[name] if format == 'url': return url else: path = mt.realizeFile(url) if format == 'image': return Image.open(path) elif format == 'path': return path else: raise Exception('Invalid format: ' + format)
def h5_to_dict(fname, *, upload_to=None, use_cache=False): if use_cache: result = H5ToDict.execute(h5_in=fname, upload_to=upload_to or '', json_out={'ext': '.json'}) if result.retcode != 0: raise Exception('Problem running H5ToDict.') return mt.loadObject(path=result.outputs['json_out']) fname = mt.realizeFile(path=fname) opts = dict(upload_to=upload_to) with h5py.File(fname, 'r') as f: opts['file'] = f return _h5_to_dict(f, opts=opts, name=None)
def plot(self, name: str, format: str = 'image'): assert self._summary_result is not None, 'No summary result found' plots = self._summary_result.get('plots', dict()) url = plots[name] if format == 'url': return url else: path = mt.realizeFile(url) if format == 'image': return Image.open(path) elif format == 'path': return path else: raise Exception('Invalid format: ' + format)
def _extract_time_ranges(obj, time_ranges): positions_path = obj['processing']['Behavior']['Position']['Position'][ '_datasets']['data']['_data'] timestamps_path = obj['processing']['Behavior']['Position']['Position'][ '_datasets']['timestamps']['_data'] spike_times_path = obj['units']['_datasets']['spike_times']['_data'] spike_times_index = obj['units']['_datasets']['spike_times_index']['_data'] positions = np.load(mt.realizeFile(path=positions_path)) timestamps = np.load(mt.realizeFile(path=timestamps_path)) spike_times = np.load(mt.realizeFile(path=spike_times_path)) selector = np.full(timestamps.shape, False) for time_range in time_ranges: a = (time_range[0] <= timestamps) & (timestamps < time_range[1]) selector = selector | a if np.all(selector): return obj timestamps = timestamps[selector] positions = positions[selector, :] selector2 = np.full(spike_times.shape, False) for time_range in time_ranges: a = (time_range[0] <= spike_times) & (spike_times < time_range[1]) selector2 = selector2 | a for j in range(len(spike_times_index)): spike_times_index[j] = np.count_nonzero( selector2[:spike_times_index[j]]) spike_times = spike_times[selector2] obj = deepcopy(obj) obj['processing']['Behavior']['Position']['Position']['_datasets']['data'][ '_data'] = _np_snapshot(positions) obj['processing']['Behavior']['Position']['Position']['_datasets'][ 'timestamps']['_data'] = _np_snapshot(timestamps) obj['units']['_datasets']['spike_times']['_data'] = _np_snapshot( spike_times) obj['units']['_datasets']['spike_times_index']['_data'] = spike_times_index return obj
def __init__(self, *, recording_directory=None, timeseries_path=None, download=True, samplerate=None, geom=None, geom_path=None, params_path=None): RecordingExtractor.__init__(self) if recording_directory: timeseries_path = recording_directory + '/raw.mda' geom_path = recording_directory + '/geom.csv' params_path = recording_directory + '/params.json' self._timeseries_path = timeseries_path if params_path: self._dataset_params = read_dataset_params(params_path) self._samplerate = self._dataset_params['samplerate'] else: self._dataset_params = dict(samplerate=samplerate) self._samplerate = samplerate if download: path0 = mt.realizeFile(path=self._timeseries_path) if not path0: raise Exception('Unable to realize file: ' + self._timeseries_path) self._timeseries_path = path0 X = DiskReadMda(self._timeseries_path) if geom: self._geom = geom elif geom_path: self._geom = np.genfromtxt(geom_path, delimiter=',') else: self._geom = np.zeros((X.N1(), 2)) if self._geom.shape[0] != X.N1(): # raise Exception( # 'Incompatible dimensions between geom.csv and timeseries file {} <> {}'.format(self._geom.shape[0], X.N1())) print( 'WARNING: Incompatible dimensions between geom.csv and timeseries file {} <> {}' .format(self._geom.shape[0], X.N1())) self._geom = np.zeros((X.N1(), 2)) self._num_channels = X.N1() self._num_timepoints = X.N2() for m in range(self._num_channels): self.set_channel_property(m, 'location', self._geom[m, :])
def javascript_state_changed(self, prev_state, state): self.set_state(dict(status='running', status_message='Running')) vtp_path = state.get('vtp_path', None) download_from = state.get('download_from', None) scalar_info = state.get('scalar_info', None) vector_field_info = state.get('vector_field_info', None) arrow_subsample_factor = state.get('arrow_subsample_factor', None) if not vtp_path: self.set_state(dict(status='error', status_message='No vtp_path')) return if download_from: mt.configDownloadFrom(download_from) fname = mt.realizeFile(path=vtp_path) reader = vtkXMLPolyDataReader() reader.SetFileName(fname) reader.Update() X = reader.GetOutput() vertices = vtk_to_numpy(X.GetPoints().GetData()).T faces = vtk_to_numpy(X.GetPolys().GetData()) if scalar_info: scalars = vtk_to_numpy(X.GetPointData().GetArray( scalar_info['name'])) scalars = scalars[:, scalar_info['component']] else: scalars = None if vector_field_info: vector_field = vtk_to_numpy(X.GetPointData().GetArray( vector_field_info['name'])) vector_field = vector_field[:, vector_field_info['components']] arrows = [ dict(start=vertices[:, j] - vector_field[j, :].T / 2, end=vertices[:, j] + vector_field[j, :].T / 2) for j in range(0, vector_field.shape[0], arrow_subsample_factor) ] else: arrows = None self.set_state( dict(status='finished', vertices=vertices, faces=faces, scalars=scalars, arrows=arrows))
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) nwb_path = state.get('nwb_path', None) downsample_factor = state.get('downsample_factor', 1) if nwb_path: if nwb_path.endswith('.nwb'): self.set_python_state( dict(status_message='Realizing object from nwb file: {}'. format(nwb_path))) obj = h5_to_dict(nwb_path, use_cache=True) else: self.set_python_state( dict(status_message='Realizing object: {}'.format( nwb_path))) obj = mt.loadObject(path=nwb_path) if not obj: self.set_python_state( dict(status='error', status_message='Unable to realize object: {}'.format( nwb_path))) return try: positions_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['data']['_data'] except: self.set_python_state( dict(status='error', status_message= 'Problem extracting behavior positions in file: {}'. format(nwb_path))) return positions = np.load(mt.realizeFile(path=positions_path)) positions = positions[::downsample_factor, :] self.set_python_state( dict(status_message='Finished loading positions')) state['positions'] = positions state['status'] = 'finished' self.set_python_state(state) else: self.set_python_state( dict(status='error', status_message='Missing in state: nwb_path'))
def _initialize(context, unit_id, connection_to_parent): with StdoutSender(connection=connection_to_parent): try: print('***** Preparing efficient access recording extractor...') earx = EfficientAccessRecordingExtractor( recording=context.recordingExtractor()) print('***** Computing unit detail...') path0 = mt.realizeFile(path=ComputeUnitDetail.execute( recording=earx, sorting=context.trueSortingExtractor(), unit_id=unit_id, output=True).outputs['output']) with open(path0, 'rb') as f: result0 = pickle.load(f) print('*****') except: traceback.print_exc() raise connection_to_parent.send(dict(name='result', result=result0))
def compute_units_info_b(*, recording_dir, firings, channel_ids=[], unit_ids=[], return_format='json'): out = ComputeUnitsInfo.execute(recording_dir=recording_dir, firings=firings, unit_ids=unit_ids, channel_ids=channel_ids, json_out={ 'ext': '.json' }, _container='default').outputs fname = out['json_out'] if return_format == 'filename': return fname else: fname = mt.realizeFile(path=fname) with open(fname) as f: return json.load(f)
def __init__(self, *, path=None, recording=None, _dest_path=None): #import h5py se.RecordingExtractor.__init__(self) if path is not None: if recording is not None: raise Exception( 'Cannot pass both path and recording to EfficientAccessRecordingExtractor.' ) self._path = path elif recording is not None: if not hasattr(recording, 'hash'): print(''' Warning: Recording does not have the hash attribute. Using sampling method to compute a hash.''') setattr(recording, 'hash', _samplehash(recording)) result = CreateEfficientAccessRecordingFile.execute( recording=recording, hdf5_out=dict(ext='.hdf5', dest_path=_dest_path)) path0 = result.outputs['hdf5_out'] self._path = mt.realizeFile(path=path0) else: raise Exception('Missing argument: path or recording') with h5py.File(self._path, "r") as f: self._num_segments = int(np.array(f.get('num_segments'))[0]) self._segment_size = int(np.array(f.get('segment_size'))[0]) self._num_channels = int(np.array(f.get('num_channels'))[0]) self._channel_ids = np.array(f.get('channel_ids')).tolist() self._num_timepoints = int(np.array(f.get('num_timepoints'))[0]) self._samplerate = np.array(f.get('samplerate'))[0] self._recording_hash = np.array( f.get('recording_hash'))[0].decode() assert type(self._recording_hash) == str geom = np.array(f.get('geom')) channel_locations = [ geom[m, :].ravel() for m in range(self._num_channels) ] self.set_channel_locations(channel_ids=self._channel_ids, locations=channel_locations)
def javascript_state_changed(self, prev_state, state): self._set_status('running', 'Running') path = state.get('path', None) download_from = state.get('download_from', []) mt.configDownloadFrom(download_from) if not path: self._set_status('finished') return path = mt.realizeFile(path) if not path: self._set_error('Unable to realize file.') return with open(path, 'rb') as f: video_data = f.read() video_data_b64 = base64.b64encode(video_data).decode() self.set_python_state( dict(video_data_b64=video_data_b64, status='finished', status_message=''))
def nwb_to_dict(fname, *, upload_to=None, use_cache=False, exclude_data=False, verbose=False): if fname.endswith('.json'): with open(fname, 'r') as f: return simplejson.load(f) if use_cache: result = NWBToDict.execute(h5_in=fname, upload_to=upload_to or '', json_out={'ext': '.json'}) if result.retcode != 0: raise Exception('Problem running NWBToDict.') return mt.loadObject(path=result.outputs['json_out']) fname = mt.realizeFile(path=fname) opts = dict(upload_to=upload_to, exclude_data=exclude_data, verbose=verbose) with h5py.File(fname, 'r') as f: opts['file'] = f return _nwb_to_dict(f, opts=opts, name=None, path='')
def javascript_state_changed(self, prev_state, state): self.set_status('running') self.set_status_message('Running') mt.configDownloadFrom(state.get('download_from', [])) nwb_query = state.get('nwb_query', None) downsample_factor = state.get('downsample_factor', 1) if nwb_query: self.set_status_message('Loading nwb object') obj = _load_nwb_object(nwb_query) if not obj: self.set_error('Unable to load nwb object') return self.set_status_message('Loading positions and timestamps from') try: positions_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['data']['_data'] timestamps_path = obj['processing']['Behavior']['Position'][ 'Position']['_datasets']['timestamps']['_data'] except: self.set_error( 'Problem extracting behavior positions or timestamps') return positions = np.load(mt.realizeFile(path=positions_path)) positions = positions[::downsample_factor, :] timestamps = np.load(mt.realizeFile(path=timestamps_path)) timestamps = timestamps[::downsample_factor] self.set_status_message('Loading spike times') try: spike_times_path = obj['units']['_datasets']['spike_times'][ '_data'] spike_times_index = obj['units']['_datasets'][ 'spike_times_index']['_data'] spike_times_index_id = obj['units']['_datasets']['id']['_data'] if 'cluster_name' in obj['units']['_datasets']: cluster_names = obj['units']['_datasets']['cluster_name'][ '_data'] else: cluster_names = [] except: self.set_error('Problem extracting spike times') return spike_times = np.load(mt.realizeFile(path=spike_times_path)) spike_time_indices = _find_closest(timestamps, spike_times) spike_labels = np.zeros(spike_time_indices.shape) aa = 0 for i, val in enumerate(spike_times_index): spike_labels[aa:val] = spike_times_index_id[i] aa = val all_unit_ids = sorted(list(set(spike_labels))) state['positions'] = positions state['status'] = 'finished' state['spike_time_indices'] = spike_time_indices state['spike_labels'] = spike_labels state['all_unit_ids'] = all_unit_ids state['cluster_names'] = cluster_names self.set_python_state(state) self.set_status('finished') else: self.set_error('Missing in state: nwb_query')
return None # studysets_to_include = ['PAIRED_BOYDEN', 'PAIRED_CRCNS_HC1', 'PAIRED_MEA64C_YGER', 'PAIRED_KAMPFF', 'PAIRED_MONOTRODE', 'SYNTH_MONOTRODE', 'SYNTH_MAGLAND', 'SYNTH_MEAREC_NEURONEXUS', 'SYNTH_MEAREC_TETRODE', 'SYNTH_MONOTRODE', 'SYNTH_VISAPY', 'HYBRID_JANELIA', 'MANUAL_FRANKLAB'] studysets_to_include = ['SYNTH_BIONET'] fnames = ['geom.csv', 'params.json', 'raw.mda', 'firings_true.mda'] # fnames = ['geom.csv', 'params.json', 'firings_true.mda'] # fnames = ['geom.csv', 'params.json'] for studyset in X['StudySets']: print('STUDYSET: {}'.format(studyset['name'])) if studyset['name'] in studysets_to_include: for study in studyset['studies']: study_name = study['name'] print('STUDY: {}'.format(study_name)) for recording in study['recordings']: recname = recording['name'] recdir = recording['directory'] print('RECORDING: {}'.format(recname), recdir) sha1 = get_sha1_part_of_sha1dir(recdir) if sha1: ff = mt.realizeFile('sha1://' + sha1) print('Storing directory index file: {} for sha1={}'.format(ff, sha1)) ka.store_file(ff) for fname in fnames: print('Realizing file: {}'.format(recdir + '/' + fname)) ff = mt.realizeFile(path=recdir + '/' + fname) if ff: print('Storing file: {}'.format(ff)) ka.store_file(ff) else: print('WARNING: could not realize file: {}'.format(recdir + '/' + fname))
def compute_mandelbrot_parallel(*, xmin=-2, xmax=0.5, ymin=-1.25, ymax=1.25, num_x=1000, num_iter=1000, num_parallel=1, _force_run=False, _container=None, srun_opts=None, use_slurm=False): subsampling_factor = num_parallel jobs = [] job_args = [ dict(num_x=num_x, num_iter=num_iter, subsampling_factor=subsampling_factor, subsampling_offset=offset, output_npy=dict(ext='.npy', upload=True), _force_run=_force_run, _container=_container, _compute_requirements=dict(batch_type='cpu1')) for offset in range(subsampling_factor) ] jobs = ComputeMandelbrot.createJobs(job_args) working_dir = 'tmp_slurm_working_dir_' + _random_string(5) if use_slurm: H = mlpr.SlurmJobHandler(working_dir=working_dir) H.addBatchType(name='cpu1', num_workers_per_batch=4, num_cores_per_job=2, use_slurm=False) else: H = mlpr.ParallelJobHandler(num_workers=num_parallel) with mlpr.JobQueue(job_handler=H) as jq: results = [] for job in jobs: result0 = job.execute() results.append(result0) # results = mlpr.executeBatch(jobs=jobs, srun_opts=srun_opts) X_list = [] for result0 in results: # result0.wait() # X0 = np.load(mt.realizeFile(result0.outputs['output_npy'])) # X_list.append(X0) X_list.append(result0.outputs['output_npy']) # X = combine_subsampled_mandelbrot(X_list) A = CombineSubsampledMandelbrot.execute( num_x=num_x, X_list=X_list, X_out={'ext': '.npy'}, _force_run=_force_run, _container=_container, _compute_requirements=dict(batch_type='cpu1')) jq.wait() X_path = A.outputs['X_out'] X = np.load(mt.realizeFile(X_path)) return X