def test1(): import kachery_p2p as kp w = fs.load_workspace() print(w.get_uri()) geom_uri = 'sha1://fce1fb4c8637a36edb34669e1ac612700ce7151e/lens_r01.go3' job_cache = hi.JobCache(feed_name='job-cache') with hi.Config(use_container=True, show_console=True, job_cache=None): j: hi.Job = miniwasp_hither.run(geom_uri=geom_uri, omega=3.141592*2/330.0, ppw=50) H, E = j.wait().return_value H = cast(np.ndarray, H) E = cast(np.ndarray, E) print(H.shape, E.shape) [nc, N1, N2, N3] = H.shape transformation = np.array([ [1/N1, 0, 0, 0], [0, 1/N2, 0, 0], [0, 0, 1/N3, 0] ]) f = fs.FieldModel(label='miniwasp-H', data=H, components=['x', 'y', 'z'], transformation=transformation) w.add_field_model(f) f = fs.FieldModel(label='miniwasp-E', data=E, components=['x', 'y', 'z'], transformation=transformation) w.add_field_model(f)
def createjob_get_firing_data(labbox, sorting_object, recording_object, configuration): jh = labbox.get_job_handler('partition1') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): return get_firing_data.run(sorting_object=sorting_object, recording_object=recording_object, configuration=configuration)
def example4(): import kachery_p2p as kp geom_uri = 'sha1://fce1fb4c8637a36edb34669e1ac612700ce7151e/lens_r01.go3' job_cache = hi.JobCache(feed_name='job-cache') with hi.Config(use_container=True, show_console=True, job_cache=None): j: hi.Job = example4_hither.run(geom_uri=geom_uri, omega=3.141592*2/330.0, ppw=50) x = j.wait().return_value return x
def preload_extract_snippets(labbox, recording_object, sorting_object): from labbox_ephys import prepare_snippets_h5 jh = labbox.get_job_handler('partition2') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): snippets_h5 = prepare_snippets_h5.run( recording_object=recording_object, sorting_object=sorting_object) return snippets_h5
def createjob_get_unit_snrs(labbox, sorting_object, recording_object, configuration={}): jh = labbox.get_job_handler('partition1') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): snippets_h5 = prepare_snippets_h5.run( recording_object=recording_object, sorting_object=sorting_object) return get_unit_snrs.run(snippets_h5=snippets_h5)
def createjob_individual_cluster_features(labbox, recording_object, sorting_object, unit_id): from labbox_ephys import prepare_snippets_h5 jh = labbox.get_job_handler('partition1') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): snippets_h5 = prepare_snippets_h5.run( recording_object=recording_object, sorting_object=sorting_object) return individual_cluster_features.run(snippets_h5=snippets_h5, unit_id=unit_id)
def createjob_fetch_average_waveform_plot_data(labbox, recording_object, sorting_object, unit_id): from labbox_ephys import prepare_snippets_h5 jh = labbox.get_job_handler('partition1') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): snippets_h5 = prepare_snippets_h5.run( recording_object=recording_object, sorting_object=sorting_object) return fetch_average_waveform_plot_data.run(snippets_h5=snippets_h5, unit_id=unit_id)
def createjob_sample_data_object_slices(labbox, data_uri, slices, component_indices: List[int], mode: str): # sample_data_object_slices(data_uri, slices, component_indices) with hi.Config(job_cache=labbox.get_job_cache()): return sample_data_object_slices.run( data_uri=data_uri, slices=slices, component_indices=component_indices, mode=mode)
def createjob_get_sorting_unit_snippets(labbox, recording_object, sorting_object, unit_id, time_range, max_num_snippets): from labbox_ephys import prepare_snippets_h5 jh = labbox.get_job_handler('partition1') jc = labbox.get_job_cache() with hi.Config(job_cache=jc, job_handler=jh, use_container=jh.is_remote()): snippets_h5 = prepare_snippets_h5.run( recording_object=recording_object, sorting_object=sorting_object) return get_sorting_unit_snippets.run(snippets_h5=snippets_h5, unit_id=unit_id, time_range=time_range, max_num_snippets=max_num_snippets)
def main(): (params, std_args) = init_configuration() study_sets = load_study_records(params.study_source_file) study_matrix = parse_sorters(params.sorter_spec_file, list(study_sets.keys())) sorting_matrix = populate_sorting_matrix(study_matrix, study_sets) sorting_matrix = remove_preexisting_records(sorting_matrix, params.workspace_uri) hither_config = extract_hither_config(std_args) jobs: hi.Job = [] try: with hi.Config(**hither_config): sortings = list(sorting_loop(sorting_matrix)) with hi.Config(job_handler=None, job_cache=None): for sorting in sortings: p = { 'sorting_entry': sorting, 'workspace_uri': params.workspace_uri } jobs.append(hi.Job(hi_post_result_to_workspace, p)) hi.wait(None) finally: call_cleanup(hither_config)
def main(): (args, std_args) = init_configuration() study_sets = load_study_records(args['study_source_file']) study_matrix = parse_sorters(args['sorter_spec_file'], list(study_sets.keys())) sorting_matrix = populate_sorting_matrix(study_matrix, study_sets) hither_config = extract_hither_config(std_args) try: with hi.Config(**hither_config): sortings = list(sorting_loop(sorting_matrix)) hi.wait(None) finally: call_cleanup(hither_config) results: List[OutputRecord] = [make_output_record(job) for job in sortings] output_records(results, std_args)
def main(): (args, std_args) = init_configuration() sortings = load_sortings(args['sortingsfile']) if args['recordingset'] is not None and args['recordingset'] != '': sortings = [ s for s in sortings if s['studyName'] == args['recordingset'] ] hither_config = extract_hither_config(std_args) comparison_list = [] try: print(f"\t\tScript execution beginning at {time.ctime()}") start_time = time.time() with hi.Config(**hither_config): extraction_loop(sortings, comparison_list, std_args['test']) hi.wait(None) finally: call_cleanup(hither_config) output_results(comparison_list, std_args['outfile']) print(f"\n\n\t\tElapsed time: {time.time() - start_time:.3f} sec") print(f"\t\tScript execution complete at {time.ctime()}")
def test_sorting(sorter_func): import sortingview as sv recording_name = 'paired_kampff/2014_11_25_Pair_3_0' recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json' sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json' recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True) sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri) channel_ids = recording.get_channel_ids() samplerate = recording.get_sampling_frequency() num_timepoints = recording.get_num_frames() print(f'{recording_name}') print( f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})' ) unit_ids = sorting_true.get_unit_ids() spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0]) print(f'Unit {unit_ids[0]} has {len(spike_train)} events') jh = hi.ParallelJobHandler(num_workers=4) # jh = hi.SlurmJobHandler(num_jobs_per_allocation=4, max_simultaneous_allocations=4, srun_command='') log = hi.Log() with hi.Config(use_container=True, job_handler=jh, log=log, show_console=True): sorting_object = hi.Job(sorter_func, { 'recording_object': recording.object() }).wait().return_value sorting = sv.LabboxEphysSortingExtractor(sorting_object) unit_ids = sorting.get_unit_ids() spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0]) print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
def test_sorting(sorter_func, *, show_console=True, job_handler: Union[None, hi.JobHandler] = None): import sortingview as sv recording_name = 'paired_kampff/2014_11_25_Pair_3_0' recording_uri = 'sha1://a205f87cef8b7f86df7a09cddbc79a1fbe5df60f/2014_11_25_Pair_3_0.json' sorting_uri = 'sha1://c656add63d85a17840980084a1ff1cdc662a2cd5/2014_11_25_Pair_3_0.firings_true.json' recording = sv.LabboxEphysRecordingExtractor(recording_uri, download=True) sorting_true = sv.LabboxEphysSortingExtractor(sorting_uri) channel_ids = recording.get_channel_ids() samplerate = recording.get_sampling_frequency() num_timepoints = recording.get_num_frames() print(f'{recording_name}') print( f'Recording has {len(channel_ids)} channels and {num_timepoints} timepoints (samplerate: {samplerate})' ) unit_ids = sorting_true.get_unit_ids() spike_train = sorting_true.get_unit_spike_train(unit_id=unit_ids[0]) print(f'Unit {unit_ids[0]} has {len(spike_train)} events') with hi.Config(use_container=True, show_console=show_console, job_handler=job_handler): sorting_object = hi.Job(sorter_func, { 'recording_object': recording.object() }).wait().return_value sorting = sv.LabboxEphysSortingExtractor(sorting_object) unit_ids = sorting.get_unit_ids() spike_train = sorting.get_unit_spike_train(unit_id=unit_ids[0]) print(f'Unit {unit_ids[0]} has {len(spike_train)} events')
def handle_message(self, msg): type0 = msg.get('type') if type0 == 'hitherCreateJob': functionName = msg['functionName'] kwargs = msg['kwargs'] client_job_id = msg['clientJobId'] self._log(f'hitherCreateJob-1 {functionName} {client_job_id}') try: f = hi2.get_function(functionName) if f is not None: with hi2.Config(log=self._hither_log): job_or_result = f(**kwargs, labbox=self._labbox_context) else: raise Exception( f'Hither function not registered: {functionName}') except Exception as err: self._send_message({ 'type': 'hitherJobError', 'job_id': client_job_id, 'client_job_id': client_job_id, 'error_message': f'Error creating outer job: {str(err)}', 'runtime_info': None }) return if isinstance(job_or_result, hi2.Job): job: hi2.Job = job_or_result setattr(job, '_client_job_id', client_job_id) job_id = job.job_id self._jobs_by_id[job_id] = job print( f'======== Created hither job (2): {job_id} {functionName}' ) self._log( f'hitherCreateJob-2 {functionName} {client_job_id} {job_id}' ) self._send_message({ 'type': 'hitherJobCreated', 'job_id': job_id, 'client_job_id': client_job_id }) else: self._log(f'hitherCreateJob-3 {functionName} {client_job_id}') result = job_or_result msg = { 'type': 'hitherJobFinished', 'client_job_id': client_job_id, 'job_id': client_job_id, # 'result': _make_json_safe(result), 'result_sha1': _get_sha1_from_uri(kp.store_json(_make_json_safe(result))), 'runtime_info': {} } elif type0 == 'hitherCancelJob': job_id = msg['job_id'] self._log(f'hitherCancelJob-1 {job_id}') assert job_id, 'Missing job_id' assert job_id in self._jobs_by_id, f'No job with id: {job_id}' job = self._jobs_by_id[job_id] job.cancel() elif type0 == 'subfeedMessageRequest': request_id = msg['requestId'] feed_uri = msg['feedUri'] if not feed_uri: feed_uri = 'feed://' + self._default_feed_id subfeed_name = msg['subfeedName'] position = msg['position'] wait_msec = msg['waitMsec'] self._log( f'subfeed_message_request-1 {request_id} {feed_uri} {subfeed_name} {position} {wait_msec}' ) self._subfeed_message_requests[request_id] = { 'feed_uri': feed_uri, 'subfeed_name': subfeed_name, 'position': position, 'wait_msec': wait_msec, 'timestamp': time.time() }
def run_sorter_docker(sorter_name, recording, output_folder, delete_output_folder=False, grouping_property=None, parallel=False, verbose=False, raise_error=True, n_jobs=-1, joblib_backend='loky', use_docker=True, container=None, **params): if use_docker: # if container is None: # assert sorter_name in default_docker_images, f"Default docker image for {sorter_name} not found" # docker_image = default_docker_images[sorter_name] # # print(f"Running in docker image {docker_image.get_name()}") output_folder = Path(output_folder).absolute() output_folder.mkdir(exist_ok=True, parents=True) # dump recording with relative file paths to docker container /input folder dump_dict_container, input_directory = modify_input_folder( recording.dump_to_dict(), '/input') with hither.Config(use_container=False, show_console=True): kwargs = dict(recording_dict=dump_dict_container, sorter_name=sorter_name, output_folder=str(output_folder), delete_output_folder=False, grouping_property=grouping_property, parallel=parallel, verbose=verbose, raise_error=raise_error, n_jobs=n_jobs, joblib_backend=joblib_backend) kwargs.update(params) kwargs.update({ 'input_directory': str(input_directory), 'output_directory': str(output_folder) }) sorting_job = hither.Job(run_sorter_docker_with_container, kwargs) sorting_job.wait() sorting = se.NpzSortingExtractor(output_folder / "sorting_docker.npz") else: # standard call sorting = ss.run_sorter(sorter_name, recording, output_folder=output_folder, delete_output_folder=delete_output_folder, grouping_property=grouping_property, parallel=parallel, verbose=verbose, raise_error=raise_error, n_jobs=n_jobs, joblib_backend=joblib_backend, **params) return sorting
def preload_download_recording(labbox, recording_object): jh = labbox.get_job_handler('partition1') with hi.Config(job_handler=jh, use_container=jh.is_remote()): return preload_download_recording_2.run( recording_object=recording_object)
def preload_check_sorting_downloaded(labbox, sorting_object): jh = labbox.get_job_handler('partition1') with hi.Config(job_handler=jh, use_container=jh.is_remote()): return preload_check_sorting_downloaded_2.run( sorting_object=sorting_object)