def initiate_next_module(next_module, npx_directory, json_directory): logger_dict[npx_directory].info('initiating {} for {}'.format( next_module, npx_directory)) now = datetime.datetime.now() this_module_info = module_info(None, None, None, now, None, None) info_dict[npx_directory][next_module] = this_module_info failed = 0 try: session_id = os.path.basename(npx_directory) input_json = os.path.join( json_directory, session_id + '_' + next_module + '-input.json') output_json = os.path.join( json_directory, session_id + '_' + next_module + '-output.json') dir_name, file_name = os.path.split(npx_directory) file_name = file_name + '_sorted' extract_from = npx_directories[npx_directory].backup1 extract_to = os.path.join(dir_name, file_name) info = createInputJson(input_json, npx_directory=extract_from, extracted_data_directory=extract_to) command_string = [ "python", "-m", "ecephys_spike_sorting.modules." + next_module, "--input_json", input_json, "--output_json", output_json ] logger_dict[npx_directory].info(command_string) start_module(npx_directory, next_module, command_string, info) except Exception as E: logger_dict[npx_directory].error("Error initiating " + next_module + " for " + npx_directory) logger_dict[npx_directory].exception(E) failed = E return next_module, this_module_info, failed
continuous_file = os.path.join(input_data_directory, fileName) metaName = runName + '_g' + spec[2] + '_t' + repr( first_trig) + '.imec.ap.meta' input_meta_fullpath = os.path.join(input_data_directory, metaName) print(input_meta_fullpath) info = createInputJson(catGT_input_json[i], npx_directory=input_data_directory, continuous_file=continuous_file, kilosort_output_directory=catGT_dest, spikeGLX_data=True, input_meta_path=input_meta_fullpath, catGT_run_name=spec[0], gate_string=spec[2], trigger_string=trigger_str, probe_string='0', catGT_stream_string='-ap', catGT_car_mode=car_mode, catGT_loccar_min_um=loccar_min, catGT_loccar_max_um=loccar_max, catGT_cmd_string=catGT_cmd_string + ' ' + extract_string, extracted_data_directory=catGT_sub) #create json files for the other modules session_id.append(spec[0] + '_imec' + prb) module_input_json.append( os.path.join(json_directory, session_id[i] + '-input.json'))
prb_list = SpikeGLX_utils.ParseProbeStr(spec[3]) # build path to the first probe folder run_folder_name = spec[0] + '_g' + spec[1] prb0_fld_name = run_folder_name + '_imec' + prb_list[0] prb0_fld = os.path.join(npx_directory, run_folder_name, prb0_fld_name) first_trig, last_trig = SpikeGLX_utils.ParseTrigStr(spec[2], prb0_fld) trigger_str = repr(first_trig) + ',' + repr(last_trig) print('Creating json file for preprocessing') info = createInputJson(input_json, npx_directory=npx_directory, continuous_file=None, spikeGLX_data='True', kilosort_output_directory=catGT_dest, catGT_run_name=session_id, gate_string=spec[1], trigger_string=trigger_str, probe_string=spec[3], catGT_stream_string=catGT_stream_string, catGT_cmd_string=catGT_cmd_string, extracted_data_directory=catGT_dest) # CatGT operates on whole runs with multiple probes, so gets called in just # once per run_spec if run_CatGT: command = "python -W ignore -m ecephys_spike_sorting.modules." + 'catGT_helper' + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' ')) # parse the CatGT log and write results to command line logPath = os.getcwd()
# get region specific parameters ks_Th = ksTh_dict.get(spec[1][0]) refPerMS = refPerMS_dict.get(spec[1][0]) info = createInputJson(module_input_json[i], npx_directory=npx_directory, continuous_file=continuous_file, spikeGLX_data=True, input_meta_path=input_meta_fullpath, kilosort_output_directory=kilosort_output_dir, ks_make_copy=ks_make_copy, noise_template_use_rf=False, catGT_run_name=session_id[i], ks_remDup=ks_remDup, ks_finalSplits=1, ks_labelGood=1, ks_saveRez=ks_saveRez, ks_copy_fproc=ks_copy_fproc, ks_minfr_goodchannels=ks_minfr_goodchannels, ks_whiteningRadius_um=ks_whiteningRadius_um, ks_Th=ks_Th, ks_CSBseed=1, ks_LTseed=1, ks_templateRadius_um=ks_templateRadius_um, extracted_data_directory=npx_directory, c_Waves_snr_um=c_Waves_snr_um, qm_isi_thresh=refPerMS / 1000) # copy json file to data directory as record of the input parameters # loop over files again for processing.
input_json = os.path.join(json_directory, session_id + '-input.json') # kilosort_postprocessing and noise_templates moduules alter the files # that are input to phy. If using these modules, keep a copy of the # original phy output if ('kilosort_postprocessing' in modules) or ('noise_templates' in modules): ks_make_copy = True else: ks_make_copy = False print('Creating json file for KS2 and postprocessing') print('npx_directory:', npx_directory) print('continuous file: ', os.path.join(npx_directory, name)) info = createInputJson(input_json, npx_directory=npx_directory, continuous_file=os.path.join(npx_directory, name), spikeGLX_data='True', kilosort_output_directory=kilosort_output_dir, ks_make_copy=ks_make_copy, extracted_data_directory=npx_directory, noise_template_use_rf=False, CSBseed=run_seed) for module in modules: output_json = os.path.join(json_directory, session_id + '-' + module + '-output.json') command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' '))
#'/mnt/md0/data/mouse412804/766640955_412804_20181022_probeC_sorted/continuous/Neuropix-3a-100.0'] #npx_directories = [r'L:\766640955_412804_20181022_probeC'] probe_type = '3A' json_directory = r'C:\Users\svc_neuropix\Documents\json_files' #json_directory = r'C:\Users\svc_neuropix\Documents\json_files' for directory in sorted_directories: session_id = os.path.basename(directory) input_json = os.path.join(json_directory, session_id + '-input.json') output_json = os.path.join(json_directory, session_id + '-output.json') info = createInputJson(input_json, npx_directory=directory) modules = [ #'extract_from_npx', #'depth_estimation' #'median_subtraction', #'kilosort_helper', #'kilosort_postprocessing' #'noise_templates', 'mean_waveforms', 'quality_metrics' ] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json
os.path.join(local_sorting_directory,'probe_info.json')) session_id = os.path.basename(local_sorting_directory) target_directory = os.path.join(r'\\10.128.50.77', 'sd5.3', 'RE-SORT', session_id[:-7], session_id, 'continuous', 'Neuropix-3a-100.0') input_json = os.path.join(json_directory, session_id + '_resort-input.json') output_json = os.path.join(json_directory, session_id + '_resort-output.json') info = createInputJson(input_json, kilosort_output_directory=local_sorting_directory, extracted_data_directory=local_sorting_directory) modules = [ 'kilosort_helper', 'kilosort_postprocessing', 'noise_templates', 'mean_waveforms', 'quality_metrics'] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' ')) copy_sorted_files(local_sorting_directory, target_directory)
] #npx_directories = [r'L:\766640955_412804_20181022_probeC'] probe_type = '3A' json_directory = '/mnt/md0/data/json_files' #json_directory = r'C:\Users\svc_neuropix\Documents\json_files' for directory in sorted_directories: session_id = os.path.basename(directory) input_json = os.path.join(json_directory, session_id + '-input.json') output_json = os.path.join(json_directory, session_id + '-output.json') info = createInputJson(input_json, kilosort_output_directory=directory) modules = [ #'extract_from_npx', #'depth_estimation', ##'median_subtraction', #'kilosort_helper', #'kilosort_postprocessing', #'noise_templates', #'mean_waveforms', 'quality_metrics' ] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json
input_json = os.path.join(fi, session_id +'-inputClassifier.json') output_json = os.path.join(fi, session_id +'-outputClassifier.json') print( 'Creating json file postprocessing') info = createInputJson(input_json, npx_directory=npx_directory, continuous_file = npx_file[0], kilosort_output_directory=fi, ks_make_copy = False, noise_template_use_rf = False, cluster_group_file_name = 'classifier_cluster_heuristic.txt', extracted_data_directory = npx_directory ) print(info['noise_waveform_params']['use_preclustered']) for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' '))
probe_type = 'NP1' json_directory = r'D:\ecephys_fork\json_files' for directory in sorted_directories: session_id = os.path.basename(directory) input_json = os.path.join(json_directory, session_id + '-input.json') output_json = os.path.join(json_directory, session_id + '-output.json') info = createInputJson(input_json, npx_directory=None, continuous_file = None, kilosort_output_directory=directory, kilosort_output_tmp=directory, probe_type=probe_type) modules = [ 'kilosort_postprocessing' ] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' '))
local_probe_directory = os.path.join(local_data_path, session_id) destination_directory = os.path.join(dest_path, os.path.basename(remote_probe_directory), session_id) if not os.path.exists(local_probe_directory): os.mkdir(local_probe_directory) continuous_file = glob.glob(os.path.join(remote_probe_directory, 'continuous','Neuropix-*-100.0','*.dat'))[0] if os.path.exists(continuous_file): input_json = os.path.join(json_directory, session_id + '-input.json') output_json = os.path.join(json_directory, session_id + '-output.json') info = createInputJson(input_json, continuous_file=continuous_file, extracted_data_directory=remote_probe_directory, kilosort_output_directory=local_probe_directory) modules = [ #'extract_from_npx', #'depth_estimation', ##'median_subtraction', 'kilosort_helper', 'kilosort_postprocessing', 'noise_templates', 'mean_waveforms', 'quality_metrics'] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json
session_id = os.path.basename(local_directory) extracted_data_directory = local_directory + '_sorted' target_directory = os.path.join(r'\\sd5', 'sd5.3', session_id[:-7], session_id) print(remote_directory) print(local_directory) print(extracted_data_directory) print(target_directory) input_json = os.path.join(json_directory, session_id + '-input.json') output_json = os.path.join(json_directory, session_id + '-output.json') info = createInputJson(input_json, npx_directory=local_directory, extracted_data_directory=extracted_data_directory) modules = ['extract_from_npx', 'depth_estimation', 'median_subtraction'] for module in modules: command = "python -W ignore -m ecephys_spike_sorting.modules." + module + " --input_json " + input_json \ + " --output_json " + output_json subprocess.check_call(command.split(' ')) copy_to_remote(extracted_data_directory, target_directory) shutil.rmtree(local_directory) shutil.rmtree(extracted_data_directory)
pkl_file = glob.glob(mouse_directory + '/*.stim.pkl')[0] session_id = os.path.basename(pkl_file).split('.')[0] print(session_id) print(mouse_directory) for module in modules: input_json = os.path.join( json_directory, session_id + '-' + module + '-input.json') output_json = os.path.join( json_directory, session_id + '-' + module + '-output.json') info, last_unit_id = createInputJson(mouse_directory, probe_data_directory, module, input_json, last_unit_id) if not os.path.exists(info['output_path']): print('Running ' + module) command_string = [ "python", "-W", "ignore", "-m", module, "--input_json", input_json, "--output_json", output_json ] subprocess.check_call(command_string) #except: # print('Error processing')