def __init__( self, data_path: str, animal_name: str, dates: list, nwb_metadata: MetadataManager, output_path: str = '', video_path: str = '', extract_analog: bool = True, extract_spikes: bool = False, extract_lfps: bool = False, extract_dio: bool = True, extract_mda: bool = True, overwrite: bool = True, lfp_export_args: tuple = _DEFAULT_LFP_EXPORT_ARGS, mda_export_args: tuple = _DEFAULT_MDA_EXPORT_ARGS, analog_export_args: tuple = _DEFAULT_ANALOG_EXPORT_ARGS, dio_export_args: tuple = _DEFAULT_DIO_EXPORT_ARGS, time_export_args: tuple = _DEFAULT_TIME_EXPORT_ARGS, spikes_export_args: tuple = _DEFAULT_SPIKE_EXPORT_ARGS, parallel_instances: int = 4, trodes_rec_export_args: tuple = _DEFAULT_TRODES_REC_EXPORT_ARGS): validation_registrator = ValidationRegistrator() validation_registrator.register(NotEmptyValidator(data_path)) validation_registrator.register(NotEmptyValidator(animal_name)) validation_registrator.register(NotEmptyValidator(dates)) validation_registrator.validate() self.extract_analog = extract_analog self.extract_spikes = extract_spikes self.extract_dio = extract_dio self.extract_lfps = extract_lfps self.extract_mda = extract_mda self.lfp_export_args = lfp_export_args + trodes_rec_export_args self.mda_export_args = mda_export_args + trodes_rec_export_args self.analog_export_args = analog_export_args + trodes_rec_export_args self.dio_export_args = dio_export_args + trodes_rec_export_args self.time_export_args = time_export_args + trodes_rec_export_args self.spikes_export_args = spikes_export_args + trodes_rec_export_args self.overwrite = overwrite self.animal_name = animal_name self.data_path = data_path self.dates = dates self.metadata = nwb_metadata.metadata self.output_path = output_path self.video_path = video_path self.probes = nwb_metadata.probes self.nwb_metadata = nwb_metadata self.parallel_instances = parallel_instances self.trodes_rec_export_args = trodes_rec_export_args
def test_registers_only_validators_correctly(self): validationRegistrator = ValidationRegistrator() validationRegistrator.register( NTrodeValidator(dict(), Mock(spec=Header), list())) validationRegistrator.register(None) validationRegistrator.register(Dataset(name="name")) self.assertEqual(1, len(validationRegistrator.validators))
def test_registers_validators_correctly(self): validationRegistrator = ValidationRegistrator() validationRegistrator.register( NTrodeValidator(dict(), Mock(spec=Header), list())) validationRegistrator.register( NTrodeValidator(dict(), Mock(spec=Header), list())) self.assertEqual(2, len(validationRegistrator.validators))
def __init__(self, metadata): if 'associated_files' in metadata: validation_registrator = ValidationRegistrator() validation_registrator.register( AssociatedFilesValidator(metadata['associated_files'])) validation_registrator.validate() self.fl_associated_files_manager = FlAssociatedFilesManager( metadata['associated_files']) self.associated_files_creator = AssociatedFilesCreator() self.associated_files_injector = AssociatedFilesInjector()
def __init__(self, data_path: str, animal_name: str, date: str, nwb_metadata: MetadataManager, process_dio: bool = True, process_mda: bool = True, process_analog: bool = True, process_pos_timestamps: bool = True, video_path: str = '', output_file: str = 'output.nwb', reconfig_header: str = ''): logger.info('NWBFileBuilder initialization') logger.info('NWB builder initialization parameters: \n' + 'data_path = ' + str(data_path) + '\n' + 'animal_name = ' + str(animal_name) + '\n' + 'date = ' + str(date) + '\n' + 'nwb_metadata = ' + str(nwb_metadata) + '\n' + 'process_dio = ' + str(process_dio) + '\n' + 'process_mda = ' + str(process_mda) + '\n' + 'process_analog = ' + str(process_analog) + '\n' + 'output_file = ' + str(output_file) + '\n') validation_registrator = ValidationRegistrator() validation_registrator.register(PathValidator(data_path)) validation_registrator.validate() self.animal_name = animal_name self.date = date self.data_path = data_path self.metadata = nwb_metadata.metadata metadata_section_validator = MetadataSectionValidator(self.metadata) metadata_section_validator.validate_sections() if self.metadata.get('associated_files', []): associated_files_existance_validator = AssociatedFilesExistanceValidator( self.metadata['associated_files']) if associated_files_existance_validator.files_exist(): pass else: raise Exception( "one or more associated file listed in metadata.yaml file does not exist" ) self.probes = nwb_metadata.probes self.process_dio = process_dio self.process_mda = process_mda self.process_analog = process_analog self.process_pos_timestamps = process_pos_timestamps self.output_file = output_file self.video_path = video_path self.link_to_notes = self.metadata.get('link to notes', None) data_types_for_scanning = { 'pos': True, 'time': True, 'mda': process_mda, 'DIO': process_dio, 'analog': process_analog } rec_files_list = RecFileFinder().find_rec_files( path=(self.data_path + '/' + self.animal_name + '/raw/' + self.date)) header_file = HeaderProcessor.process_headers(rec_files_list) if reconfig_header: self.header = Header(reconfig_header) else: self.header = Header(header_file) self.data_scanner = DataScanner(data_path, animal_name, nwb_metadata) self.dataset_names = self.data_scanner.get_all_epochs(date) full_data_path = data_path + '/' + animal_name + '/preprocessing/' + date validation_registrator = ValidationRegistrator() validation_registrator.register( NTrodeValidator(self.metadata, self.header, self.probes)) validation_registrator.register( PreprocessingValidator(full_data_path, self.dataset_names, data_types_for_scanning)) validation_registrator.register(TaskValidator(self.metadata['tasks'])) validation_registrator.validate() self.__extract_datasets(animal_name, date) self.corrupted_data_manager = CorruptedDataManager(self.metadata) self.shanks_electrode_originator = ShanksElectrodeOriginator( self.probes, self.metadata) self.shanks_originator = ShanksOriginator(self.probes, self.metadata) self.fl_probe_manager = FlProbeManager(self.probes) self.device_injector = DeviceInjector() self.device_factory = DeviceFactory() self.electrode_group_originator = ElectrodeGroupOriginator( self.metadata) self.electrodes_originator = ElectrodesOriginator( self.probes, self.metadata) self.session_time_extractor = SessionTimeExtractor( self.datasets, self.animal_name, self.date, self.dataset_names) self.mda_valid_time_originator = MdaValidTimeOriginator( self.header, self.metadata) self.mda_invalid_time_originator = MdaInvalidTimeOriginator( self.header, self.metadata) self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata) self.pos_invalid_time_originator = PosInvalidTimeOriginator( self.metadata) self.epochs_originator = EpochsOriginator(self.datasets) if 'associated_files' in self.metadata: self.associated_files_originator = AssociatedFilesOriginator( self.metadata) self.electrodes_extension_originator = ElectrodesExtensionOriginator( self.probes, self.metadata, self.header) self.sample_count_timestamp_corespondence_originator =\ SampleCountTimestampCorespondenceOriginator(self.datasets) self.processing_module_originator = ProcessingModuleOriginator() self.task_originator = TaskOriginator(self.metadata) self.camera_device_originator = CameraDeviceOriginator(self.metadata) self.header_device_originator = HeaderDeviceOriginator( self.header, self.metadata) self.probes_originator = ProbeOriginator(self.device_factory, self.device_injector, self.probes) self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator( self.data_path + "/" + animal_name + "/raw/" + self.date + "/") self.video_files_originator = VideoFilesOriginator( self.data_path + "/" + animal_name + "/raw/" + self.date + "/", self.video_path, self.metadata["associated_video_files"], ) self.data_acq_device_originator = DataAcqDeviceOriginator( device_factory=self.device_factory, device_injector=self.device_injector, metadata=self.metadata['data acq device']) if self.process_mda: self.mda_originator = MdaOriginator(self.datasets, self.header, self.metadata) if self.process_dio: self.dio_originator = DioOriginator(self.metadata, self.datasets) if self.process_analog: self.analog_originator = AnalogOriginator(self.datasets, self.metadata) self.position_originator = PositionOriginator( self.datasets, self.metadata, self.dataset_names, self.process_pos_timestamps)
def __validate(metadata_path, probes_paths): validation_registrator = ValidationRegistrator() validation_registrator.register( MetadataValidator(metadata_path, probes_paths)) validation_registrator.validate()
def __validate_xml_header(cls, xml_header_path): validation_registrator = ValidationRegistrator() validation_registrator.register(XmlFilesValidator(xml_header_path)) validation_registrator.validate()
def __init__( self, data_path: str, animal_name: str, date: str, nwb_metadata: MetadataManager, process_dio: bool = True, process_mda: bool = True, process_analog: bool = True, process_pos_timestamps: bool = True, preprocessing_path: str = '', video_path: str = '', output_file: str = 'output.nwb', reconfig_header: str = '', is_old_dataset: bool = False, session_start_time=None, ): logger.info('NWBFileBuilder initialization') logger.info( 'NWB builder initialization parameters: \n' + 'data_path = ' + str(data_path) + '\n' + 'animal_name = ' + str(animal_name) + '\n' + 'date = ' + str(date) + '\n' + 'nwb_metadata = ' + str(nwb_metadata) + '\n' + 'process_dio = ' + str(process_dio) + '\n' + 'process_mda = ' + str(process_mda) + '\n' + 'process_analog = ' + str(process_analog) + '\n' + 'output_file = ' + str(output_file) + '\n' ) validation_registrator = ValidationRegistrator() validation_registrator.register(PathValidator(data_path)) validation_registrator.validate() self.animal_name = animal_name self.date = date self.data_path = data_path self.metadata = nwb_metadata.metadata metadata_section_validator = MetadataSectionValidator(self.metadata) metadata_section_validator.validate_sections() if self.metadata.get('associated_files', []): associated_files_existance_validator = AssociatedFilesExistanceValidator( self.metadata['associated_files']) if not associated_files_existance_validator.files_exist(): raise Exception( "one or more associated file listed in metadata.yaml file does not exist") self.probes = nwb_metadata.probes self.process_dio = process_dio self.process_mda = process_mda self.process_analog = process_analog self.process_pos_timestamps = process_pos_timestamps if not preprocessing_path: self.preprocessing_path = data_path else: self.preprocessing_path = preprocessing_path self.output_file = output_file self.video_path = video_path self.is_old_dataset = is_old_dataset self.link_to_notes = self.metadata.get('link to notes', None) data_types_for_scanning = {'pos': True, 'time': True, 'mda': process_mda, 'DIO': process_dio, 'analog': process_analog} rec_files_list = RecFileFinder().find_rec_files( path=os.path.join( self.data_path, self.animal_name, 'raw', self.date) ) if not preprocessing_path: header_path = None # default else: header_path = os.path.join( self.preprocessing_path, self.animal_name, 'headers', self.date) os.makedirs(header_path, exist_ok=True) header_file = HeaderProcessor.process_headers( rec_files_list, copy_dir=header_path) if reconfig_header: self.header = Header(reconfig_header) else: self.header = Header(header_file) self.data_scanner = DataScanner( self.preprocessing_path, animal_name, nwb_metadata) self.dataset_names = self.data_scanner.get_all_epochs(date) full_data_path = os.path.join(self.preprocessing_path, self.animal_name, 'preprocessing', date) validation_registrator = ValidationRegistrator() validation_registrator.register(NTrodeValidator( self.metadata, self.header, self.probes)) validation_registrator.register(PreprocessingValidator( full_data_path, self.dataset_names, data_types_for_scanning )) validation_registrator.register(TaskValidator(self.metadata['tasks'])) validation_registrator.validate() self.__extract_datasets(animal_name, date) self.corrupted_data_manager = CorruptedDataManager(self.metadata) self.shanks_electrode_originator = ShanksElectrodeOriginator( self.probes, self.metadata) self.shanks_originator = ShanksOriginator(self.probes, self.metadata) self.fl_probe_manager = FlProbeManager(self.probes) self.device_injector = DeviceInjector() self.device_factory = DeviceFactory() self.electrode_group_originator = ElectrodeGroupOriginator( self.metadata) self.electrodes_originator = ElectrodesOriginator( self.probes, self.metadata) if self.is_old_dataset: if not session_start_time: raise ValueError( 'session_start_time is required for old dataset.') self.session_start_time = session_start_time else: session_time_extractor = SessionTimeExtractor( self.datasets, self.animal_name, self.date, self.dataset_names ) self.session_start_time = session_time_extractor.get_session_start_time() self.mda_valid_time_originator = MdaValidTimeOriginator( self.header, self.metadata) self.mda_invalid_time_originator = MdaInvalidTimeOriginator( self.header, self.metadata) self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata) self.pos_invalid_time_originator = PosInvalidTimeOriginator( self.metadata) self.epochs_originator = EpochsOriginator(self.datasets) if 'associated_files' in self.metadata: self.associated_files_originator = AssociatedFilesOriginator( self.metadata) self.electrodes_extension_originator = ElectrodesExtensionOriginator( self.probes, self.metadata, self.header ) self.sample_count_timestamp_corespondence_originator =\ SampleCountTimestampCorespondenceOriginator(self.datasets) self.processing_module_originator = ProcessingModuleOriginator() self.task_originator = TaskOriginator(self.metadata) self.camera_device_originator = CameraDeviceOriginator(self.metadata) self.header_device_originator = HeaderDeviceOriginator( self.header, self.metadata) self.probes_originator = ProbeOriginator( self.device_factory, self.device_injector, self.probes) self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator( os.path.join(self.data_path, self.animal_name, 'raw', self.date)) if self.is_old_dataset: self.video_files_originator = VideoFilesOriginator( os.path.join(self.data_path, self.animal_name, 'raw', self.date), self.video_path, self.metadata["associated_video_files"], convert_timestamps=_CONVERT_OLD_TIMESTAMPS, return_timestamps=_RETURN_OLD_TIMESTAMPS, ) else: self.video_files_originator = VideoFilesOriginator( os.path.join(self.data_path, self.animal_name, 'raw', self.date), self.video_path, self.metadata["associated_video_files"], ) self.data_acq_device_originator = DataAcqDeviceOriginator( device_factory=self.device_factory, device_injector=self.device_injector, metadata=self.metadata['data_acq_device'] ) if self.process_mda: self.mda_originator = MdaOriginator( self.datasets, self.header, self.metadata) if self.process_dio: if self.is_old_dataset: self.dio_originator = DioOriginator( self.metadata, self.datasets, convert_timestamps=_CONVERT_OLD_TIMESTAMPS) else: self.dio_originator = DioOriginator( self.metadata, self.datasets) if self.process_analog: if self.is_old_dataset: self.analog_originator = AnalogOriginator( self.datasets, self.metadata, convert_timestamps=_CONVERT_OLD_TIMESTAMPS, return_timestamps=_RETURN_OLD_TIMESTAMPS) else: self.analog_originator = AnalogOriginator( self.datasets, self.metadata) self.position_originator = PositionOriginator( self.datasets, self.metadata, self.dataset_names)
def __init__( self, data_path: str, animal_name: str, dates: list, nwb_metadata: MetadataManager, output_path: str = '', video_path: str = '', preprocessing_path: str = '', extract_analog: bool = True, extract_spikes: bool = False, extract_lfps: bool = False, extract_dio: bool = True, extract_mda: bool = True, overwrite: bool = True, lfp_export_args: tuple = None, mda_export_args: tuple = None, analog_export_args: tuple = None, dio_export_args: tuple = None, time_export_args: tuple = None, spikes_export_args: tuple = None, parallel_instances: int = 4, trodes_rec_export_args: tuple = None, ): validation_registrator = ValidationRegistrator() validation_registrator.register(NotEmptyValidator(data_path)) validation_registrator.register(NotEmptyValidator(animal_name)) validation_registrator.register(NotEmptyValidator(dates)) validation_registrator.validate() trodes_version = get_trodes_version_from_path()[0] if lfp_export_args is None: if trodes_version < 2.0: lfp_export_args = ('-highpass', '0', '-lowpass', '400', '-interp', '0', '-userefs', '0', '-outputrate', '1500') else: lfp_export_args = ('-lfphighpass', '0', '-lfplowpass', '400', '-interp', '0', '-uselfprefs', '0', '-outputrate', '1500', '-sortingmode', '1') if mda_export_args is None: if trodes_version < 2.0: mda_export_args = ('-usespikefilters', '0', '-interp', '0', '-userefs', '0') else: mda_export_args = ('-usespikefilters', '0', '-interp', '1', '-userawrefs', '0', '-usespikerefs', '0', '-sortingmode', '1') if analog_export_args is None: analog_export_args = () if dio_export_args is None: dio_export_args = () if spikes_export_args is None: spikes_export_args = () if time_export_args is None: time_export_args = () if trodes_rec_export_args is None: trodes_rec_export_args = () self.extract_analog = extract_analog self.extract_spikes = extract_spikes self.extract_dio = extract_dio self.extract_lfps = extract_lfps self.extract_mda = extract_mda self.lfp_export_args = lfp_export_args + trodes_rec_export_args self.mda_export_args = mda_export_args + trodes_rec_export_args self.analog_export_args = analog_export_args + trodes_rec_export_args self.dio_export_args = dio_export_args + trodes_rec_export_args self.time_export_args = time_export_args + trodes_rec_export_args self.spikes_export_args = spikes_export_args + trodes_rec_export_args self.overwrite = overwrite self.animal_name = animal_name self.data_path = data_path self.dates = dates self.metadata = nwb_metadata.metadata self.output_path = output_path self.video_path = video_path if not preprocessing_path: self.preprocessing_path = data_path else: self.preprocessing_path = preprocessing_path self.probes = nwb_metadata.probes self.nwb_metadata = nwb_metadata self.parallel_instances = parallel_instances self.trodes_rec_export_args = trodes_rec_export_args self.is_old_dataset = self.__is_old_dataset()