def __init__(
self,
data_path: str,
animal_name: str,
dates: list,
nwb_metadata: MetadataManager,
output_path: str = '',
video_path: str = '',
extract_analog: bool = True,
extract_spikes: bool = False,
extract_lfps: bool = False,
extract_dio: bool = True,
extract_mda: bool = True,
overwrite: bool = True,
lfp_export_args: tuple = _DEFAULT_LFP_EXPORT_ARGS,
mda_export_args: tuple = _DEFAULT_MDA_EXPORT_ARGS,
analog_export_args: tuple = _DEFAULT_ANALOG_EXPORT_ARGS,
dio_export_args: tuple = _DEFAULT_DIO_EXPORT_ARGS,
time_export_args: tuple = _DEFAULT_TIME_EXPORT_ARGS,
spikes_export_args: tuple = _DEFAULT_SPIKE_EXPORT_ARGS,
parallel_instances: int = 4,
trodes_rec_export_args: tuple = _DEFAULT_TRODES_REC_EXPORT_ARGS):
validation_registrator = ValidationRegistrator()
validation_registrator.register(NotEmptyValidator(data_path))
validation_registrator.register(NotEmptyValidator(animal_name))
validation_registrator.register(NotEmptyValidator(dates))
validation_registrator.validate()
self.extract_analog = extract_analog
self.extract_spikes = extract_spikes
self.extract_dio = extract_dio
self.extract_lfps = extract_lfps
self.extract_mda = extract_mda
self.lfp_export_args = lfp_export_args + trodes_rec_export_args
self.mda_export_args = mda_export_args + trodes_rec_export_args
self.analog_export_args = analog_export_args + trodes_rec_export_args
self.dio_export_args = dio_export_args + trodes_rec_export_args
self.time_export_args = time_export_args + trodes_rec_export_args
self.spikes_export_args = spikes_export_args + trodes_rec_export_args
self.overwrite = overwrite
self.animal_name = animal_name
self.data_path = data_path
self.dates = dates
self.metadata = nwb_metadata.metadata
self.output_path = output_path
self.video_path = video_path
self.probes = nwb_metadata.probes
self.nwb_metadata = nwb_metadata
self.parallel_instances = parallel_instances
self.trodes_rec_export_args = trodes_rec_export_args
def test_registers_only_validators_correctly(self):
validationRegistrator = ValidationRegistrator()
validationRegistrator.register(
NTrodeValidator(dict(), Mock(spec=Header), list()))
validationRegistrator.register(None)
validationRegistrator.register(Dataset(name="name"))
self.assertEqual(1, len(validationRegistrator.validators))
def test_registers_validators_correctly(self):
validationRegistrator = ValidationRegistrator()
validationRegistrator.register(
NTrodeValidator(dict(), Mock(spec=Header), list()))
validationRegistrator.register(
NTrodeValidator(dict(), Mock(spec=Header), list()))
self.assertEqual(2, len(validationRegistrator.validators))
def __init__(self, metadata):
if 'associated_files' in metadata:
validation_registrator = ValidationRegistrator()
validation_registrator.register(
AssociatedFilesValidator(metadata['associated_files']))
validation_registrator.validate()
self.fl_associated_files_manager = FlAssociatedFilesManager(
metadata['associated_files'])
self.associated_files_creator = AssociatedFilesCreator()
self.associated_files_injector = AssociatedFilesInjector()
def __init__(self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = ''):
logger.info('NWBFileBuilder initialization')
logger.info('NWB builder initialization parameters: \n' +
'data_path = ' + str(data_path) + '\n' + 'animal_name = ' +
str(animal_name) + '\n' + 'date = ' + str(date) + '\n' +
'nwb_metadata = ' + str(nwb_metadata) + '\n' +
'process_dio = ' + str(process_dio) + '\n' +
'process_mda = ' + str(process_mda) + '\n' +
'process_analog = ' + str(process_analog) + '\n' +
'output_file = ' + str(output_file) + '\n')
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if associated_files_existance_validator.files_exist():
pass
else:
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist"
)
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
self.output_file = output_file
self.video_path = video_path
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {
'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog
}
rec_files_list = RecFileFinder().find_rec_files(
path=(self.data_path + '/' + self.animal_name + '/raw/' +
self.date))
header_file = HeaderProcessor.process_headers(rec_files_list)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = data_path + '/' + animal_name + '/preprocessing/' + date
validation_registrator = ValidationRegistrator()
validation_registrator.register(
NTrodeValidator(self.metadata, self.header, self.probes))
validation_registrator.register(
PreprocessingValidator(full_data_path, self.dataset_names,
data_types_for_scanning))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
self.session_time_extractor = SessionTimeExtractor(
self.datasets, self.animal_name, self.date, self.dataset_names)
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes, self.metadata, self.header)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(self.device_factory,
self.device_injector,
self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/")
self.video_files_originator = VideoFilesOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/",
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data acq device'])
if self.process_mda:
self.mda_originator = MdaOriginator(self.datasets, self.header,
self.metadata)
if self.process_dio:
self.dio_originator = DioOriginator(self.metadata, self.datasets)
if self.process_analog:
self.analog_originator = AnalogOriginator(self.datasets,
self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata, self.dataset_names,
self.process_pos_timestamps)
def __validate(metadata_path, probes_paths):
validation_registrator = ValidationRegistrator()
validation_registrator.register(
MetadataValidator(metadata_path, probes_paths))
validation_registrator.validate()
def __validate_xml_header(cls, xml_header_path):
validation_registrator = ValidationRegistrator()
validation_registrator.register(XmlFilesValidator(xml_header_path))
validation_registrator.validate()
def __init__(
self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
preprocessing_path: str = '',
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = '',
is_old_dataset: bool = False,
session_start_time=None,
):
logger.info('NWBFileBuilder initialization')
logger.info(
'NWB builder initialization parameters: \n'
+ 'data_path = ' + str(data_path) + '\n'
+ 'animal_name = ' + str(animal_name) + '\n'
+ 'date = ' + str(date) + '\n'
+ 'nwb_metadata = ' + str(nwb_metadata) + '\n'
+ 'process_dio = ' + str(process_dio) + '\n'
+ 'process_mda = ' + str(process_mda) + '\n'
+ 'process_analog = ' + str(process_analog) + '\n'
+ 'output_file = ' + str(output_file) + '\n'
)
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if not associated_files_existance_validator.files_exist():
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist")
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
if not preprocessing_path:
self.preprocessing_path = data_path
else:
self.preprocessing_path = preprocessing_path
self.output_file = output_file
self.video_path = video_path
self.is_old_dataset = is_old_dataset
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog}
rec_files_list = RecFileFinder().find_rec_files(
path=os.path.join(
self.data_path, self.animal_name, 'raw', self.date)
)
if not preprocessing_path:
header_path = None # default
else:
header_path = os.path.join(
self.preprocessing_path, self.animal_name, 'headers',
self.date)
os.makedirs(header_path, exist_ok=True)
header_file = HeaderProcessor.process_headers(
rec_files_list, copy_dir=header_path)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(
self.preprocessing_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = os.path.join(self.preprocessing_path,
self.animal_name, 'preprocessing', date)
validation_registrator = ValidationRegistrator()
validation_registrator.register(NTrodeValidator(
self.metadata, self.header, self.probes))
validation_registrator.register(PreprocessingValidator(
full_data_path,
self.dataset_names,
data_types_for_scanning
))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
if self.is_old_dataset:
if not session_start_time:
raise ValueError(
'session_start_time is required for old dataset.')
self.session_start_time = session_start_time
else:
session_time_extractor = SessionTimeExtractor(
self.datasets,
self.animal_name,
self.date,
self.dataset_names
)
self.session_start_time = session_time_extractor.get_session_start_time()
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes,
self.metadata,
self.header
)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(
self.device_factory, self.device_injector, self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
os.path.join(self.data_path, self.animal_name, 'raw', self.date))
if self.is_old_dataset:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS,
)
else:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data_acq_device']
)
if self.process_mda:
self.mda_originator = MdaOriginator(
self.datasets, self.header, self.metadata)
if self.process_dio:
if self.is_old_dataset:
self.dio_originator = DioOriginator(
self.metadata, self.datasets,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS)
else:
self.dio_originator = DioOriginator(
self.metadata, self.datasets)
if self.process_analog:
if self.is_old_dataset:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS)
else:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata,
self.dataset_names)
def __init__(
self,
data_path: str,
animal_name: str,
dates: list,
nwb_metadata: MetadataManager,
output_path: str = '',
video_path: str = '',
preprocessing_path: str = '',
extract_analog: bool = True,
extract_spikes: bool = False,
extract_lfps: bool = False,
extract_dio: bool = True,
extract_mda: bool = True,
overwrite: bool = True,
lfp_export_args: tuple = None,
mda_export_args: tuple = None,
analog_export_args: tuple = None,
dio_export_args: tuple = None,
time_export_args: tuple = None,
spikes_export_args: tuple = None,
parallel_instances: int = 4,
trodes_rec_export_args: tuple = None,
):
validation_registrator = ValidationRegistrator()
validation_registrator.register(NotEmptyValidator(data_path))
validation_registrator.register(NotEmptyValidator(animal_name))
validation_registrator.register(NotEmptyValidator(dates))
validation_registrator.validate()
trodes_version = get_trodes_version_from_path()[0]
if lfp_export_args is None:
if trodes_version < 2.0:
lfp_export_args = ('-highpass', '0', '-lowpass', '400',
'-interp', '0', '-userefs', '0',
'-outputrate', '1500')
else:
lfp_export_args = ('-lfphighpass', '0', '-lfplowpass', '400',
'-interp', '0', '-uselfprefs', '0',
'-outputrate', '1500', '-sortingmode', '1')
if mda_export_args is None:
if trodes_version < 2.0:
mda_export_args = ('-usespikefilters', '0', '-interp', '0',
'-userefs', '0')
else:
mda_export_args = ('-usespikefilters', '0', '-interp', '1',
'-userawrefs', '0', '-usespikerefs', '0',
'-sortingmode', '1')
if analog_export_args is None:
analog_export_args = ()
if dio_export_args is None:
dio_export_args = ()
if spikes_export_args is None:
spikes_export_args = ()
if time_export_args is None:
time_export_args = ()
if trodes_rec_export_args is None:
trodes_rec_export_args = ()
self.extract_analog = extract_analog
self.extract_spikes = extract_spikes
self.extract_dio = extract_dio
self.extract_lfps = extract_lfps
self.extract_mda = extract_mda
self.lfp_export_args = lfp_export_args + trodes_rec_export_args
self.mda_export_args = mda_export_args + trodes_rec_export_args
self.analog_export_args = analog_export_args + trodes_rec_export_args
self.dio_export_args = dio_export_args + trodes_rec_export_args
self.time_export_args = time_export_args + trodes_rec_export_args
self.spikes_export_args = spikes_export_args + trodes_rec_export_args
self.overwrite = overwrite
self.animal_name = animal_name
self.data_path = data_path
self.dates = dates
self.metadata = nwb_metadata.metadata
self.output_path = output_path
self.video_path = video_path
if not preprocessing_path:
self.preprocessing_path = data_path
else:
self.preprocessing_path = preprocessing_path
self.probes = nwb_metadata.probes
self.nwb_metadata = nwb_metadata
self.parallel_instances = parallel_instances
self.trodes_rec_export_args = trodes_rec_export_args
self.is_old_dataset = self.__is_old_dataset()