def __init__(self, header, metadata):
header_path = str(path) + '/../../../data/' + \
metadata['default_header_file_path']
default_header = Header(header_path)
self.fl_header_device_manager = FlHeaderDeviceManager(
'header_device', header.configuration.global_configuration,
default_header.configuration.global_configuration)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
class CameraDeviceOriginator:
def __init__(self, metadata):
self.fl_camera_device_manager = FlCameraDeviceManager(metadata)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
def make(self, nwb_content):
logger.info('CameraDevice: Building')
fl_camera_devices = self.fl_camera_device_manager.get_fl_device_manager()
logger.info('CameraDevice: Creating')
camera_devices = [self.device_factory.create_camera_device(single_camera_device)
for single_camera_device in fl_camera_devices]
logger.info('CameraDevice: Injecting into NWB')
self.device_injector.inject_all_devices(nwb_content, camera_devices)
class HeaderDeviceOriginator:
def __init__(self, header, metadata):
header_path = str(path) + '/../../../data/' + \
metadata['default_header_file_path']
default_header = Header(header_path)
self.fl_header_device_manager = FlHeaderDeviceManager(
'header_device', header.configuration.global_configuration,
default_header.configuration.global_configuration)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
def make(self, nwb_content):
logger.info('HeaderDevice: Building')
fl_header_device = self.fl_header_device_manager.get_fl_header_device()
logger.info('HeaderDevice: Creating')
header_device = self.device_factory.create_header_device(
fl_header_device)
logger.info('HeaderDevice: Injecting into NWB')
self.device_injector.inject_all_devices(nwb_content, [header_device])
def __init__(self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = ''):
logger.info('NWBFileBuilder initialization')
logger.info('NWB builder initialization parameters: \n' +
'data_path = ' + str(data_path) + '\n' + 'animal_name = ' +
str(animal_name) + '\n' + 'date = ' + str(date) + '\n' +
'nwb_metadata = ' + str(nwb_metadata) + '\n' +
'process_dio = ' + str(process_dio) + '\n' +
'process_mda = ' + str(process_mda) + '\n' +
'process_analog = ' + str(process_analog) + '\n' +
'output_file = ' + str(output_file) + '\n')
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if associated_files_existance_validator.files_exist():
pass
else:
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist"
)
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
self.output_file = output_file
self.video_path = video_path
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {
'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog
}
rec_files_list = RecFileFinder().find_rec_files(
path=(self.data_path + '/' + self.animal_name + '/raw/' +
self.date))
header_file = HeaderProcessor.process_headers(rec_files_list)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = data_path + '/' + animal_name + '/preprocessing/' + date
validation_registrator = ValidationRegistrator()
validation_registrator.register(
NTrodeValidator(self.metadata, self.header, self.probes))
validation_registrator.register(
PreprocessingValidator(full_data_path, self.dataset_names,
data_types_for_scanning))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
self.session_time_extractor = SessionTimeExtractor(
self.datasets, self.animal_name, self.date, self.dataset_names)
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes, self.metadata, self.header)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(self.device_factory,
self.device_injector,
self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/")
self.video_files_originator = VideoFilesOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/",
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data acq device'])
if self.process_mda:
self.mda_originator = MdaOriginator(self.datasets, self.header,
self.metadata)
if self.process_dio:
self.dio_originator = DioOriginator(self.metadata, self.datasets)
if self.process_analog:
self.analog_originator = AnalogOriginator(self.datasets,
self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata, self.dataset_names,
self.process_pos_timestamps)
def __init__(self, metadata):
self.fl_camera_device_manager = FlCameraDeviceManager(metadata)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
def setUpClass(cls):
cls.header_device_1 = Mock(spec=HeaderDevice)
cls.header_device_1.name = 'HeaderDevice_1'
cls.header_device_1.headstage_serial = 'Sample headstage_serial'
cls.header_device_1.headstage_smart_ref_on = 'Sample headstage_smart_ref_on'
cls.header_device_1.realtime_mode = 'Sample realtime_mode'
cls.header_device_1.headstage_auto_settle_on = 'Sample headstage_auto_settle_on'
cls.header_device_1.timestamp_at_creation = 'Sample timestamp_at_creation'
cls.header_device_1.controller_firmware_version = 'Sample controller_firmware_version'
cls.header_device_1.controller_serial = 'Sample controller_serial'
cls.header_device_1.save_displayed_chan_only = 'Sample save_displayed_chan_only'
cls.header_device_1.headstage_firmware_version = 'Sample headstage_firmware_version'
cls.header_device_1.qt_version = 'Sample qt_version'
cls.header_device_1.compile_date = 'Sample compile_date'
cls.header_device_1.compile_time = 'Sample compile_time'
cls.header_device_1.file_prefix = 'Sample file_prefix'
cls.header_device_1.headstage_gyro_sensor_on = 'Sample headstage_gyro_sensor_on'
cls.header_device_1.headstage_mag_sensor_on = 'Sample headstage_mag_sensor_on'
cls.header_device_1.trodes_version = 'Sample trodes_version'
cls.header_device_1.headstage_accel_sensor_on = 'Sample headstage_accel_sensor_on'
cls.header_device_1.commit_head = 'Sample commit_head'
cls.header_device_1.system_time_at_creation = 'Sample system_time_at_creation'
cls.header_device_1.file_path = 'Sample file_path'
cls.probe_1 = Mock(spec=Probe)
cls.probe_1.name = 'Probe_1'
cls.probe_1.probe_type = 'type_1'
cls.probe_1.id = 1
cls.probe_1.contact_size = 20.0
cls.probe_1.num_shanks = 20
cls.probe_2 = Mock(spec=Probe)
cls.probe_2.name = 'Probe_2'
cls.probe_2.probe_type = 'type_2'
cls.probe_2.id = 2
cls.probe_2.contact_size = 25.0
cls.probe_2.num_shanks = 30
cls.data_acq_device_0 = Mock(spec=DataAcqDevice)
cls.data_acq_device_0.name = 'Acq_0'
cls.data_acq_device_0.system = 'system_0'
cls.data_acq_device_0.amplifier = 'amplifier_0'
cls.data_acq_device_0.adc_circuit = 'adc_circuit_0'
cls.data_acq_device_1 = Mock(spec=DataAcqDevice)
cls.data_acq_device_1.name = 'Acq_1'
cls.data_acq_device_1.system = 'system_1'
cls.data_acq_device_1.amplifier = 'amplifier_1'
cls.data_acq_device_1.adc_circuit = 'adc_circuit_1'
cls.device_1 = Mock(spec=Device)
cls.device_1.name = 'Device_1'
cls.device_2 = Mock(spec=Device)
cls.device_2.name = 'Device_2'
cls.probes_dict = {'Probe_1': cls.probe_1, 'Probe_2': cls.probe_2}
cls.data_acq_device_dict = {
'Acq_0': cls.data_acq_device_0,
'Acq_1': cls.data_acq_device_1
}
cls.header_device_dict = {'HeaderDevice_1': cls.header_device_1}
cls.device_dict = {'Device_1': cls.device_1, 'Device_2': cls.device_2}
cls.device_injector = DeviceInjector()
def __init__(
self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
preprocessing_path: str = '',
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = '',
is_old_dataset: bool = False,
session_start_time=None,
):
logger.info('NWBFileBuilder initialization')
logger.info(
'NWB builder initialization parameters: \n'
+ 'data_path = ' + str(data_path) + '\n'
+ 'animal_name = ' + str(animal_name) + '\n'
+ 'date = ' + str(date) + '\n'
+ 'nwb_metadata = ' + str(nwb_metadata) + '\n'
+ 'process_dio = ' + str(process_dio) + '\n'
+ 'process_mda = ' + str(process_mda) + '\n'
+ 'process_analog = ' + str(process_analog) + '\n'
+ 'output_file = ' + str(output_file) + '\n'
)
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if not associated_files_existance_validator.files_exist():
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist")
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
if not preprocessing_path:
self.preprocessing_path = data_path
else:
self.preprocessing_path = preprocessing_path
self.output_file = output_file
self.video_path = video_path
self.is_old_dataset = is_old_dataset
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog}
rec_files_list = RecFileFinder().find_rec_files(
path=os.path.join(
self.data_path, self.animal_name, 'raw', self.date)
)
if not preprocessing_path:
header_path = None # default
else:
header_path = os.path.join(
self.preprocessing_path, self.animal_name, 'headers',
self.date)
os.makedirs(header_path, exist_ok=True)
header_file = HeaderProcessor.process_headers(
rec_files_list, copy_dir=header_path)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(
self.preprocessing_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = os.path.join(self.preprocessing_path,
self.animal_name, 'preprocessing', date)
validation_registrator = ValidationRegistrator()
validation_registrator.register(NTrodeValidator(
self.metadata, self.header, self.probes))
validation_registrator.register(PreprocessingValidator(
full_data_path,
self.dataset_names,
data_types_for_scanning
))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
if self.is_old_dataset:
if not session_start_time:
raise ValueError(
'session_start_time is required for old dataset.')
self.session_start_time = session_start_time
else:
session_time_extractor = SessionTimeExtractor(
self.datasets,
self.animal_name,
self.date,
self.dataset_names
)
self.session_start_time = session_time_extractor.get_session_start_time()
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes,
self.metadata,
self.header
)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(
self.device_factory, self.device_injector, self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
os.path.join(self.data_path, self.animal_name, 'raw', self.date))
if self.is_old_dataset:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS,
)
else:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data_acq_device']
)
if self.process_mda:
self.mda_originator = MdaOriginator(
self.datasets, self.header, self.metadata)
if self.process_dio:
if self.is_old_dataset:
self.dio_originator = DioOriginator(
self.metadata, self.datasets,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS)
else:
self.dio_originator = DioOriginator(
self.metadata, self.datasets)
if self.process_analog:
if self.is_old_dataset:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS)
else:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata,
self.dataset_names)