def test_manager_builds_FlProbes_successfully(self):
mock_shank_1 = Mock(spec=Shank)
mock_shank_2 = Mock(spec=Shank)
mock_shank_3 = Mock(spec=Shank)
mock_shank_4 = Mock(spec=Shank)
mock_shank_5 = Mock(spec=Shank)
mock_shank_6 = Mock(spec=Shank)
mock_shanks_dict = {
'tetrode_12.5': [mock_shank_1, mock_shank_2],
'128c-4s8mm6cm-20um-40um-sl': [mock_shank_1, mock_shank_2,
mock_shank_3, mock_shank_4,
mock_shank_5, mock_shank_6]
}
probes_valid_map = {'128c-4s8mm6cm-20um-40um-sl'}
fl_probe_manager = FlProbeManager(
probes_metadata=self.probes_metadata,
)
fl_probes = fl_probe_manager.get_fl_probes(
shanks_dict=mock_shanks_dict,
probes_valid_map=probes_valid_map
)
self.assertIsInstance(fl_probes[0], FlProbe)
self.assertIsInstance(fl_probes[0].probe_id, int)
self.assertIsInstance(fl_probes[0].name, str)
self.assertIsInstance(fl_probes[0].probe_type, str)
self.assertIsInstance(fl_probes[0].units, str)
self.assertIsInstance(fl_probes[0].probe_description, str)
self.assertIsInstance(fl_probes[0].contact_size, float)
self.assertIsInstance(fl_probes[0].shanks, list)
self.assertEqual(len(fl_probes[0].shanks), 6)
self.assertEqual(fl_probes[0].probe_id, 0)
self.assertEqual(fl_probes[0].name, 'probe 0')
self.assertEqual(fl_probes[0].probe_type, '128c-4s8mm6cm-20um-40um-sl')
self.assertEqual(fl_probes[0].units, 'mm')
self.assertEqual(fl_probes[0].probe_description, 'sample description 2')
self.assertEqual(fl_probes[0].contact_size, 20.0)
self.assertEqual(fl_probes[0].shanks, [mock_shank_1, mock_shank_2,
mock_shank_3, mock_shank_4,
mock_shank_5, mock_shank_6]
)
class ProbeOriginator:
def __init__(self, device_factory, device_injector, probes_metadata):
self.device_factory = device_factory
self.device_injector = device_injector
self.fl_probe_manager = FlProbeManager(probes_metadata)
def make(
self,
nwb_content,
shanks_dict,
probes_valid_map,
):
logger.info('Probes: Building')
fl_probes = self.fl_probe_manager.get_fl_probes(
shanks_dict, probes_valid_map)
logger.info('Probes: Creating probes')
probes = [
self.device_factory.create_probe(fl_probe)
for fl_probe in fl_probes
]
logger.info('Probes: Injecting probes into NWB')
self.device_injector.inject_all_devices(nwb_content, probes)
return probes
def __init__(self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = ''):
logger.info('NWBFileBuilder initialization')
logger.info('NWB builder initialization parameters: \n' +
'data_path = ' + str(data_path) + '\n' + 'animal_name = ' +
str(animal_name) + '\n' + 'date = ' + str(date) + '\n' +
'nwb_metadata = ' + str(nwb_metadata) + '\n' +
'process_dio = ' + str(process_dio) + '\n' +
'process_mda = ' + str(process_mda) + '\n' +
'process_analog = ' + str(process_analog) + '\n' +
'output_file = ' + str(output_file) + '\n')
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if associated_files_existance_validator.files_exist():
pass
else:
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist"
)
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
self.output_file = output_file
self.video_path = video_path
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {
'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog
}
rec_files_list = RecFileFinder().find_rec_files(
path=(self.data_path + '/' + self.animal_name + '/raw/' +
self.date))
header_file = HeaderProcessor.process_headers(rec_files_list)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = data_path + '/' + animal_name + '/preprocessing/' + date
validation_registrator = ValidationRegistrator()
validation_registrator.register(
NTrodeValidator(self.metadata, self.header, self.probes))
validation_registrator.register(
PreprocessingValidator(full_data_path, self.dataset_names,
data_types_for_scanning))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
self.session_time_extractor = SessionTimeExtractor(
self.datasets, self.animal_name, self.date, self.dataset_names)
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes, self.metadata, self.header)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(self.device_factory,
self.device_injector,
self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/")
self.video_files_originator = VideoFilesOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/",
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data acq device'])
if self.process_mda:
self.mda_originator = MdaOriginator(self.datasets, self.header,
self.metadata)
if self.process_dio:
self.dio_originator = DioOriginator(self.metadata, self.datasets)
if self.process_analog:
self.analog_originator = AnalogOriginator(self.datasets,
self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata, self.dataset_names,
self.process_pos_timestamps)
def __init__(self, device_factory, device_injector, probes_metadata):
self.device_factory = device_factory
self.device_injector = device_injector
self.fl_probe_manager = FlProbeManager(probes_metadata)
def test_manager_fails_creating_FlProbes_due_to_None_param(self):
fl_probe_manager = FlProbeManager(
probes_metadata=self.probes_metadata
)
fl_probe_manager.get_fl_probes(None, None)
def test_manager_fails_due_to_None_param(self):
FlProbeManager(
probes_metadata=None
)
def __init__(
self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
preprocessing_path: str = '',
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = '',
is_old_dataset: bool = False,
session_start_time=None,
):
logger.info('NWBFileBuilder initialization')
logger.info(
'NWB builder initialization parameters: \n'
+ 'data_path = ' + str(data_path) + '\n'
+ 'animal_name = ' + str(animal_name) + '\n'
+ 'date = ' + str(date) + '\n'
+ 'nwb_metadata = ' + str(nwb_metadata) + '\n'
+ 'process_dio = ' + str(process_dio) + '\n'
+ 'process_mda = ' + str(process_mda) + '\n'
+ 'process_analog = ' + str(process_analog) + '\n'
+ 'output_file = ' + str(output_file) + '\n'
)
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if not associated_files_existance_validator.files_exist():
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist")
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
if not preprocessing_path:
self.preprocessing_path = data_path
else:
self.preprocessing_path = preprocessing_path
self.output_file = output_file
self.video_path = video_path
self.is_old_dataset = is_old_dataset
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog}
rec_files_list = RecFileFinder().find_rec_files(
path=os.path.join(
self.data_path, self.animal_name, 'raw', self.date)
)
if not preprocessing_path:
header_path = None # default
else:
header_path = os.path.join(
self.preprocessing_path, self.animal_name, 'headers',
self.date)
os.makedirs(header_path, exist_ok=True)
header_file = HeaderProcessor.process_headers(
rec_files_list, copy_dir=header_path)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(
self.preprocessing_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = os.path.join(self.preprocessing_path,
self.animal_name, 'preprocessing', date)
validation_registrator = ValidationRegistrator()
validation_registrator.register(NTrodeValidator(
self.metadata, self.header, self.probes))
validation_registrator.register(PreprocessingValidator(
full_data_path,
self.dataset_names,
data_types_for_scanning
))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
if self.is_old_dataset:
if not session_start_time:
raise ValueError(
'session_start_time is required for old dataset.')
self.session_start_time = session_start_time
else:
session_time_extractor = SessionTimeExtractor(
self.datasets,
self.animal_name,
self.date,
self.dataset_names
)
self.session_start_time = session_time_extractor.get_session_start_time()
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes,
self.metadata,
self.header
)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(
self.device_factory, self.device_injector, self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
os.path.join(self.data_path, self.animal_name, 'raw', self.date))
if self.is_old_dataset:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS,
)
else:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data_acq_device']
)
if self.process_mda:
self.mda_originator = MdaOriginator(
self.datasets, self.header, self.metadata)
if self.process_dio:
if self.is_old_dataset:
self.dio_originator = DioOriginator(
self.metadata, self.datasets,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS)
else:
self.dio_originator = DioOriginator(
self.metadata, self.datasets)
if self.process_analog:
if self.is_old_dataset:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS)
else:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata,
self.dataset_names)