def test_data_scanner_return_all_data_with_None_date__successfully(self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('pos')), 3)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('DIO')), 7)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('LFP')), 4)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('mda')), 4)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('metadata')), 1)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('spikes')), 3)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('time')), 2)
def test_data_scanner_failed_due_to_None_parameter(self):
DataScanner(data_path=None,
animal_name='alien',
nwb_metadata=MetadataManager(
metadata_path=str(path) + '/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
def test_data_scanner_find_all_data_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.1.pos/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('pos'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.DIO/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('DIO'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.LFP/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('LFP'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.mda/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('mda'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.metadata/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('metadata'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.spikes/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('spikes'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.time/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('time'))
def test_data_scanner_find_all_datasets_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual(len(data_scanner.data.values()), 1)
self.assertEqual(len(data_scanner.data['alien'].values()), 1)
self.assertEqual(len(data_scanner.data['alien']['21251015'].values()),
2)
def test_data_scanner_find_all_probes_file_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
probe_path = path + '/res/probe_test'
probes = data_scanner.get_probes_from_directory(probe_path)
probes.sort()
self.assertTrue(probes[0].endswith('probe1.yml'))
self.assertTrue(probes[1].endswith('probe2.yml'))
self.assertTrue(probes[2].endswith('probe21.yml'))
for dataset in dataset_names
]
if __name__ == "__main__":
animal_name = 'beans'
date = '20190718'
data_path = 'C:/Users/wbodo/Desktop/resy/test/'
# data_path = str(path) + '/test/test_data/'
nwb_metadata = MetadataManager(
str(path) + '/test/processing/res/metadata.yml', [
str(path) + '/test/processing/res/probe1.yml',
str(path) + '/test/processing/res/probe2.yml',
str(path) + '/test/processing/res/probe3.yml'
])
data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
datasets = extract_datasets(data_scanner, animal_name, date)
pos_timestamps_files = [
get_posonline_data_file(dataset) for dataset in datasets
]
mda_timestamps_files = [
dataset.get_mda_timestamps() for dataset in datasets
]
continuous_time_files = [
dataset.get_continuous_time() for dataset in datasets
]
timestamps = [
read_pos_timestamps(timestamps_file)
for timestamps_file in pos_timestamps_files
def __init__(self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = ''):
logger.info('NWBFileBuilder initialization')
logger.info('NWB builder initialization parameters: \n' +
'data_path = ' + str(data_path) + '\n' + 'animal_name = ' +
str(animal_name) + '\n' + 'date = ' + str(date) + '\n' +
'nwb_metadata = ' + str(nwb_metadata) + '\n' +
'process_dio = ' + str(process_dio) + '\n' +
'process_mda = ' + str(process_mda) + '\n' +
'process_analog = ' + str(process_analog) + '\n' +
'output_file = ' + str(output_file) + '\n')
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if associated_files_existance_validator.files_exist():
pass
else:
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist"
)
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
self.output_file = output_file
self.video_path = video_path
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {
'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog
}
rec_files_list = RecFileFinder().find_rec_files(
path=(self.data_path + '/' + self.animal_name + '/raw/' +
self.date))
header_file = HeaderProcessor.process_headers(rec_files_list)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = data_path + '/' + animal_name + '/preprocessing/' + date
validation_registrator = ValidationRegistrator()
validation_registrator.register(
NTrodeValidator(self.metadata, self.header, self.probes))
validation_registrator.register(
PreprocessingValidator(full_data_path, self.dataset_names,
data_types_for_scanning))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
self.session_time_extractor = SessionTimeExtractor(
self.datasets, self.animal_name, self.date, self.dataset_names)
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes, self.metadata, self.header)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(self.device_factory,
self.device_injector,
self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/")
self.video_files_originator = VideoFilesOriginator(
self.data_path + "/" + animal_name + "/raw/" + self.date + "/",
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data acq device'])
if self.process_mda:
self.mda_originator = MdaOriginator(self.datasets, self.header,
self.metadata)
if self.process_dio:
self.dio_originator = DioOriginator(self.metadata, self.datasets)
if self.process_analog:
self.analog_originator = AnalogOriginator(self.datasets,
self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata, self.dataset_names,
self.process_pos_timestamps)
def __init__(
self,
data_path: str,
animal_name: str,
date: str,
nwb_metadata: MetadataManager,
process_dio: bool = True,
process_mda: bool = True,
process_analog: bool = True,
process_pos_timestamps: bool = True,
preprocessing_path: str = '',
video_path: str = '',
output_file: str = 'output.nwb',
reconfig_header: str = '',
is_old_dataset: bool = False,
session_start_time=None,
):
logger.info('NWBFileBuilder initialization')
logger.info(
'NWB builder initialization parameters: \n'
+ 'data_path = ' + str(data_path) + '\n'
+ 'animal_name = ' + str(animal_name) + '\n'
+ 'date = ' + str(date) + '\n'
+ 'nwb_metadata = ' + str(nwb_metadata) + '\n'
+ 'process_dio = ' + str(process_dio) + '\n'
+ 'process_mda = ' + str(process_mda) + '\n'
+ 'process_analog = ' + str(process_analog) + '\n'
+ 'output_file = ' + str(output_file) + '\n'
)
validation_registrator = ValidationRegistrator()
validation_registrator.register(PathValidator(data_path))
validation_registrator.validate()
self.animal_name = animal_name
self.date = date
self.data_path = data_path
self.metadata = nwb_metadata.metadata
metadata_section_validator = MetadataSectionValidator(self.metadata)
metadata_section_validator.validate_sections()
if self.metadata.get('associated_files', []):
associated_files_existance_validator = AssociatedFilesExistanceValidator(
self.metadata['associated_files'])
if not associated_files_existance_validator.files_exist():
raise Exception(
"one or more associated file listed in metadata.yaml file does not exist")
self.probes = nwb_metadata.probes
self.process_dio = process_dio
self.process_mda = process_mda
self.process_analog = process_analog
self.process_pos_timestamps = process_pos_timestamps
if not preprocessing_path:
self.preprocessing_path = data_path
else:
self.preprocessing_path = preprocessing_path
self.output_file = output_file
self.video_path = video_path
self.is_old_dataset = is_old_dataset
self.link_to_notes = self.metadata.get('link to notes', None)
data_types_for_scanning = {'pos': True,
'time': True,
'mda': process_mda,
'DIO': process_dio,
'analog': process_analog}
rec_files_list = RecFileFinder().find_rec_files(
path=os.path.join(
self.data_path, self.animal_name, 'raw', self.date)
)
if not preprocessing_path:
header_path = None # default
else:
header_path = os.path.join(
self.preprocessing_path, self.animal_name, 'headers',
self.date)
os.makedirs(header_path, exist_ok=True)
header_file = HeaderProcessor.process_headers(
rec_files_list, copy_dir=header_path)
if reconfig_header:
self.header = Header(reconfig_header)
else:
self.header = Header(header_file)
self.data_scanner = DataScanner(
self.preprocessing_path, animal_name, nwb_metadata)
self.dataset_names = self.data_scanner.get_all_epochs(date)
full_data_path = os.path.join(self.preprocessing_path,
self.animal_name, 'preprocessing', date)
validation_registrator = ValidationRegistrator()
validation_registrator.register(NTrodeValidator(
self.metadata, self.header, self.probes))
validation_registrator.register(PreprocessingValidator(
full_data_path,
self.dataset_names,
data_types_for_scanning
))
validation_registrator.register(TaskValidator(self.metadata['tasks']))
validation_registrator.validate()
self.__extract_datasets(animal_name, date)
self.corrupted_data_manager = CorruptedDataManager(self.metadata)
self.shanks_electrode_originator = ShanksElectrodeOriginator(
self.probes, self.metadata)
self.shanks_originator = ShanksOriginator(self.probes, self.metadata)
self.fl_probe_manager = FlProbeManager(self.probes)
self.device_injector = DeviceInjector()
self.device_factory = DeviceFactory()
self.electrode_group_originator = ElectrodeGroupOriginator(
self.metadata)
self.electrodes_originator = ElectrodesOriginator(
self.probes, self.metadata)
if self.is_old_dataset:
if not session_start_time:
raise ValueError(
'session_start_time is required for old dataset.')
self.session_start_time = session_start_time
else:
session_time_extractor = SessionTimeExtractor(
self.datasets,
self.animal_name,
self.date,
self.dataset_names
)
self.session_start_time = session_time_extractor.get_session_start_time()
self.mda_valid_time_originator = MdaValidTimeOriginator(
self.header, self.metadata)
self.mda_invalid_time_originator = MdaInvalidTimeOriginator(
self.header, self.metadata)
self.pos_valid_time_originator = PosValidTimeOriginator(self.metadata)
self.pos_invalid_time_originator = PosInvalidTimeOriginator(
self.metadata)
self.epochs_originator = EpochsOriginator(self.datasets)
if 'associated_files' in self.metadata:
self.associated_files_originator = AssociatedFilesOriginator(
self.metadata)
self.electrodes_extension_originator = ElectrodesExtensionOriginator(
self.probes,
self.metadata,
self.header
)
self.sample_count_timestamp_corespondence_originator =\
SampleCountTimestampCorespondenceOriginator(self.datasets)
self.processing_module_originator = ProcessingModuleOriginator()
self.task_originator = TaskOriginator(self.metadata)
self.camera_device_originator = CameraDeviceOriginator(self.metadata)
self.header_device_originator = HeaderDeviceOriginator(
self.header, self.metadata)
self.probes_originator = ProbeOriginator(
self.device_factory, self.device_injector, self.probes)
self.camera_sample_frame_counts_originator = CameraSampleFrameCountsOriginator(
os.path.join(self.data_path, self.animal_name, 'raw', self.date))
if self.is_old_dataset:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS,
)
else:
self.video_files_originator = VideoFilesOriginator(
os.path.join(self.data_path, self.animal_name,
'raw', self.date),
self.video_path,
self.metadata["associated_video_files"],
)
self.data_acq_device_originator = DataAcqDeviceOriginator(
device_factory=self.device_factory,
device_injector=self.device_injector,
metadata=self.metadata['data_acq_device']
)
if self.process_mda:
self.mda_originator = MdaOriginator(
self.datasets, self.header, self.metadata)
if self.process_dio:
if self.is_old_dataset:
self.dio_originator = DioOriginator(
self.metadata, self.datasets,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS)
else:
self.dio_originator = DioOriginator(
self.metadata, self.datasets)
if self.process_analog:
if self.is_old_dataset:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata,
convert_timestamps=_CONVERT_OLD_TIMESTAMPS,
return_timestamps=_RETURN_OLD_TIMESTAMPS)
else:
self.analog_originator = AnalogOriginator(
self.datasets, self.metadata)
self.position_originator = PositionOriginator(
self.datasets, self.metadata,
self.dataset_names)