def test_get_metadata_lib_path():
"""Tests metadata_lib_path function."""
metadata_path = '/home/user/metadata'
del os.environ['NSDS_METADATA_PATH']
assert metadata_path == get_metadata_lib_path(metadata_path)
with pytest.raises(ValueError):
get_metadata_lib_path(None)
metadata_path2 = '/home/user/new_metadata'
os.environ['NSDS_METADATA_PATH'] = metadata_path2
assert metadata_path == get_metadata_lib_path(metadata_path)
assert metadata_path2 == get_metadata_lib_path(None)
class TestCase_MetadataManager(unittest.TestCase):
metadata_lib_path = get_metadata_lib_path()
metadata_save_path = '_test/'
def test_metadata_manager_case1_old_data(self):
''' detect/collect metadata needed to build the NWB file '''
block_name = 'R56_B13'
block_metadata_path = '_data/R56/R56_B13.yaml'
nwb_metadata = MetadataManager(
block_folder=block_name,
block_metadata_path=block_metadata_path,
metadata_lib_path=self.metadata_lib_path,
metadata_save_path=self.metadata_save_path)
nwb_metadata.extract_metadata()
def test_metadata_manager_case2_new_data(self):
''' detect/collect metadata needed to build the NWB file '''
block_name = 'RVG16_B01'
block_metadata_path = '_data/RVG16/RVG16_B01.yaml'
nwb_metadata = MetadataManager(
block_folder=block_name,
block_metadata_path=block_metadata_path,
metadata_lib_path=self.metadata_lib_path,
metadata_save_path=self.metadata_save_path,
legacy_block=False)
nwb_metadata.extract_metadata()
def __init__(self,
block_metadata_path: str,
metadata_lib_path=None,
stim_lib_path=None,
block_folder=None,
metadata_save_path=None,
experiment_type=_DEFAULT_EXPERIMENT_TYPE,
legacy_block=None,
):
self.block_metadata_path = block_metadata_path
self.metadata_lib_path = get_metadata_lib_path(metadata_lib_path)
self.stim_lib_path = get_stim_lib_path(stim_lib_path)
self.block_folder = block_folder
self.surgeon_initials, self.animal_name, self.block_tag = split_block_folder(block_folder)
self.metadata_save_path = metadata_save_path
self.experiment_type = experiment_type
self.yaml_lib_path = os.path.join(self.metadata_lib_path, self.experiment_type)
self.__detect_legacy_block(legacy_block)
if self.metadata_save_path is not None:
os.makedirs(self.metadata_save_path, exist_ok=True)
if self.legacy_block:
self.metadata_reader = LegacyMetadataReader(
block_metadata_path=self.block_metadata_path,
metadata_lib_path=self.metadata_lib_path,
block_folder=self.block_folder,
metadata_save_path=self.metadata_save_path)
else:
self.metadata_reader = MetadataReader(
block_metadata_path=self.block_metadata_path,
metadata_lib_path=self.metadata_lib_path,
block_folder=self.block_folder,
metadata_save_path=self.metadata_save_path)
def __init__(self,
block_metadata_path: str,
metadata_lib_path: str,
block_folder: str,
metadata_save_path=None,
):
self.block_metadata_path = block_metadata_path
self.metadata_lib_path = get_metadata_lib_path(metadata_lib_path)
self.block_folder = block_folder
self.surgeon_initials, self.animal_name, self.block_tag = split_block_folder(block_folder)
self.metadata_save_path = metadata_save_path
class TestCase_Tokenizers(unittest.TestCase):
data_path = get_data_path()
metadata_lib_path = get_metadata_lib_path()
stim_lib_path = get_stim_lib_path()
def test_wn_tokenizer(self):
''' test white noise stimulus tokenizer '''
block_name = 'RVG16_B01'
self.__run_test_tokenizer(block_name)
def test_tone_tokenizer(self):
''' test tone stimulus tokenizer '''
block_name = 'RVG16_B06'
self.__run_test_tokenizer(block_name)
def test_timit_tokenizer(self):
''' test TIMIT stimulus tokenizer '''
block_name = 'RVG16_B08'
self.__run_test_tokenizer(block_name)
def __run_test_tokenizer(self, block_name):
_, animal_name, _ = split_block_folder(block_name)
block_metadata_path = os.path.join(self.data_path, animal_name,
block_name, f"{block_name}.yaml")
metadata = MetadataManager(block_folder=block_name,
block_metadata_path=block_metadata_path,
metadata_lib_path=self.metadata_lib_path,
legacy_block=False).extract_metadata()
stim_configs = metadata['stimulus']
stim_vals = StimValueExtractor(stim_configs,
self.stim_lib_path).extract()
dataset = AuditoryDataScanner(block_name,
data_path=self.data_path,
stim_lib_path=self.stim_lib_path,
use_htk=False).extract_dataset()
# create an empty NWB file
nwb_content = NWBFile(
session_description='test stim tokenizers', # required
identifier=str(uuid.uuid1()), # required
session_start_time=get_default_time()) # required
# add mark track
mark_time_series = MarkManager(dataset).get_mark_track(
starting_time=0.0)
nwb_content.add_stimulus(mark_time_series)
# tokenize and add trials
trials_manager = TrialsManager(block_name, stim_configs)
trials_manager.add_trials(nwb_content, stim_vals)
def convert_block(block_folder: str,
save_path: str,
data_path=None,
stim_lib_path=None,
metadata_lib_path=None,
block_metadata_path=None,
metadata_save_path=None,
resample_data=True,
use_htk=False,
process_stim=True,
write_nwb=True,
add_preprocessing=False):
'''Wrapper for converting a single block of data using NWBBuilder.
'''
logger.debug(f'Converting block {block_folder}')
data_path = get_data_path(data_path)
metadata_lib_path = get_metadata_lib_path(metadata_lib_path)
stim_lib_path = get_stim_lib_path(stim_lib_path)
logger.debug(f'data_path={data_path}')
logger.debug(f'metadata_lib_path={metadata_lib_path}')
logger.debug(f'stim_lib_path={stim_lib_path}')
logger.debug(f'save_path={save_path}')
# create a builder for the block
nwb_builder = NWBBuilder(block_folder=block_folder,
data_path=data_path,
save_path=save_path,
block_metadata_path=block_metadata_path,
metadata_save_path=metadata_save_path,
stim_lib_path=stim_lib_path,
resample_data=resample_data,
use_htk=use_htk)
# build the NWB file content
nwb_content = nwb_builder.build(process_stim=process_stim)
# write to file
if write_nwb:
nwb_builder.write(nwb_content)
else:
logger.info('Finishing without writing to a file, because write_nwb is set to False.')
if add_preprocessing:
# use default parameters for preprocessing
preprocess_block(nwb_builder.output_file,
acq_name='ECoG', # for now ecog only?
**DEFAULT_PREPROCESSING_KWARGS,
logger=logger)
def test_legacy_nwb_builder(tmpdir, block_folder):
"""Runs the NWB pipline on a block."""
if not os.path.isdir(os.environ['NSDS_DATA_PATH']):
pytest.xfail('Testing data folder on catscan not found')
_, animal_name, _ = split_block_folder(block_folder)
data_path = get_data_path()
metadata_path = get_metadata_lib_path()
block_metadata = os.path.join(metadata_path, "auditory", "legacy", "yaml", "block",
animal_name, f"{block_folder}.yaml")
nwb_builder = NWBBuilder(
data_path=data_path,
block_folder=block_folder,
save_path=tmpdir,
block_metadata_path=block_metadata,
resample_data=RESAMPLE_DATA)
# build the NWB file content
nwb_content = nwb_builder.build()
# write to file
nwb_builder.write(nwb_content)
class TestCase_StimValueExtractor(unittest.TestCase):
stim_lib_path = get_stim_lib_path()
metadata_lib_path = get_metadata_lib_path()
def __test_stim(self, stim_name_input):
stim_name, stim_info = check_stimulus_name(stim_name_input)
stim_yaml_path = os.path.join(self.metadata_lib_path, 'auditory', 'yaml',
'stimulus', stim_name + '.yaml')
stim_configs = read_yaml(stim_yaml_path)
sve = StimValueExtractor(stim_configs, self.stim_lib_path)
stim_values = sve.extract()
return stim_values
def test_white_noise_stimuli(self):
stim_name = 'wn2'
stim_values = self.__test_stim(stim_name)
assert stim_values is not None
def test_tone_stimuli(self):
stim_name = 'tone'
stim_values = self.__test_stim(stim_name)
assert stim_values is not None
stim_name = 'tone150'
stim_values = self.__test_stim(stim_name)
assert stim_values is not None
def test_timit_stimuli(self):
stim_name = 'timit'
stim_values = self.__test_stim(stim_name)
assert stim_values is not None
def test_dmr_stimuli(self):
stim_name = 'dmr'
stim_values = self.__test_stim(stim_name)
assert stim_values is None
def __init__(self,
data_path: str,
block_folder: str,
save_path: str,
block_metadata_path: str = None,
metadata_lib_path: str = None,
stim_lib_path: str = None,
metadata_save_path: str = None,
resample_data=False,
use_htk=False):
self.data_path = get_data_path(data_path)
self.metadata_lib_path = get_metadata_lib_path(metadata_lib_path)
self.stim_lib_path = get_stim_lib_path(stim_lib_path)
self.surgeon_initials, self.animal_name, self.block_name = split_block_folder(
block_folder)
self.block_folder = block_folder
self.save_path = save_path
self.block_metadata_path = self._get_block_metadata_path(
block_metadata_path)
self.metadata_save_path = metadata_save_path
self.resample_data = resample_data
self.use_htk = use_htk
self.source_script, self.source_script_file_name = self._get_source_script(
)
logger.info('=======================================')
logger.info(f'Building NWB for block {block_folder}.')
logger.info('Collecting metadata for NWB conversion...')
self.metadata = self._collect_nwb_metadata()
self.experiment_type = self.metadata['experiment_type']
self.bad_block, incomplete_block = self._check_bad_block()
if incomplete_block:
self.bad_block = True
logger.info(
'Incomplete block. Escaping __init__ before originators.')
return
logger.info('Collecting relevant input data paths...')
self.dataset = self._collect_dataset_paths()
logger.info('Preparing output path...')
rat_out_dir = os.path.join(self.save_path, self.animal_name)
os.makedirs(rat_out_dir, exist_ok=True)
self.output_file = os.path.join(rat_out_dir,
f'{self.block_folder}.nwb')
logger.info('Initializing recordings manager...')
self.rec_manager = RecManager(self.dataset)
logger.info('Creating originator instances...')
self.electrodes_originator = ElectrodesOriginator(self.metadata)
self.neural_data_originator = NeuralDataOriginator(
self.rec_manager, self.metadata, resample_flag=self.resample_data)
self.stimulus_originator = StimulusOriginator(self.rec_manager,
self.dataset,
self.metadata)
logger.info('Extracting session start time...')
self.session_start_time = self._extract_session_start_time()