def test_metadata_manager_failed_reading_metadata_due_to_None_param(self):
MetadataManager(metadata_path=None,
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
])
def test_data_scanner_return_all_data_with_None_date__successfully(self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('pos')), 3)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('DIO')), 7)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('LFP')), 4)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('mda')), 4)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('metadata')), 1)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('spikes')), 3)
self.assertEqual(
len(data_scanner.data['alien']['21251015']
['01_s1'].get_all_data_from_dataset('time')), 2)
def test_build_and_read_nwb(self):
metadata = MetadataManager(
str(path) +
'/rec_to_nwb/rec_to_nwb/test/processing/res/metadata.yml', [
str(path) +
'/rec_to_nwb/rec_to_nwb/test/processing/res/probe1.yml',
str(path) +
'/rec_to_nwb/rec_to_nwb/test/processing/res/probe2.yml',
str(path) +
'/rec_to_nwb/rec_to_nwb/test/processing/res/probe3.yml'
])
builder = RawToNWBBuilder(
animal_name='beans',
data_path=str(path) + '/rec_to_nwb/rec_to_nwb/test/test_data/',
dates=['20190718'],
nwb_metadata=metadata,
output_path='',
extract_spikes=False,
extract_mda=True,
extract_lfps=False,
extract_analog=True,
extract_dio=True,
overwrite=True,
trodes_rec_export_args=_DEFAULT_TRODES_REC_EXPORT_ARGS,
video_path=str(path) + '/rec_to_nwb/rec_to_nwb/test/test_data')
builder.build_nwb()
self.assertTrue(os.path.exists('beans20190718.nwb'))
with pynwb.NWBHDF5IO('beans20190718.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
print(nwb_file)
if os.path.isfile('beans20190718.nwb'):
os.remove('beans20190718.nwb')
def test_metadata_manager_failed_reading_metadata_due_to_incorrect_type_of_param(
self):
MetadataManager(metadata_path=123,
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
])
def test_metadata_manager_reading_metadata_successfully(self):
nwb_metadata = MetadataManager(metadata_path=str(path) +
'/../res/metadata.yml',
probes_paths=[
str(path) + '/../res/probe1.yml',
str(path) + '/../res/probe2.yml',
str(path) + '/../res/probe3.yml'
])
metadata_fields = nwb_metadata.metadata.keys()
self.assertIn('experimenter name', metadata_fields)
self.assertIn('lab', metadata_fields)
self.assertIn('institution', metadata_fields)
self.assertIn('session_id', metadata_fields)
self.assertIn('experiment description', metadata_fields)
self.assertIn('session description', metadata_fields)
self.assertIn('subject', metadata_fields)
self.assertIn('tasks', metadata_fields)
self.assertIn('behavioral_events', metadata_fields)
self.assertIn('electrode groups', metadata_fields)
self.assertIn('ntrode electrode group channel map', metadata_fields)
self.assertIn('units', metadata_fields)
self.assertIn('unspecified', nwb_metadata.metadata['units']['analog'])
self.assertIn('unspecified',
nwb_metadata.metadata['units']['behavioral_events'])
subject_fields = nwb_metadata.metadata['subject'].keys()
self.assertIn('description', subject_fields)
self.assertIn('genotype', subject_fields)
self.assertIn('sex', subject_fields)
self.assertIn('species', subject_fields)
self.assertIn('subject id', subject_fields)
self.assertIn('weight', subject_fields)
tasks_fields = nwb_metadata.metadata['tasks'][0].keys()
self.assertIn('task_description', tasks_fields)
self.assertIn('task_name', tasks_fields)
behavioral_event_fields = nwb_metadata.metadata['behavioral_events'][
0].keys()
self.assertIn('description', behavioral_event_fields)
self.assertIn('name', behavioral_event_fields)
electrode_groups_fields = nwb_metadata.metadata['electrode groups'][
0].keys()
self.assertIn('id', electrode_groups_fields)
self.assertIn('location', electrode_groups_fields)
self.assertIn('device_type', electrode_groups_fields)
self.assertIn('description', electrode_groups_fields)
ntrode_probe_channel_map_fields = nwb_metadata.metadata[
'ntrode electrode group channel map'][0].keys()
self.assertIn('map', ntrode_probe_channel_map_fields)
self.assertIn('electrode_group_id', ntrode_probe_channel_map_fields)
self.assertIn('ntrode_id', ntrode_probe_channel_map_fields)
def test_data_scanner_failed_due_to_None_parameter(self):
DataScanner(data_path=None,
animal_name='alien',
nwb_metadata=MetadataManager(
metadata_path=str(path) + '/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
def convert_kibbles_single_day(animal_name: str,
date: str,
yaml_path='../yaml',
reconfig_file=None):
dates = [date] # single date for now
# Specify the paths for the data, the output nwb file, and the video files
# data are saved in data_path/animal_name/raw/date/*.*
# data_path = '/data2/data1_backup/jason/'
data_path = '/stelmo/jhbak/Data/' # temporary copy with renamed files
out_path = '/stelmo/jhbak/nwb/conversion/' # both preprocessing and NWB
output_path = out_path + animal_name + '/out/'
video_path = out_path + animal_name + '/video/'
# Specify metadata files
animal_metadata_file = '{}_{}_metadata.yml'.format(animal_name, date)
probe_metadata_files = [
'tetrode_12.5.yml',
'32c-2s8mm6cm-20um-40um-dl.yml',
'64c-3s6mm6cm-20um-40um-sl.yml', # kibbles
'64c-4s6mm6cm-20um-40um-dl.yml', # kibbles
]
# specify the locations of the metadata files for the animal and the probe(s).
# Note that specifying all possible probes is fine
animal_metadata = os.path.join(yaml_path, animal_metadata_file)
probe_metadata = [
os.path.join(yaml_path, file) for file in probe_metadata_files
]
# Specify any optional trodes export flags
if reconfig_file is None:
trodes_rec_export_args = ()
else:
trodes_rec_export_args = ('-reconfig', reconfig_file)
metadata = MetadataManager(animal_metadata, probe_metadata)
builder = RawToNWBBuilder(
animal_name=animal_name,
data_path=data_path,
dates=dates,
nwb_metadata=metadata,
overwrite=OVERWRITE,
preprocessing_path=out_path, # new
output_path=output_path,
video_path=video_path,
trodes_rec_export_args=trodes_rec_export_args)
_ = builder.build_nwb(run_preprocessing=RUN_PREPROCESSING)
def test_data_scanner_find_all_data_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.1.pos/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('pos'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.DIO/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('DIO'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.LFP/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('LFP'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.mda/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('mda'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.metadata/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('metadata'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.spikes/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('spikes'))
self.assertEqual((
path +
'/res/scanner_test/alien/preprocessing/21251015/21251015_alien_01_s1.time/'
), data_scanner.data['alien']['21251015']
['01_s1'].get_data_path_from_dataset('time'))
def setUpClass(cls):
cls.metadata = MetadataManager(
str(path) + '/processing/res/metadata.yml', [
str(path) + '/processing/res/probe1.yml',
str(path) + '/processing/res/probe2.yml',
str(path) + '/processing/res/probe3.yml'
])
cls.nwb_builder = NWBFileBuilder(data_path=str(path) + '/test_data/',
animal_name='beans',
date='20190718',
nwb_metadata=cls.metadata,
process_dio=True,
process_mda=True,
process_analog=True,
process_pos_timestamps=True,
video_path=str(path) + '/test_data')
def setUpClass(cls):
cls.metadata = MetadataManager(
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/kibbles20170216_metadata.yml',
[
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/64c-3s6mm6cm-20um-40um-sl.yml',
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/64c-4s6mm6cm-20um-40um-dl.yml'
])
cls.nwb_builder = NWBFileBuilder(data_path=str(path) + '/test_data/',
animal_name='KF2',
date='20170120',
nwb_metadata=cls.metadata,
process_dio=True,
process_mda=True,
process_analog=True,
process_pos_timestamps=True,
video_path=str(path) + '/test_data')
def test_data_scanner_find_all_datasets_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
data_scanner.extract_data_from_all_dates_folders()
self.assertEqual(len(data_scanner.data.values()), 1)
self.assertEqual(len(data_scanner.data['alien'].values()), 1)
self.assertEqual(len(data_scanner.data['alien']['21251015'].values()),
2)
def test_data_scanner_find_all_probes_file_with_None_date_param_successfully(
self):
data_scanner = DataScanner(
data_path=path + '/res/scanner_test/',
animal_name='alien',
nwb_metadata=MetadataManager(metadata_path=str(path) +
'/res/metadata.yml',
probes_paths=[
str(path) + '/res/probe1.yml',
str(path) + '/res/probe2.yml',
str(path) + '/res/probe3.yml'
]))
probe_path = path + '/res/probe_test'
probes = data_scanner.get_probes_from_directory(probe_path)
probes.sort()
self.assertTrue(probes[0].endswith('probe1.yml'))
self.assertTrue(probes[1].endswith('probe2.yml'))
self.assertTrue(probes[2].endswith('probe21.yml'))
def test_metadata_manager_reading_probes_successfully(self):
nwb_metadata = MetadataManager(metadata_path=str(path) +
'/../res/metadata.yml',
probes_paths=[
str(path) + '/../res/probe1.yml',
str(path) + '/../res/probe2.yml',
str(path) + '/../res/probe3.yml'
])
shanks_fields = nwb_metadata.probes[0]['shanks'][0].keys()
self.assertIn('shank_id', shanks_fields)
self.assertIn('electrodes', shanks_fields)
self.assertIn('electrodes', shanks_fields)
self.assertEqual(nwb_metadata.probes[0]['probe_type'], 'tetrode_12.5')
self.assertEqual(nwb_metadata.probes[0]['probe_description'],
'four wire electrode')
self.assertEqual(nwb_metadata.probes[0]['contact_side_numbering'],
True)
self.assertEqual(nwb_metadata.probes[0]['contact_size'], 12.5)
def setUp(self):
self.metadata = MetadataManager(
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/kibbles20170216_metadata.yml',
[
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/64c-3s6mm6cm-20um-40um-sl.yml',
'C:/Users/wmery/PycharmProjects/rec_to_nwb/rec_to_nwb/test/test_data/KF2/raw/20170120/64c-4s6mm6cm-20um-40um-dl.yml'
])
self.builder = RawToNWBBuilder(
animal_name='KF2',
data_path=str(path) + '/../test_data/',
dates=['20170120'],
nwb_metadata=self.metadata,
output_path='',
video_path=str(path) + '/../test_data',
extract_spikes=False,
extract_mda=True,
extract_lfps=False,
extract_analog=True,
extract_dio=True,
overwrite=True,
trodes_rec_export_args=_DEFAULT_TRODES_REC_EXPORT_ARGS)
def setUp(self):
self.metadata = MetadataManager(
str(path) + '/../processing/res/metadata.yml',
[
str(path) + '/../processing/res/probe1.yml',
str(path) + '/../processing/res/probe2.yml',
str(path) + '/../processing/res/probe3.yml'
]
)
self.builder = RawToNWBBuilder(
animal_name='beans',
data_path=str(path) + '/../test_data/',
dates=['20190718'],
nwb_metadata=self.metadata,
output_path='',
video_path=str(path) + '/../test_data',
extract_spikes=False,
extract_mda=True,
extract_lfps=False,
extract_analog=True,
extract_dio=True,
overwrite=True,
trodes_rec_export_args=_DEFAULT_TRODES_REC_EXPORT_ARGS
)
data_scanner.extract_data_from_date_folder(date)
dataset_names = data_scanner.get_all_epochs(date)
return [
data_scanner.data[animal_name][date][dataset]
for dataset in dataset_names
]
if __name__ == "__main__":
animal_name = 'beans'
date = '20190718'
data_path = 'C:/Users/wbodo/Desktop/resy/test/'
# data_path = str(path) + '/test/test_data/'
nwb_metadata = MetadataManager(
str(path) + '/test/processing/res/metadata.yml', [
str(path) + '/test/processing/res/probe1.yml',
str(path) + '/test/processing/res/probe2.yml',
str(path) + '/test/processing/res/probe3.yml'
])
data_scanner = DataScanner(data_path, animal_name, nwb_metadata)
datasets = extract_datasets(data_scanner, animal_name, date)
pos_timestamps_files = [
get_posonline_data_file(dataset) for dataset in datasets
]
mda_timestamps_files = [
dataset.get_mda_timestamps() for dataset in datasets
]
continuous_time_files = [
dataset.get_continuous_time() for dataset in datasets
]
def main():
print('Setting variables', flush=True)
# set the animal name and the date or list of dates to process
# animal_name = 'beans'
# date = '20190718'
# animal_metadata_file = 'beans20190718_metadata.yml'
# reconfig_file = '/stelmo/loren/beans/Probe_128ch_allnT_DIOs_PTP_reconfig_export_shanks.xml'
animal_name = 'despereaux'
date = '20191125'
animal_metadata_file = 'despereaux20191125.yml'
reconfig_file = None
#reconfig_file = '/stelmo/loren/despereaux/raw/20191125/20191125_despereaux_01_s1.rec_header.xml'
# animal_name = 'Jaq'
# date = '20190826'
# animal_metadata_file = 'jaq20190826.yml'
# reconfig_file = None
yaml_path = '/home/loren/Src/NWB/franklabnwb/yaml'
probe1_metadata_file = '128c-4s8mm6cm-20um-40um-sl.yml'
probe2_metadata_file = 'tetrode_12.5.yml'
#Specify the paths for the data, the output nwb file, and the video files
data_path = '/stelmo/loren/'
output_path = '/stelmo/nwb/'
video_path = '/stelmo/nwb/video/'
# data_path = '/Users/loren/data/nwb_builder_test_data/'
# output_path='//Users/loren/data/nwb_builder_test_data/tmp'
# video_path='/Users/loren/data/nwb_builder_test_data/tmp'
# specify the locations of the metadata files for the animal and the probe(s).
# Note that specifying all possible probes is fine
animal_metadata = os.path.join(yaml_path, animal_metadata_file)
probe_metadata = [
os.path.join(yaml_path, probe1_metadata_file),
os.path.join(yaml_path, probe2_metadata_file)
]
print(probe_metadata)
# Specify whether data should be reextracted.
overwrite = False
# metadata parameters
metadata = MetadataManager(animal_metadata, probe_metadata)
print(metadata, flush=True)
print('Creating Builder', flush=True)
if reconfig_file is not None:
# Specify any optional trodes export flags
trodes_rec_export_args = ('-reconfig', reconfig_file)
builder = RawToNWBBuilder(
animal_name=animal_name,
data_path=data_path,
dates=[date],
nwb_metadata=metadata,
overwrite=overwrite,
output_path=output_path,
video_path=video_path,
extract_analog=True,
trodes_rec_export_args=trodes_rec_export_args)
else:
builder = RawToNWBBuilder(animal_name=animal_name,
data_path=data_path,
dates=[date],
nwb_metadata=metadata,
overwrite=overwrite,
output_path=output_path,
video_path=video_path,
extract_analog=True)
print('Building and Writing', flush=True)
builder.build_nwb(process_mda_valid_time='False',
process_mda_invalid_time='False',
process_pos_valid_time='False',
process_pos_invalid_time='False')
print('Done', flush=True)