def test_import_data(self):
"""Verify import_data successfully imports data"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
output_fnames = []
root, ext = os.path.splitext(os.path.basename(sample_utwin_file))
for dataset in expected_utwin_data:
output_fnames.append(
os.path.join(pathfinder.data_path(),
root + "_" + dataset + ".hdf5"))
batchui_ctrl.import_data(sample_utwin_file)
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
fname = os.path.join(pathfinder.data_path(),
root + "_" + dataset + ".hdf5")
self.assertTrue(os.path.exists(fname))
self.assertTrue(
np.array_equal(expected_utwin_data[dataset],
dataio.get_data(fname)))
for fname in output_fnames:
try:
if os.path.exists(fname):
os.remove(fname)
except WindowsError: # file in use (Windows)
pass
except OSError: # other OS error
pass
def test_read_data(self):
"""Verify returning converted data from various file formats"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
# Verify Winspect 6/7 data retrieval
sample_winspect_file = os.path.join(sample_data_folder,
'sample_data.sdt')
expected_winspect_data = dataio.get_winspect_data(sample_winspect_file)
retrieved_winspect_data = batchui_ctrl.read_data(sample_winspect_file)
for dataset in expected_winspect_data:
self.assertTrue(
np.array_equal(
dataset.data,
retrieved_winspect_data[dataset.data_type + "0"]))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder,
'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
retrieved_img_data = batchui_ctrl.read_data(sample_img_file)
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin Retrieval
sample_utwin_file = os.path.join(sample_data_folder, "CScanData.csc")
utwin_data = dataio.get_utwin_data(sample_utwin_file)
retrieved_utwin_data = batchui_ctrl.read_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
for dataset in expected_utwin_data:
if expected_utwin_data is not None:
self.assertTrue(
np.array_equal(expected_utwin_data[dataset],
retrieved_utwin_data[dataset]))
def test_read_data(self):
"""Verify returning converted data from various file formats"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
# Verify Winspect 6/7 data retrieval
sample_winspect_file = os.path.join(sample_data_folder, 'sample_data.sdt')
expected_winspect_data = dataio.get_winspect_data(sample_winspect_file)
retrieved_winspect_data = batchui_ctrl.read_data(sample_winspect_file)
for dataset in expected_winspect_data:
self.assertTrue(np.array_equal(dataset.data, retrieved_winspect_data[dataset.data_type + "0"]))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder, 'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
retrieved_img_data = batchui_ctrl.read_data(sample_img_file)
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin Retrieval
sample_utwin_file = os.path.join(sample_data_folder, "CScanData.csc")
utwin_data = dataio.get_utwin_data(sample_utwin_file)
retrieved_utwin_data = batchui_ctrl.read_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
for dataset in expected_utwin_data:
if expected_utwin_data is not None:
self.assertTrue(np.array_equal(expected_utwin_data[dataset], retrieved_utwin_data[dataset]))
def test_import_data(self):
"""Verify import_data successfully imports data"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
output_fnames = []
root, ext = os.path.splitext(os.path.basename(sample_utwin_file))
for dataset in expected_utwin_data:
output_fnames.append(os.path.join(pathfinder.data_path(), root + "_" + dataset + ".hdf5"))
batchui_ctrl.import_data(sample_utwin_file)
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
fname = os.path.join(pathfinder.data_path(), root + "_" + dataset + ".hdf5")
self.assertTrue(os.path.exists(fname))
self.assertTrue(np.array_equal(expected_utwin_data[dataset], dataio.get_data(fname)))
for fname in output_fnames:
try:
if os.path.exists(fname):
os.remove(fname)
except WindowsError: # file in use (Windows)
pass
except OSError: # other OS error
pass
def test_read_data(self):
"""Verify returning NumPy data from a file"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
# Verify Winspect 6/7 retrieval
sample_winspect_file = os.path.join(sample_data_folder, 'sample_data.sdt')
winspect_adapter = self.create_adapter(sample_winspect_file)
winspect_adapter.read_data()
returned_winspect_data = dataio.get_winspect_data(sample_winspect_file)
for dataset in returned_winspect_data:
expected_winspect_data = dataset.data
retrieved_winspect_data = winspect_adapter.data[dataset.data_type + "0"]
self.assertTrue(np.array_equal(expected_winspect_data, retrieved_winspect_data))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder, 'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
img_adapter = self.create_adapter(sample_img_file)
img_adapter.read_data()
retrieved_img_data = img_adapter.data
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin retrieval
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_adapter = self.create_adapter(sample_utwin_file)
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
utwin_adapter.read_data()
retrieved_utwin_data = utwin_adapter.data
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
self.assertTrue(np.array_equal(expected_utwin_data[dataset], retrieved_utwin_data[dataset]))
def test_get_utwin_data(self):
"""Verify returning UTWin data"""
sample_data_file = os.path.join(os.path.dirname(__file__), 'support_files', 'CScanData.csc')
sample_reader = dataio.UTWinCScanDataFile(sample_data_file)
sample_reader.read_data()
expected_data = sample_reader.data
returned_data = dataio.get_utwin_data(sample_data_file)
for datatype in expected_data:
self.assertTrue(np.array_equal(expected_data[datatype], returned_data[datatype]))
def test_get_utwin_data(self):
"""Verify returning UTWin data"""
sample_data_file = os.path.join(os.path.dirname(__file__),
'support_files', 'CScanData.csc')
sample_reader = dataio.UTWinCScanDataFile(sample_data_file)
sample_reader.read_data()
expected_data = sample_reader.data
returned_data = dataio.get_utwin_data(sample_data_file)
for datatype in expected_data:
self.assertTrue(
np.array_equal(expected_data[datatype],
returned_data[datatype]))
def test_run_plugin_multi_datasets(self):
"""Verify run_plugin convenience function correctly handles datafiles with
multiple datasets"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
output_fnames = []
root, ext = os.path.splitext(os.path.basename(sample_utwin_file))
for dataset in expected_utwin_data:
output_fnames.append(
os.path.join(pathfinder.batchoutput_path(),
root + "_" + dataset + ".hdf5"))
# Verify no output saved
batchui_ctrl.run_plugin(self.toolkit_class,
sample_utwin_file,
save_data=False)
for fname in output_fnames:
self.assertFalse(os.path.exists(fname))
# Verify output saved
batchui_ctrl.run_plugin(self.toolkit_class,
sample_utwin_file,
save_data=True)
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
fname = os.path.join(pathfinder.batchoutput_path(),
root + "_" + dataset + ".hdf5")
self.assertTrue(os.path.exists(fname))
plugin_names, plugin_classes = self.get_available_plugins()
for idx in range(len(plugin_names)):
if plugin_names[idx] == self.toolkit_class:
plugin_instance = plugin_classes[idx]()
plugin_instance.data = expected_utwin_data[dataset]
plugin_instance.run()
expected_data = plugin_instance.data
returned_data = dataio.get_data(fname)
self.assertTrue(
np.array_equal(expected_data, returned_data))
break
for fname in output_fnames:
try:
if os.path.exists(fname):
os.remove(fname)
except WindowsError: # file in use (Windows)
pass
except OSError: # other OS error
pass
def test_run_plugin_multi_datasets(self):
"""Verify run_plugin convenience function correctly handles datafiles with
multiple datasets"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
output_fnames = []
root, ext = os.path.splitext(os.path.basename(sample_utwin_file))
for dataset in expected_utwin_data:
output_fnames.append(os.path.join(pathfinder.batchoutput_path(), root + "_" + dataset + ".hdf5"))
# Verify no output saved
batchui_ctrl.run_plugin(self.toolkit_class, sample_utwin_file, save_data=False)
for fname in output_fnames:
self.assertFalse(os.path.exists(fname))
# Verify output saved
batchui_ctrl.run_plugin(self.toolkit_class, sample_utwin_file, save_data=True)
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
fname = os.path.join(pathfinder.batchoutput_path(), root + "_" + dataset + ".hdf5")
self.assertTrue(os.path.exists(fname))
plugin_names, plugin_classes = self.get_available_plugins()
for idx in range(len(plugin_names)):
if plugin_names[idx] == self.toolkit_class:
plugin_instance = plugin_classes[idx]()
plugin_instance.data = expected_utwin_data[dataset]
plugin_instance.run()
expected_data = plugin_instance.data
returned_data = dataio.get_data(fname)
self.assertTrue(np.array_equal(expected_data, returned_data))
break
for fname in output_fnames:
try:
if os.path.exists(fname):
os.remove(fname)
except WindowsError: # file in use (Windows)
pass
except OSError: # other OS error
pass
def test_read_data(self):
"""Verify returning NumPy data from a file"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
# Verify Winspect 6/7 retrieval
sample_winspect_file = os.path.join(sample_data_folder,
'sample_data.sdt')
winspect_adapter = self.create_adapter(sample_winspect_file)
winspect_adapter.read_data()
returned_winspect_data = dataio.get_winspect_data(sample_winspect_file)
for dataset in returned_winspect_data:
expected_winspect_data = dataset.data
retrieved_winspect_data = winspect_adapter.data[dataset.data_type +
"0"]
self.assertTrue(
np.array_equal(expected_winspect_data,
retrieved_winspect_data))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder,
'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
img_adapter = self.create_adapter(sample_img_file)
img_adapter.read_data()
retrieved_img_data = img_adapter.data
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin retrieval
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_adapter = self.create_adapter(sample_utwin_file)
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
utwin_adapter.read_data()
retrieved_utwin_data = utwin_adapter.data
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
self.assertTrue(
np.array_equal(expected_utwin_data[dataset],
retrieved_utwin_data[dataset]))
def read_data(filename, filetype=None):
"""Attempts to import the specified file based on the provided filetype, or automatically guesses the file format
based on the file extension if no filetype is given. Returns the data as a NumPy array if successfully imported as
a NumPy array if the file contained a single dataset or as a dict if multiple datasets were found."""
tof_counter = 0
amp_counter = 0
waveform_counter = 0
data = {}
if filetype is None:
filetype = get_file_type(filename)
if filetype is not None and filetype in available_file_types():
if filetype == 'nditoolbox':
data = dataio.get_data(filename)
if filetype == 'winspect':
raw_data = dataio.get_winspect_data(filename)
# Handle any files that may have stored multiple datasets of
# a given type(s)
for dataset in raw_data:
dataset_key = os.path.basename(filename)
if dataset.data_type == 'waveform':
dataset_key = 'waveform' + str(waveform_counter)
waveform_counter +=1
elif dataset.data_type == 'amplitude':
dataset_key = 'amplitude' + str(amp_counter)
amp_counter += 1
elif dataset.data_type == 'tof': #TODO -confirm SDT files use tof
dataset_key = 'tof' + str(tof_counter)
tof_counter += 1
data[dataset_key] = dataset.data
if filetype == 'csv':
data = dataio.get_txt_data(filename)
if filetype == 'image':
data = dataio.get_img_data(filename, flatten=True)
if filetype == 'dicom':
data = dataio.get_dicom_data(filename)
if filetype == 'utwin':
raw_data = dataio.get_utwin_data(filename)
for k in raw_data.keys():
for idx in range(len(raw_data[k])):
data[k + str(idx)] = raw_data[k][idx]
return data