def test_read_data(self):
"""Verify returning converted data from various file formats"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
# Verify Winspect 6/7 data retrieval
sample_winspect_file = os.path.join(sample_data_folder, 'sample_data.sdt')
expected_winspect_data = dataio.get_winspect_data(sample_winspect_file)
retrieved_winspect_data = batchui_ctrl.read_data(sample_winspect_file)
for dataset in expected_winspect_data:
self.assertTrue(np.array_equal(dataset.data, retrieved_winspect_data[dataset.data_type + "0"]))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder, 'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
retrieved_img_data = batchui_ctrl.read_data(sample_img_file)
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin Retrieval
sample_utwin_file = os.path.join(sample_data_folder, "CScanData.csc")
utwin_data = dataio.get_utwin_data(sample_utwin_file)
retrieved_utwin_data = batchui_ctrl.read_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
for dataset in expected_utwin_data:
if expected_utwin_data is not None:
self.assertTrue(np.array_equal(expected_utwin_data[dataset], retrieved_utwin_data[dataset]))
def test_read_data(self):
"""Verify returning converted data from various file formats"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
# Verify Winspect 6/7 data retrieval
sample_winspect_file = os.path.join(sample_data_folder,
'sample_data.sdt')
expected_winspect_data = dataio.get_winspect_data(sample_winspect_file)
retrieved_winspect_data = batchui_ctrl.read_data(sample_winspect_file)
for dataset in expected_winspect_data:
self.assertTrue(
np.array_equal(
dataset.data,
retrieved_winspect_data[dataset.data_type + "0"]))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder,
'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
retrieved_img_data = batchui_ctrl.read_data(sample_img_file)
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin Retrieval
sample_utwin_file = os.path.join(sample_data_folder, "CScanData.csc")
utwin_data = dataio.get_utwin_data(sample_utwin_file)
retrieved_utwin_data = batchui_ctrl.read_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
for dataset in expected_utwin_data:
if expected_utwin_data is not None:
self.assertTrue(
np.array_equal(expected_utwin_data[dataset],
retrieved_utwin_data[dataset]))
def test_read_data(self):
"""Verify returning NumPy data from a file"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models', 'tests', 'support_files')
# Verify Winspect 6/7 retrieval
sample_winspect_file = os.path.join(sample_data_folder, 'sample_data.sdt')
winspect_adapter = self.create_adapter(sample_winspect_file)
winspect_adapter.read_data()
returned_winspect_data = dataio.get_winspect_data(sample_winspect_file)
for dataset in returned_winspect_data:
expected_winspect_data = dataset.data
retrieved_winspect_data = winspect_adapter.data[dataset.data_type + "0"]
self.assertTrue(np.array_equal(expected_winspect_data, retrieved_winspect_data))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder, 'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
img_adapter = self.create_adapter(sample_img_file)
img_adapter.read_data()
retrieved_img_data = img_adapter.data
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin retrieval
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_adapter = self.create_adapter(sample_utwin_file)
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type + str(idx)] = utwin_data[data_type][idx]
utwin_adapter.read_data()
retrieved_utwin_data = utwin_adapter.data
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
self.assertTrue(np.array_equal(expected_utwin_data[dataset], retrieved_utwin_data[dataset]))
def test_import_winspect(self):
"""Verify import of Winspect data through convenience function"""
sample_data_file = os.path.join(os.path.dirname(__file__), 'support_files', 'sample_data.sdt')
assert(os.path.exists(sample_data_file))
output_basename, ext = os.path.splitext(sample_data_file)
amp_dest_file = os.path.join(pathfinder.data_path(),
os.path.basename(output_basename) + "_ampdata0" + ext + ".hdf5")
waveform_dest_file = os.path.join(pathfinder.data_path(),
os.path.basename(output_basename) + "_waveformdata0" + ext + ".hdf5")
dataio.import_winspect(sample_data_file)
expected_data_list = dataio.get_winspect_data(sample_data_file)
for dataset in expected_data_list:
if "amplitude" in dataset.data_type:
dest_file = amp_dest_file
elif "waveform" in dataset.data_type:
dest_file = waveform_dest_file
with h5py.File(dest_file, "r") as fidin:
root, ext = os.path.splitext(os.path.basename(dest_file))
for key in fidin.keys():
if key.startswith(root):
read_data = fidin[key][...]
self.assertTrue(np.array_equal(dataset.data, read_data))
try:
if os.path.exists(dest_file):
os.remove(dest_file)
except WindowsError: # file in use
pass
def test_import_winspect(self):
"""Verify import of Winspect data through convenience function"""
sample_data_file = os.path.join(os.path.dirname(__file__),
'support_files', 'sample_data.sdt')
assert (os.path.exists(sample_data_file))
output_basename, ext = os.path.splitext(sample_data_file)
amp_dest_file = os.path.join(
pathfinder.data_path(),
os.path.basename(output_basename) + "_ampdata0" + ext + ".hdf5")
waveform_dest_file = os.path.join(
pathfinder.data_path(),
os.path.basename(output_basename) + "_waveformdata0" + ext +
".hdf5")
dataio.import_winspect(sample_data_file)
expected_data_list = dataio.get_winspect_data(sample_data_file)
for dataset in expected_data_list:
if "amplitude" in dataset.data_type:
dest_file = amp_dest_file
elif "waveform" in dataset.data_type:
dest_file = waveform_dest_file
with h5py.File(dest_file, "r") as fidin:
root, ext = os.path.splitext(os.path.basename(dest_file))
for key in fidin.keys():
if key.startswith(root):
read_data = fidin[key][...]
self.assertTrue(np.array_equal(dataset.data,
read_data))
try:
if os.path.exists(dest_file):
os.remove(dest_file)
except WindowsError: # file in use
pass
def test_get_winspect_data(self):
"""Verify retrieval of Winspect data through convenience function"""
sample_data_file = os.path.join(os.path.dirname(__file__), 'support_files', 'sample_data.sdt')
assert(os.path.exists(sample_data_file))
scan_reader = dataio.WinspectReader(sample_data_file)
expected_data_list = scan_reader.get_winspect_data()
retrieved_data_list = dataio.get_winspect_data(sample_data_file)
self.assertEqual(len(expected_data_list), len(retrieved_data_list))
for data_array_idx in range(len(expected_data_list)):
self.assertTrue(np.array_equal(expected_data_list[data_array_idx].data, retrieved_data_list[data_array_idx].data))
def test_get_winspect_data(self):
"""Verify retrieval of Winspect data through convenience function"""
sample_data_file = os.path.join(os.path.dirname(__file__),
'support_files', 'sample_data.sdt')
assert (os.path.exists(sample_data_file))
scan_reader = dataio.WinspectReader(sample_data_file)
expected_data_list = scan_reader.get_winspect_data()
retrieved_data_list = dataio.get_winspect_data(sample_data_file)
self.assertEqual(len(expected_data_list), len(retrieved_data_list))
for data_array_idx in range(len(expected_data_list)):
self.assertTrue(
np.array_equal(expected_data_list[data_array_idx].data,
retrieved_data_list[data_array_idx].data))
def test_read_data(self):
"""Verify returning NumPy data from a file"""
sample_data_folder = os.path.join(pathfinder.app_path(), 'models',
'tests', 'support_files')
# Verify Winspect 6/7 retrieval
sample_winspect_file = os.path.join(sample_data_folder,
'sample_data.sdt')
winspect_adapter = self.create_adapter(sample_winspect_file)
winspect_adapter.read_data()
returned_winspect_data = dataio.get_winspect_data(sample_winspect_file)
for dataset in returned_winspect_data:
expected_winspect_data = dataset.data
retrieved_winspect_data = winspect_adapter.data[dataset.data_type +
"0"]
self.assertTrue(
np.array_equal(expected_winspect_data,
retrieved_winspect_data))
# Verify bitmap retrieval
sample_img_file = os.path.join(sample_data_folder,
'austin_sky320x240.jpg')
expected_img_data = dataio.get_img_data(sample_img_file)
img_adapter = self.create_adapter(sample_img_file)
img_adapter.read_data()
retrieved_img_data = img_adapter.data
self.assertTrue(np.array_equal(expected_img_data, retrieved_img_data))
# Verify UTWin retrieval
sample_utwin_file = os.path.join(sample_data_folder, 'CScanData.csc')
utwin_adapter = self.create_adapter(sample_utwin_file)
utwin_data = dataio.get_utwin_data(sample_utwin_file)
expected_utwin_data = {}
for data_type in utwin_data.keys():
for idx in range(len(utwin_data[data_type])):
expected_utwin_data[data_type +
str(idx)] = utwin_data[data_type][idx]
utwin_adapter.read_data()
retrieved_utwin_data = utwin_adapter.data
for dataset in expected_utwin_data:
if expected_utwin_data[dataset] is not None:
self.assertTrue(
np.array_equal(expected_utwin_data[dataset],
retrieved_utwin_data[dataset]))
def read_data(filename, filetype=None):
"""Attempts to import the specified file based on the provided filetype, or automatically guesses the file format
based on the file extension if no filetype is given. Returns the data as a NumPy array if successfully imported as
a NumPy array if the file contained a single dataset or as a dict if multiple datasets were found."""
tof_counter = 0
amp_counter = 0
waveform_counter = 0
data = {}
if filetype is None:
filetype = get_file_type(filename)
if filetype is not None and filetype in available_file_types():
if filetype == 'nditoolbox':
data = dataio.get_data(filename)
if filetype == 'winspect':
raw_data = dataio.get_winspect_data(filename)
# Handle any files that may have stored multiple datasets of
# a given type(s)
for dataset in raw_data:
dataset_key = os.path.basename(filename)
if dataset.data_type == 'waveform':
dataset_key = 'waveform' + str(waveform_counter)
waveform_counter +=1
elif dataset.data_type == 'amplitude':
dataset_key = 'amplitude' + str(amp_counter)
amp_counter += 1
elif dataset.data_type == 'tof': #TODO -confirm SDT files use tof
dataset_key = 'tof' + str(tof_counter)
tof_counter += 1
data[dataset_key] = dataset.data
if filetype == 'csv':
data = dataio.get_txt_data(filename)
if filetype == 'image':
data = dataio.get_img_data(filename, flatten=True)
if filetype == 'dicom':
data = dataio.get_dicom_data(filename)
if filetype == 'utwin':
raw_data = dataio.get_utwin_data(filename)
for k in raw_data.keys():
for idx in range(len(raw_data[k])):
data[k + str(idx)] = raw_data[k][idx]
return data
