class DatReaderTests(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.data_list = self.loader.load(find("detector_square.dat"))
# Same data file, but first 15 rows of data removed
self.data_list_rectangle = self.loader.load(
find("detector_rectangular.DAT"))
def _check_common_data(self, f, removed):
self.assertEqual(len(f.qx_data), BASE_POINTS - removed * BASE_LENGTH)
self.assertEqual(len(f.x_bins) + removed, len(f.y_bins))
self.assertEqual(len(f.x_bins), 192 - removed)
self.assertEqual(f.qx_data[len(f.qx_data) - 1], 0.2908819)
self.assertEqual(f.Q_unit, 'A^{-1}')
self.assertEqual(f.I_unit, 'cm^{-1}')
self.assertEqual(f.meta_data['loader'], "IGOR/DAT 2D Q_map")
def test_check_square_data(self):
"""
Test square detector data loaded by the .DAT reader
"""
f = self.data_list[0]
# The length of the data is 10
self.assertEqual(len(self.data_list), 1)
self.assertEqual(f.qx_data[0], -0.03573497)
self._check_common_data(f, 0)
def test_check_rectangular_data(self):
"""
Test rectangular detector data loaded by the .DAT reader
"""
f = self.data_list_rectangle[0]
self.assertEqual(len(self.data_list_rectangle), 1)
self.assertEqual(f.qx_data[0], -0.009160664)
self._check_common_data(f, 15)
class cansas_reader_hdf5(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.datafile_basic = "simpleexamplefile.h5"
self.datafile_multiplesasentry = "cansas_1Dand2D_samedatafile.h5"
self.datafile_multiplesasdata = "cansas_1Dand2D_samesasentry.h5"
self.datafile_multiplesasdata_multiplesasentry = "cansas_1Dand2D_multiplesasentry_multiplesasdata.h5"
def test_real_data(self):
self.data = self.loader.load(self.datafile_basic)
self._check_example_data(self.data[0])
def test_multiple_sasentries(self):
self.data = self.loader.load(self.datafile_multiplesasentry)
self.assertTrue(len(self.data) == 2)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
self._check_1d_data(self.data[0])
def _check_multiple_data(self, data):
self.assertTrue(data.title == "MH4_5deg_16T_SLOW")
self.assertTrue(data.run[0] == '33837')
self.assertTrue(len(data.run) == 1)
self.assertTrue(data.instrument == "SANS2D")
self.assertTrue(data.source.radiation == "Spallation Neutron Source")
self.assertTrue(len(data.detector) == 1)
self.assertTrue(data.detector[0].name == "rear-detector")
self.assertTrue(data.detector[0].distance == 4.385281)
self.assertTrue(data.detector[0].distance_unit == 'm')
self.assertTrue(len(data.trans_spectrum) == 1)
def _check_1d_data(self, data):
self.assertTrue(isinstance(data, Data1D))
self.assertTrue(len(data.x) == 66)
self.assertTrue(len(data.x) == len(data.y))
self.assertTrue(data.dy[10] == 0.20721350111248701)
self.assertTrue(data.y[10] == 24.193889608153476)
self.assertTrue(data.x[10] == 0.008981127988654792)
def _check_2d_data(self, data):
self.assertTrue(isinstance(data, Data2D))
self.assertTrue(len(data.x) == 66)
self.assertTrue(len(data.x) == len(data.y))
self.assertTrue(data.dy[10] == 0.20721350111248701)
self.assertTrue(data.y[10] == 24.193889608153476)
self.assertTrue(data.x[10] == 0.008981127988654792)
def _check_example_data(self, data):
self.assertTrue(data.title == "")
self.assertTrue(data.x.size == 100)
self.assertTrue(data._xunit == "A^{-1}")
self.assertTrue(data._yunit == "cm^{-1}")
self.assertTrue(data.y.size == 100)
self.assertAlmostEqual(data.y[9], 0.952749011516985)
self.assertAlmostEqual(data.x[9], 0.3834415188257777)
self.assertAlmostEqual(len(data.meta_data), 0)
class nxcansas_writer(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.writer = NXcanSASWriter()
self.read_file_1d = find("cansas1d.xml")
self.write_file_1d = find("export1d.h5")
self.read_file_2d = find("exp18_14_igor_2dqxqy.dat")
self.write_file_2d = find("export2d.h5")
self.data_1d = self.loader.load(self.read_file_1d)[0]
self.data_2d = self.loader.load(self.read_file_2d)[0]
self.data_2d.detector[0].name = ''
self.data_2d.source.radiation = 'neutron'
def test_write_1d(self):
self.writer.write([self.data_1d], self.write_file_1d)
data = self.loader.load(self.write_file_1d)
self.assertTrue(len(data) == 1)
data = data[0]
self.assertTrue(len(data.x) == len(self.data_1d.x))
self.assertTrue(len(data.y) == len(self.data_1d.y))
self.assertTrue(len(data.dy) == len(self.data_1d.dy))
self._check_metadata(data, self.data_1d)
def test_write_2d(self):
self.writer.write([self.data_2d], self.write_file_2d)
data = self.loader.load(self.write_file_2d)
self.assertTrue(len(data) == 1)
data = data[0]
self.assertTrue(len(data.data) == len(self.data_2d.data))
self.assertTrue(len(data.qx_data) == len(self.data_2d.qx_data))
self.assertTrue(len(data.qy_data) == len(self.data_2d.qy_data))
self._check_metadata(data, self.data_2d)
def _check_metadata(self, written, correct):
self.assertTrue(written.title == correct.title)
self.assertTrue(written.sample.name == correct.sample.name)
self.assertAlmostEqual(written.sample.thickness,
correct.sample.thickness)
self.assertAlmostEqual(written.sample.temperature,
correct.sample.temperature)
self.assertTrue(written.instrument == correct.instrument)
self.assertTrue(len(written.detector) == len(correct.detector))
for i in range(len(written.detector)):
written_det = written.detector[i]
correct_det = correct.detector[i]
self.assertAlmostEqual(written_det.distance, correct_det.distance)
self.assertTrue(written_det.name == correct_det.name)
self.assertTrue(written.source.radiation == correct.source.radiation)
def tearDown(self):
if os.path.isfile(self.write_file_1d):
os.remove(self.write_file_1d)
if os.path.isfile(self.write_file_2d):
os.remove(self.write_file_2d)
class nxcansas_writer(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.writer = NXcanSASWriter()
self.read_file_1d = "cansas1d.xml"
self.write_file_1d = "export1d.h5"
self.read_file_2d = "exp18_14_igor_2dqxqy.dat"
self.write_file_2d = "export2d.h5"
self.data_1d = self.loader.load(self.read_file_1d)[0]
self.data_2d = self.loader.load(self.read_file_2d)[0]
self.data_2d.detector[0].name = ''
self.data_2d.source.radiation = 'neutron'
def test_write_1d(self):
self.writer.write([self.data_1d], self.write_file_1d)
data = self.loader.load(self.write_file_1d)
self.assertTrue(len(data) == 1)
data = data[0]
self.assertTrue(len(data.x) == len(self.data_1d.x))
self.assertTrue(len(data.y) == len(self.data_1d.y))
self.assertTrue(len(data.dy) == len(self.data_1d.dy))
self._check_metadata(data, self.data_1d)
def test_write_2d(self):
self.writer.write([self.data_2d], self.write_file_2d)
data = self.loader.load(self.write_file_2d)
self.assertTrue(len(data) == 1)
data = data[0]
self.assertTrue(len(data.data) == len(self.data_2d.data))
self.assertTrue(len(data.qx_data) == len(self.data_2d.qx_data))
self.assertTrue(len(data.qy_data) == len(self.data_2d.qy_data))
self._check_metadata(data, self.data_2d)
def _check_metadata(self, written, correct):
self.assertTrue(written.title == correct.title)
self.assertTrue(written.sample.name == correct.sample.name)
self.assertAlmostEqual(written.sample.thickness, correct.sample.thickness)
self.assertAlmostEqual(written.sample.temperature, correct.sample.temperature)
self.assertTrue(written.instrument == correct.instrument)
self.assertTrue(len(written.detector) == len(correct.detector))
for i in range(len(written.detector)):
written_det = written.detector[i]
correct_det = correct.detector[i]
self.assertAlmostEqual(written_det.distance, correct_det.distance)
self.assertTrue(written_det.name == correct_det.name)
self.assertTrue(written.source.radiation == correct.source.radiation)
def tearDown(self):
if os.path.isfile(self.write_file_1d):
os.remove(self.write_file_1d)
if os.path.isfile(self.write_file_2d):
os.remove(self.write_file_2d)
def test_old_cansas_files(self):
reader1 = XMLreader(self.cansas1d, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
file_loader = Loader()
file_loader.load(self.cansas1d)
reader2 = XMLreader(self.cansas1d_units, self.schema_1_0)
self.assertTrue(reader2.validate_xml())
reader3 = XMLreader(self.cansas1d_badunits, self.schema_1_0)
self.assertTrue(reader3.validate_xml())
reader4 = XMLreader(self.cansas1d_slit, self.schema_1_0)
self.assertTrue(reader4.validate_xml())
def test_old_cansas_files(self):
reader1 = XMLreader(self.cansas1d, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
file_loader = Loader()
file_loader.load(self.cansas1d)
reader2 = XMLreader(self.cansas1d_units, self.schema_1_0)
self.assertTrue(reader2.validate_xml())
reader3 = XMLreader(self.cansas1d_badunits, self.schema_1_0)
self.assertTrue(reader3.validate_xml())
reader4 = XMLreader(self.cansas1d_slit, self.schema_1_0)
self.assertTrue(reader4.validate_xml())
def test_double_trans_spectra(self):
xmlreader = XMLreader(self.isis_1_1_doubletrans, self.schema_1_1)
self.assertTrue(xmlreader.validate_xml())
reader = Loader()
data = reader.load(self.isis_1_1_doubletrans)
for item in data:
self._check_data(item)
def test_double_trans_spectra(self):
xmlreader = XMLreader(self.isis_1_1_doubletrans, self.schema_1_1)
self.assertTrue(xmlreader.validate_xml())
reader = Loader()
data = reader.load(self.isis_1_1_doubletrans)
for item in data:
self._check_data(item)
def load_data(filename):
# type: (str) -> Data
"""
Load data using a sasview loader.
"""
from sas.sascalc.dataloader.loader import Loader # type: ignore
loader = Loader()
# Allow for one part in multipart file
if '[' in filename:
filename, indexstr = filename[:-1].split('[')
index = int(indexstr)
else:
index = None
datasets = loader.load(filename)
if datasets is None:
raise IOError("Data %r could not be loaded" % filename)
if not isinstance(datasets, list):
datasets = [datasets]
if index is None and len(datasets) > 1:
raise ValueError("Need to specify filename[index] for multipart data")
data = datasets[index if index is not None else 0]
if hasattr(data, 'x'):
data.qmin, data.qmax = data.x.min(), data.x.max()
data.mask = (np.isnan(data.y) if data.y is not None else np.zeros_like(
data.x, dtype='bool'))
elif hasattr(data, 'qx_data'):
data.mask = ~data.mask
return data
def load_data(filename):
"""
Load data using a sasview loader.
"""
from sas.sascalc.dataloader.loader import Loader
loader = Loader()
data = loader.load(filename)
if data is None:
raise IOError("Data %r could not be loaded" % filename)
return data
def load_data(filename):
# type: (str) -> Data
"""
Load data using a sasview loader.
"""
from sas.sascalc.dataloader.loader import Loader # type: ignore
loader = Loader()
data = loader.load(filename)
if data is None:
raise IOError("Data %r could not be loaded" % filename)
return data
class testLoader(unittest.TestCase):
def setUp(self):
self.L = Loader()
self.L.find_plugins('../plugins')
def testplugin(self):
"""
test loading with a test reader only
found in the plugins directory
"""
output = self.L.load('test_data.test')
self.assertEqual(output.x[0], 1234.)
def test_cansas_xml(self):
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
class testLoader(unittest.TestCase):
def setUp(self):
self.L=Loader()
self.L.find_plugins('../plugins')
def testplugin(self):
"""
test loading with a test reader only
found in the plugins directory
"""
output = self.L.load('test_data.test')
self.assertEqual(output.x[0], 1234.)
def test_cansas_xml(self):
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
def test_cansas_xml(self):
filename = "isis_1_1_write_test.xml"
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
fo = open(self.isis_1_1)
str = fo.read()
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(filename, dataloader[i], None)
fo = open(filename)
str = fo.read()
reader2 = Loader()
return_data = reader2.load(filename)
written_data = return_data[0]
self._check_data(written_data)
def test_cansas_xml(self):
filename = "isis_1_1_write_test.xml"
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
fo = open(self.isis_1_1)
str = fo.read()
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(filename, dataloader[i], None)
fo = open(filename)
str = fo.read()
reader2 = Loader()
return_data = reader2.load(filename)
written_data = return_data[0]
self._check_data(written_data)
class abs_reader(unittest.TestCase):
def setUp(self):
self.loader = Loader()
def test_checkdata(self):
"""
Test .DAT file loaded as IGOR/DAT 2D Q_map
"""
f = self.loader.load("exp18_14_igor_2dqxqy.dat")
# The length of the data is 10
self.assertEqual(len(f.qx_data), 36864)
self.assertEqual(f.qx_data[0], -0.03573497)
self.assertEqual(f.qx_data[36863], 0.2908819)
self.assertEqual(f.Q_unit, '1/A')
self.assertEqual(f.I_unit, '1/cm')
self.assertEqual(f.meta_data['loader'], "IGOR/DAT 2D Q_map")
class abs_reader(unittest.TestCase):
def setUp(self):
self.loader = Loader()
def test_checkdata(self):
"""
Test .DAT file loaded as IGOR/DAT 2D Q_map
"""
f = self.loader.load("exp18_14_igor_2dqxqy.dat")
# The length of the data is 10
self.assertEqual(len(f.qx_data), 36864)
self.assertEqual(f.qx_data[0],-0.03573497)
self.assertEqual(f.qx_data[36863],0.2908819)
self.assertEqual(f.Q_unit, '1/A')
self.assertEqual(f.I_unit, '1/cm')
self.assertEqual(f.meta_data['loader'],"IGOR/DAT 2D Q_map")
def test_save_cansas_v1_0(self):
filename = "isis_1_0_write_test.xml"
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
valid = xmlreader.validate_xml()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(filename, dataloader[i], None)
reader2 = Reader()
return_data = reader2.read(filename)
written_data = return_data[0]
xmlwrite = XMLreader(filename, self.schema_1_0)
valid = xmlreader.validate_xml()
self.assertTrue(valid)
self._check_data(written_data)
def test_cansas_xml(self):
self.xml_reader.set_xml_file(self.isis_1_1)
self.xml_reader.set_schema(self.schema_1_1)
self.xml_reader.set_processing_instructions()
self.assertTrue(self.xml_reader.validate_xml())
dataloader = self.loader.load(self.isis_1_1)
cansasreader = self.cansas_reader.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
self.loader.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
def test_save_cansas_v1_0(self):
filename = "isis_1_0_write_test.xml"
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
valid = xmlreader.validate_xml()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(filename, dataloader[i], None)
reader2 = Reader()
return_data = reader2.read(filename)
written_data = return_data[0]
xmlwrite = XMLreader(filename, self.schema_1_0)
valid = xmlreader.validate_xml()
self.assertTrue(valid)
self._check_data(written_data)
def test_save_cansas_v1_0(self):
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(self.write_1_0_filename, dataloader[i], None)
reader2 = Reader()
self.assertTrue(os.path.isfile(self.write_1_0_filename))
return_data = reader2.read(self.write_1_0_filename)
written_data = return_data[0]
xmlreader = XMLreader(self.write_1_0_filename, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
self._check_data(written_data)
if os.path.isfile(self.write_1_0_filename):
os.remove(self.write_1_0_filename)
def test_save_cansas_v1_0(self):
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(self.write_1_0_filename, dataloader[i], None)
reader2 = Reader()
self.assertTrue(os.path.isfile(self.write_1_0_filename))
return_data = reader2.read(self.write_1_0_filename)
written_data = return_data[0]
xmlreader = XMLreader(self.write_1_0_filename, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
self._check_data(written_data)
if os.path.isfile(self.write_1_0_filename):
os.remove(self.write_1_0_filename)
class DataReader(CalcThread):
"""
Load a data given a filename
"""
def __init__(self, path, completefn=None,
updatefn=None,
yieldtime=0.01,
worktime=0.01
):
CalcThread.__init__(self, completefn,
updatefn,
yieldtime,
worktime)
self.path = path
#Instantiate a loader
self.loader = Loader()
self.starttime = 0
def isquit(self):
"""
@raise KeyboardInterrupt: when the thread is interrupted
"""
try:
CalcThread.isquit(self)
except KeyboardInterrupt:
raise KeyboardInterrupt
def compute(self):
"""
read some data
"""
self.starttime = time.time()
try:
data = self.loader.load(self.path)
self.complete(data=data)
except KeyboardInterrupt:
# Thread was interrupted, just proceed and re-raise.
# Real code should not print, but this is an example...
raise
def OnLoad(self,event):
path = None
dlg = wx.FileDialog(None, "Choose a file", os.getcwd(), "", "*", wx.OPEN)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
mypath = os.path.basename(path)
from sas.sascalc.dataloader.loader import Loader
#Instantiate a loader
L=Loader()
#Receives data
try:
output=L.load(path)
except:
raise
newplot=Data2D(image=output.data)
newplot.setValues(output)
self.plotpanel.add_image(newplot)
dlg.Destroy()
def OnLoad(self, event):
path = None
dlg = wx.FileDialog(None, "Choose a file", os.getcwd(), "", "*",
wx.OPEN)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
mypath = os.path.basename(path)
from sas.sascalc.dataloader.loader import Loader
#Instantiate a loader
L = Loader()
#Receives data
try:
output = L.load(path)
except:
raise
newplot = Data2D(image=output.data)
newplot.setValues(output)
self.plotpanel.add_image(newplot)
dlg.Destroy()
class DataReader(CalcThread):
"""
Load a data given a filename
"""
def __init__(self,
path,
completefn=None,
updatefn=None,
yieldtime=0.01,
worktime=0.01):
CalcThread.__init__(self, completefn, updatefn, yieldtime, worktime)
self.path = path
#Instantiate a loader
self.loader = Loader()
self.starttime = 0
def isquit(self):
"""
@raise KeyboardInterrupt: when the thread is interrupted
"""
try:
CalcThread.isquit(self)
except KeyboardInterrupt:
raise KeyboardInterrupt
def compute(self):
"""
read some data
"""
self.starttime = time.time()
try:
data = self.loader.load(self.path)
self.complete(data=data)
except KeyboardInterrupt:
# Thread was interrupted, just proceed and re-raise.
# Real code should not print, but this is an example...
raise
def load_data(filename, index=0):
# type: (str) -> Data
"""
Load data using a sasview loader.
"""
from sas.sascalc.dataloader.loader import Loader # type: ignore
loader = Loader()
# Allow for one part in multipart file
if '[' in filename:
filename, indexstr = filename[:-1].split('[')
index = int(indexstr)
datasets = loader.load(filename)
if not datasets: # None or []
raise IOError("Data %r could not be loaded" % filename)
if not isinstance(datasets, list):
datasets = [datasets]
for data in datasets:
if hasattr(data, 'x'):
data.qmin, data.qmax = data.x.min(), data.x.max()
data.mask = (np.isnan(data.y) if data.y is not None else
np.zeros_like(data.x, dtype='bool'))
elif hasattr(data, 'qx_data'):
data.mask = ~data.mask
return datasets[index] if index != 'all' else datasets
class Plugin(PluginBase):
def __init__(self):
PluginBase.__init__(self, name="DataLoader")
# Default location
self._default_save_location = DEFAULT_OPEN_FOLDER
self.loader = Loader()
self._data_menu = None
def populate_file_menu(self):
"""
get a menu item and append it under file menu of the application
add load file menu item and load folder item
"""
# menu for data files
menu_list = []
data_file_hint = "load one or more data in the application"
menu_list = [('&Load Data File(s)', data_file_hint, self.load_data)]
gui_style = self.parent.get_style()
style = gui_style & GUIFRAME.MULTIPLE_APPLICATIONS
style1 = gui_style & GUIFRAME.DATALOADER_ON
if style == GUIFRAME.MULTIPLE_APPLICATIONS:
# menu for data from folder
data_folder_hint = "load multiple data in the application"
menu_list.append(('&Load Data Folder', data_folder_hint,
self._load_folder))
return menu_list
def load_data(self, event):
"""
Load data
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location == None:
self._default_save_location = os.getcwd()
cards = self.loader.get_wildcards()
temp = [APPLICATION_WLIST] + PLUGINS_WLIST
for item in temp:
if item in cards:
cards.remove(item)
wlist = '|'.join(cards)
style = wx.OPEN | wx.FD_MULTIPLE
dlg = wx.FileDialog(self.parent,
"Choose a file",
self._default_save_location, "",
wlist,
style=style)
if dlg.ShowModal() == wx.ID_OK:
file_list = dlg.GetPaths()
if len(file_list) >= 0 and not file_list[0] is None:
self._default_save_location = os.path.dirname(file_list[0])
path = self._default_save_location
dlg.Destroy()
if path is None or not file_list or file_list[0] is None:
return
self.parent._default_save_location = self._default_save_location
self.get_data(file_list)
def can_load_data(self):
"""
if return True, then call handler to laod data
"""
return True
def _load_folder(self, event):
"""
Load entire folder
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location == None:
self._default_save_location = os.getcwd()
dlg = wx.DirDialog(self.parent, "Choose a directory",
self._default_save_location,
style=wx.DD_DEFAULT_STYLE)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
self._default_save_location = path
dlg.Destroy()
if path is not None:
self._default_save_location = os.path.dirname(path)
else:
return
file_list = self.get_file_path(path)
self.get_data(file_list)
self.parent._default_save_location = self._default_save_location
def load_error(self, error=None):
"""
Pop up an error message.
:param error: details error message to be displayed
"""
if error is not None or str(error).strip() != "":
dial = wx.MessageDialog(self.parent, str(error), 'Error Loading File',
wx.OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
def get_file_path(self, path):
"""
Receive a list containing folder then return a list of file
"""
if os.path.isdir(path):
return [os.path.join(os.path.abspath(path), filename) for filename in os.listdir(path)]
def _process_data_and_errors(self, item, p_file, output, message):
"""
Check to see if data set loaded with any errors. If so, append to
error message to be sure user knows the issue.
"""
data_error = False
if hasattr(item, 'errors'):
for error_data in item.errors:
data_error = True
message += "\tError: {0}\n".format(error_data)
else:
logging.error("Loader returned an invalid object:\n %s" % str(item))
data_error = True
data = self.parent.create_gui_data(item, p_file)
output[data.id] = data
return output, message, data_error
def get_data(self, path, format=None):
"""
"""
message = ""
log_msg = ''
output = {}
any_error = False
data_error = False
error_message = ""
for p_file in path:
info = "info"
basename = os.path.basename(p_file)
_, extension = os.path.splitext(basename)
if extension.lower() in EXTENSIONS:
any_error = True
log_msg = "Data Loader cannot "
log_msg += "load: %s\n" % str(p_file)
log_msg += """Please try to open that file from "open project" """
log_msg += """or "open analysis" menu\n"""
error_message = log_msg + "\n"
logging.info(log_msg)
continue
try:
message = "Loading Data... " + str(p_file) + "\n"
self.load_update(output=output, message=message, info=info)
temp = self.loader.load(p_file, format)
if temp.__class__.__name__ == "list":
for item in temp:
output, error_message, data_error = \
self._process_data_and_errors(item,
p_file,
output,
error_message)
else:
output, error_message, data_error = \
self._process_data_and_errors(temp,
p_file,
output,
error_message)
except:
logging.error(sys.exc_value)
any_error = True
if any_error or error_message != "":
if error_message == "":
error = "Error: " + str(sys.exc_info()[1]) + "\n"
error += "while loading Data: \n%s\n" % str(basename)
error_message += "The data file you selected could not be loaded.\n"
error_message += "Make sure the content of your file"
error_message += " is properly formatted.\n\n"
error_message += "When contacting the SasView team, mention the"
error_message += " following:\n%s" % str(error)
elif data_error:
base_message = "Errors occurred while loading "
base_message += "{0}\n".format(basename)
base_message += "The data file loaded but with errors.\n"
error_message = base_message + error_message
else:
error_message += "%s\n" % str(p_file)
info = "error"
if any_error or error_message:
self.load_update(output=output, message=error_message, info=info)
else:
message = "Loading Data Complete! "
message += log_msg
self.load_complete(output=output, error_message=error_message,
message=message, path=path, info='info')
def load_update(self, output=None, message="", info="warning"):
"""
print update on the status bar
"""
if message != "":
wx.PostEvent(self.parent, StatusEvent(status=message, info=info,
type="progress"))
def load_complete(self, output, message="", error_message="", path=None,
info="warning"):
"""
post message to status bar and return list of data
"""
wx.PostEvent(self.parent, StatusEvent(status=message,
info=info,
type="stop"))
# if error_message != "":
# self.load_error(error_message)
self.parent.add_data(data_list=output)
class testLoader(unittest.TestCase):
logging.debug("Inside testLoad module")
""" test fitting """
def setUp(self):
"""
Set up the initial conditions before _each_ test
so that they all start from the same well-defined state.
"""
#Creating a loader
self.L = Loader()
def testLoad0(self):
"""test reading empty file"""
self.assertRaises(RuntimeError, self.L.load, 'empty.txt')
def testLoad1(self):
"""test reading 2 columns"""
#Testing loading a txt file of 2 columns, the only reader should be read1
output = self.L.load('test_2_columns.txt')
x = [
2.83954, 0.204082, 0.408163, 0.612245, 0.816327, 1.02041, 1.22449,
1.42857, 1.63265
]
y = [
0.6, 3.44938, 5.82026, 5.27591, 5.2781, 5.22531, 7.47487, 7.85852,
10.2278
]
dx = []
dy = []
self.assertEqual(len(output.x), len(x))
self.assertEqual(len(output.y), len(y))
for i in range(len(x)):
self.assertEqual(output.x[i], x[i])
self.assertEqual(output.y[i], y[i])
def testLoad2(self):
"""Testing loading a txt file of 3 columns"""
output = self.L.load('test_3_columns.txt')
x = [0, 0.204082, 0.408163, 0.612245, 0.816327, 1.02041, 1.22449]
y = [2.83954, 3.44938, 5.82026, 5.27591, 5.2781, 5.22531, 7.47487]
dx = []
dy = [0.6, 0.676531, 0.753061, 0.829592, 0.906122, 0.982653, 1.05918]
self.assertEqual(len(output.x), len(x))
self.assertEqual(len(output.y), len(y))
self.assertEqual(len(output.dy), len(dy))
for i in range(len(x)):
self.assertEqual(output.x[i], x[i])
self.assertEqual(output.y[i], y[i])
self.assertEqual(output.dy[i], dy[i])
def testLoad2_uppercase(self):
"""Testing loading a txt file of 3 columns"""
output = self.L.load('test_3_columns.TXT')
x = [0, 0.204082, 0.408163, 0.612245, 0.816327, 1.02041, 1.22449]
y = [2.83954, 3.44938, 5.82026, 5.27591, 5.2781, 5.22531, 7.47487]
dx = []
dy = [0.6, 0.676531, 0.753061, 0.829592, 0.906122, 0.982653, 1.05918]
self.assertEqual(len(output.x), len(x))
self.assertEqual(len(output.y), len(y))
self.assertEqual(len(output.dy), len(dy))
for i in range(len(x)):
self.assertEqual(output.x[i], x[i])
self.assertEqual(output.y[i], y[i])
self.assertEqual(output.dy[i], dy[i])
def testload3(self):
""" Testing loading Igor data"""
#tested good file.asc
output = self.L.load('MAR07232_rest.ASC')
self.assertEqual(output.xmin, -0.018558945804750416)
self.assertEqual(
output.xmax,
0.016234058202440633,
)
self.assertEqual(output.ymin, -0.01684257151702391)
self.assertEqual(output.ymax, 0.017950440578015116)
#tested corrupted file.asc
try:
self.L.load('AR07232_rest.ASC')
except ValueError, msg:
#logging.log(10,str(msg))
logging.error(str(msg))
class Plugin(PluginBase):
def __init__(self):
PluginBase.__init__(self, name="DataLoader")
# Default location
self._default_save_location = DEFAULT_OPEN_FOLDER
self.loader = Loader()
self._data_menu = None
def populate_file_menu(self):
"""
get a menu item and append it under file menu of the application
add load file menu item and load folder item
"""
# menu for data files
data_file_hint = "load one or more data in the application"
menu_list = [('&Load Data File(s)', data_file_hint, self.load_data)]
gui_style = self.parent.get_style()
style = gui_style & GUIFRAME.MULTIPLE_APPLICATIONS
if style == GUIFRAME.MULTIPLE_APPLICATIONS:
# menu for data from folder
data_folder_hint = "load multiple data in the application"
menu_list.append(
('&Load Data Folder', data_folder_hint, self._load_folder))
return menu_list
def load_data(self, event):
"""
Load data
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location is None:
self._default_save_location = os.getcwd()
cards = self.loader.get_wildcards()
temp = [APPLICATION_WLIST] + PLUGINS_WLIST
for item in temp:
if item in cards:
cards.remove(item)
wlist = '|'.join(cards)
style = wx.OPEN | wx.FD_MULTIPLE
dlg = wx.FileDialog(self.parent,
"Choose a file",
self._default_save_location,
"",
wlist,
style=style)
if dlg.ShowModal() == wx.ID_OK:
file_list = dlg.GetPaths()
if len(file_list) >= 0 and file_list[0] is not None:
self._default_save_location = os.path.dirname(file_list[0])
path = self._default_save_location
dlg.Destroy()
if path is None or not file_list or file_list[0] is None:
return
self.parent._default_save_location = self._default_save_location
self.get_data(file_list)
def can_load_data(self):
"""
if return True, then call handler to load data
"""
return True
def _load_folder(self, event):
"""
Load entire folder
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location is None:
self._default_save_location = os.getcwd()
dlg = wx.DirDialog(self.parent,
"Choose a directory",
self._default_save_location,
style=wx.DD_DEFAULT_STYLE)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
self._default_save_location = path
dlg.Destroy()
if path is not None:
self._default_save_location = os.path.dirname(path)
else:
return
file_list = self.get_file_path(path)
self.get_data(file_list)
self.parent._default_save_location = self._default_save_location
def load_error(self, error=None):
"""
Pop up an error message.
:param error: details error message to be displayed
"""
if error is not None or str(error).strip() != "":
dial = wx.MessageDialog(self.parent, str(error),
'Error Loading File',
wx.OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
def get_file_path(self, path):
"""
Receive a list containing folder then return a list of file
"""
if os.path.isdir(path):
return [
os.path.join(os.path.abspath(path), filename)
for filename in os.listdir(path)
]
def _process_data_and_errors(self, item, p_file, output, message):
"""
Check to see if data set loaded with any errors. If so, append to
error message to be sure user knows the issue.
"""
data_error = False
if hasattr(item, 'errors'):
for error_data in item.errors:
data_error = True
message += "\tError: {0}\n".format(error_data)
else:
logger.error("Loader returned an invalid object:\n %s" % str(item))
data_error = True
data = self.parent.create_gui_data(item, p_file)
output[data.id] = data
return output, message, data_error
def get_data(self, path, format=None):
"""
"""
file_errors = {}
output = {}
exception_occurred = False
for p_file in path:
basename = os.path.basename(p_file)
# Skip files that start with a period
if basename.startswith("."):
msg = "The folder included a potential hidden file - %s." \
% basename
msg += " Do you wish to load this file as data?"
msg_box = wx.MessageDialog(None, msg, 'Warning',
wx.OK | wx.CANCEL)
if msg_box.ShowModal() == wx.ID_CANCEL:
continue
_, extension = os.path.splitext(basename)
if extension.lower() in EXTENSIONS:
log_msg = "Data Loader cannot "
log_msg += "load: {}\n".format(str(p_file))
log_msg += "Please try to open that file from \"open project\""
log_msg += "or \"open analysis\" menu."
logger.info(log_msg)
file_errors[basename] = [log_msg]
continue
try:
message = "Loading {}...\n".format(p_file)
self.load_update(message=message, info="info")
temp = self.loader.load(p_file, format)
if not isinstance(temp, list):
temp = [temp]
for item in temp:
error_message = ""
output, error_message, data_error = \
self._process_data_and_errors(item,
p_file,
output,
error_message)
if data_error:
if basename in file_errors.keys():
file_errors[basename] += [error_message]
else:
file_errors[basename] = [error_message]
self.load_update(message="Loaded {}\n".format(p_file),
info="info")
except NoKnownLoaderException as e:
exception_occurred = True
error_message = "Loading data failed!\n" + e.message
self.load_complete(output=None,
message=error_message,
info="warning")
except Exception as e:
exception_occurred = True
file_err = "The Data file you selected could not be "
file_err += "loaded.\nMake sure the content of your file"
file_err += " is properly formatted.\n"
file_err += "When contacting the SasView team, mention the"
file_err += " following:\n"
file_err += e.message
self.load_complete(output=None, message=file_err, info="error")
if len(file_errors) > 0:
error_message = ""
for filename, error_array in file_errors.iteritems():
error_message += "The following issues were found whilst "
error_message += "loading {}:\n".format(filename)
for message in error_array:
error_message += message + "\n"
error_message = error_message[:-1]
self.load_complete(output=output,
message=error_message,
info="error")
elif not exception_occurred: # Everything loaded as expected
self.load_complete(output=output,
message="Loading data complete!",
info="info")
else:
self.load_complete(output=None,
message=error_message,
info="error")
def load_update(self, message="", info="warning"):
"""
print update on the status bar
"""
if message != "":
wx.PostEvent(
self.parent,
StatusEvent(status=message, info=info, type="progress"))
def load_complete(self, output, message="", info="warning"):
"""
post message to status bar and return list of data
"""
wx.PostEvent(self.parent,
StatusEvent(status=message, info=info, type="stop"))
if output is not None:
self.parent.add_data(data_list=output)
class ABSReaderTests(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.f1_list = self.loader.load(find("ascii_test_1.txt"))
self.f1 = self.f1_list[0]
self.f2_list = self.loader.load(find("ascii_test_2.txt"))
self.f2 = self.f2_list[0]
self.f3_list = self.loader.load(find("ascii_test_3.txt"))
self.f3 = self.f3_list[0]
self.f4_list = self.loader.load(find("ascii_test_4.abs"))
self.f4 = self.f4_list[0]
self.f5_list = self.loader.load(find("ascii_test_5.txt"))
self.f5 = self.f5_list[0]
def test_checkdata(self):
"""
Test .ABS file loaded as ascii
"""
# The length of the data is 10
self.assertEqual(len(self.f1_list), 1)
self.assertEqual(len(self.f2_list), 1)
self.assertEqual(len(self.f3_list), 1)
self.assertEqual(len(self.f4_list), 1)
self.assertEqual(len(self.f5_list), 1)
self.assertEqual(len(self.f1.x), 10)
self.assertEqual(self.f1.x[0],0.002618)
self.assertEqual(self.f1.x[9],0.0497)
self.assertTrue(self.f1.x_unit == 'A^{-1}')
self.assertTrue(self.f1.y_unit == 'cm^{-1}')
self.assertEqual(self.f1.meta_data['loader'],"ASCII")
def test_truncated_1(self):
"""
Test an ascii file with header and a
comment line in the middle of the data section.
The business rule says that we should stop
reading at the first comment once the data
section has started (and treat the comment
as though it were the start of a footer).
"""
# The length of the data is 5
self.assertEqual(len(self.f2.x), 5)
self.assertEqual(self.f2.x[0],0.002618)
self.assertEqual(self.f2.x[4],0.02356)
def test_truncated_2(self):
"""
Test a 6-col ascii file with header and a
line with only 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the first inconsitent line.
"""
# The length of the data is 5
self.assertEqual(len(self.f3.x), 5)
self.assertEqual(self.f3.x[0],0.002618)
self.assertEqual(self.f3.x[4],0.02356)
def test_truncated_3(self):
"""
Test a 6-col ascii file with complex header and
many lines with 2 or 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the last line of header.
"""
# The length of the data is 5
self.assertEqual(len(self.f4.x), 5)
self.assertEqual(self.f4.x[0],0.012654)
self.assertEqual(self.f4.x[4],0.02654)
def test_truncated_4(self):
"""
Test mix of 6-col and 2-col.
Only the last 5 2-col lines should be read.
"""
# The length of the data is 5
self.assertEqual(len(self.f5.x), 5)
self.assertEqual(self.f5.x[0],0.02879)
self.assertEqual(self.f5.x[4],0.0497)
def test_truncated_5(self):
"""
Test a 6-col ascii file with complex header where one of them has a
letter and many lines with 2 or 2 columns in the middle of the data
section. Will be rejected because fewer than 5 lines.
"""
# Test .ABS file loaded as ascii
f = None
try:
f = self.loader.load(find("ascii_test_6.txt"))
# The length of the data is 5
except:
self.assertEqual(f, None)
def test_nan_values(self):
"""
Test loading an ascii data file with nan values saved in x, y, or dy.
"""
f_1d = self.loader.load(find("nans_in_1d_data.dat"))[0]
f_2d = self.loader.load(find("nans_in_2d_data.DAT"))[0]
for i in range(0, len(f_1d.x) - 1):
self.assertFalse(math.isnan(f_1d.x[i]))
self.assertFalse(math.isnan(f_1d.y[i]))
self.assertFalse(math.isnan(f_1d.dy[i]))
self.assertTrue(isinstance(f_2d, Data2D))
f_2d.data = f_2d.data.flatten()
f_2d.qx_data = f_2d.qx_data.flatten()
f_2d.qy_data = f_2d.qy_data.flatten()
for i in range(0, len(f_2d.data) - 1):
self.assertFalse(math.isnan(f_2d.data[i]))
self.assertFalse(math.isnan(f_2d.qx_data[i]))
self.assertFalse(math.isnan(f_2d.qy_data[i]))
def sesans_fit(file, model, initial_vals={}, custom_params={}, param_range=[],
acceptance_angle=None):
"""
@param file: SESANS file location
@param model: Bumps model object or model name - can be model, model_1 * model_2, and/or model_1 + model_2
@param initial_vals: dictionary of {param_name : initial_value}
@param custom_params: dictionary of {custom_parameter_name : Parameter() object}
@param param_range: dictionary of {parameter_name : [minimum, maximum]}
@param constraints: dictionary of {parameter_name : constraint}
@return: FitProblem for Bumps usage
"""
logging.basicConfig()
initial_vals['background'] = 0.0
try:
loader = Loader()
data = loader.load(file)[0]
if data is None:
raise IOError("Could not load file %r"%(file))
except Exception:
raise
# If no loadable data file, generate random data
SElength = np.linspace(0, 2400, 61) # [A]
data = np.ones_like(SElength)
err_data = np.ones_like(SElength)*0.03
class Sample:
zacceptance = 0.1 # [A^-1]
thickness = 0.2 # [cm]
class SESANSData1D:
#q_zmax = 0.23 # [A^-1]
lam = 0.2 # [nm]
x = SElength
y = data
dy = err_data
sample = Sample()
has_no_finite_acceptance = acceptance_angle is not None
data = SESANSData1D()
data.acceptance_angle = acceptance_angle
data.has_no_finite_acceptance = acceptance_angle is not None
radius = initial_vals.get("radius", 1000)
data.Rmax = 30*radius # [A]
if isinstance(model, str):
model = get_bumps_model(model)
# Load custom parameters, initial values and parameter ranges
pars = model.parameters().copy()
pars.update(custom_params)
for k, v in custom_params.items():
if k not in pars:
par_error("Can't set parameter %r in model"%k, pars)
setattr(model, k, v)
model._parameter_names.append(k)
for k, v in initial_vals.items():
if k not in pars:
par_error("Can't get parameter %r from model"%k, pars)
param = pars[k]
param.value = v
for k, v in param_range.items():
if k not in pars:
par_error("Can't set range on parameter %r in model"%k, pars)
param = pars[k]
param.range(*v)
if False: # for future implementation
M_sesans = bumps_model.Experiment(data=data, model=model)
M_sans = bumps_model.Experiment(data=sans_data, model=model)
problem = FitProblem([M_sesans, M_sans])
else:
M_sesans = bumps_model.Experiment(data=data, model=model)
problem = FitProblem(M_sesans)
return problem
class cansas_reader_hdf5(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.datafile_basic = find("simpleexamplefile.h5")
self.datafile_multiplesasentry = find(
"test_data" + os.sep + "nxcansas_1Dand2D_multisasentry.h5")
self.datafile_multiplesasdata = find(
"test_data" + os.sep + "nxcansas_1Dand2D_multisasdata.h5")
self.datafile_multiplesasdata_multiplesasentry = find(
"test_data" + os.sep + "nxcansas_1Dand2D_multisasentry_multisasdata.h5")
def test_real_data(self):
self.data = self.loader.load(self.datafile_basic)
self._check_example_data(self.data[0])
def test_multiple_sasentries(self):
self.data = self.loader.load(self.datafile_multiplesasentry)
self.assertTrue(len(self.data) == 2)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
if isinstance(self.data[0], Data1D):
self._check_1d_data(self.data[0])
self._check_2d_data(self.data[1])
else:
self._check_1d_data(self.data[1])
self._check_2d_data(self.data[0])
def test_multiple_sasdatas(self):
self.data = self.loader.load(self.datafile_multiplesasdata)
self.assertTrue(len(self.data) == 2)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
if isinstance(self.data[0], Data1D):
self._check_1d_data(self.data[0])
self._check_2d_data(self.data[1])
else:
self._check_1d_data(self.data[1])
self._check_2d_data(self.data[0])
def test_multiple_sasentries_multiplesasdatas(self):
self.data = self.loader.load(
self.datafile_multiplesasdata_multiplesasentry)
self.assertTrue(len(self.data) == 4)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
self._check_multiple_data(self.data[2])
self._check_multiple_data(self.data[3])
for data in self.data:
if isinstance(data, Data1D):
self._check_1d_data(data)
else:
self._check_2d_data(data)
def _check_multiple_data(self, data):
self.assertEqual(data.title, "MH4_5deg_16T_SLOW")
self.assertEqual(data.run[0], '33837')
self.assertEqual(len(data.run), 1)
self.assertEqual(data.instrument, "SANS2D")
self.assertEqual(data.source.radiation, "Spallation Neutron Source")
self.assertEqual(len(data.detector), 2)
self.assertTrue(data.detector[0].name == "rear-detector"
or data.detector[1].name == "rear-detector")
self.assertTrue(data.detector[0].name == "front-detector"
or data.detector[1].name == "front-detector")
self.assertAlmostEqual(data.detector[0].distance +
data.detector[1].distance, 7230.54, 2)
self.assertEqual(data.detector[0].distance_unit, 'mm')
self.assertEqual(len(data.trans_spectrum), 1)
def _check_1d_data(self, data):
self.assertEqual(len(data.x), 66)
self.assertEqual(len(data.x), len(data.y))
self.assertAlmostEqual(data.dy[10], 0.207214)
self.assertAlmostEqual(data.y[10], 24.1939)
self.assertAlmostEqual(data.x[10], 0.00898113)
def _check_2d_data(self, data):
self.assertTrue(isinstance(data, Data2D))
self.assertEqual(len(data.q_data), 150*150)
self.assertEqual(len(data.q_data), len(data.data))
self.assertAlmostEqual(data.err_data[10], 0.186723989418)
self.assertAlmostEqual(data.data[10], 0.465181)
self.assertAlmostEqual(data.qx_data[10], -0.129)
self.assertAlmostEqual(data.qy_data[10], -0.149)
def _check_example_data(self, data):
self.assertEqual(data.title, "")
self.assertEqual(data.x.size, 100)
self.assertEqual(data._xunit, "A^{-1}")
self.assertEqual(data._yunit, "cm^{-1}")
self.assertEqual(data.y.size, 100)
self.assertAlmostEqual(data.y[40], 0.952749011516985)
self.assertAlmostEqual(data.x[40], 0.3834415188257777)
self.assertAlmostEqual(len(data.meta_data), 0)
class ABSReaderTests(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.f1_list = self.loader.load(find("ascii_test_1.txt"))
self.f1 = self.f1_list[0]
self.f2_list = self.loader.load(find("ascii_test_2.txt"))
self.f2 = self.f2_list[0]
self.f3_list = self.loader.load(find("ascii_test_3.txt"))
self.f3 = self.f3_list[0]
self.f4_list = self.loader.load(find("ascii_test_4.abs"))
self.f4 = self.f4_list[0]
self.f5_list = self.loader.load(find("ascii_test_5.txt"))
self.f5 = self.f5_list[0]
def test_checkdata(self):
"""
Test .ABS file loaded as ascii
"""
# The length of the data is 10
self.assertEqual(len(self.f1_list), 1)
self.assertEqual(len(self.f2_list), 1)
self.assertEqual(len(self.f3_list), 1)
self.assertEqual(len(self.f4_list), 1)
self.assertEqual(len(self.f5_list), 1)
self.assertEqual(len(self.f1.x), 10)
self.assertEqual(self.f1.x[0], 0.002618)
self.assertEqual(self.f1.x[9], 0.0497)
self.assertTrue(self.f1.x_unit == 'A^{-1}')
self.assertTrue(self.f1.y_unit == 'cm^{-1}')
self.assertEqual(self.f1.meta_data['loader'], "ASCII")
def test_truncated_1(self):
"""
Test an ascii file with header and a
comment line in the middle of the data section.
The business rule says that we should stop
reading at the first comment once the data
section has started (and treat the comment
as though it were the start of a footer).
"""
# The length of the data is 5
self.assertEqual(len(self.f2.x), 5)
self.assertEqual(self.f2.x[0], 0.002618)
self.assertEqual(self.f2.x[4], 0.02356)
def test_truncated_2(self):
"""
Test a 6-col ascii file with header and a
line with only 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the first inconsitent line.
"""
# The length of the data is 5
self.assertEqual(len(self.f3.x), 5)
self.assertEqual(self.f3.x[0], 0.002618)
self.assertEqual(self.f3.x[4], 0.02356)
def test_truncated_3(self):
"""
Test a 6-col ascii file with complex header and
many lines with 2 or 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the last line of header.
"""
# The length of the data is 5
self.assertEqual(len(self.f4.x), 5)
self.assertEqual(self.f4.x[0], 0.012654)
self.assertEqual(self.f4.x[4], 0.02654)
def test_truncated_4(self):
"""
Test mix of 6-col and 2-col.
Only the last 5 2-col lines should be read.
"""
# The length of the data is 5
self.assertEqual(len(self.f5.x), 5)
self.assertEqual(self.f5.x[0], 0.02879)
self.assertEqual(self.f5.x[4], 0.0497)
def test_truncated_5(self):
"""
Test a 6-col ascii file with complex header where one of them has a
letter and many lines with 2 or 2 columns in the middle of the data
section. Will be rejected because fewer than 5 lines.
"""
# Test .ABS file loaded as ascii
f = None
try:
f = self.loader.load(find("ascii_test_6.txt"))
# The length of the data is 5
except:
self.assertEqual(f, None)
def test_nan_values(self):
"""
Test loading an ascii data file with nan values saved in x, y, or dy.
"""
f_1d = self.loader.load(find("nans_in_1d_data.dat"))[0]
f_2d = self.loader.load(find("nans_in_2d_data.DAT"))[0]
for i in range(0, len(f_1d.x) - 1):
self.assertFalse(math.isnan(f_1d.x[i]))
self.assertFalse(math.isnan(f_1d.y[i]))
self.assertFalse(math.isnan(f_1d.dy[i]))
f_2d.data = f_2d.data.flatten()
f_2d.qx_data = f_2d.qx_data.flatten()
f_2d.qy_data = f_2d.qy_data.flatten()
for i in range(0, len(f_2d.data) - 1):
self.assertFalse(math.isnan(f_2d.data[i]))
self.assertFalse(math.isnan(f_2d.qx_data[i]))
self.assertFalse(math.isnan(f_2d.qy_data[i]))
class ABSReaderTests(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.f1_list = self.loader.load("ascii_test_1.txt")
self.f1 = self.f1_list[0]
self.f2_list = self.loader.load("ascii_test_2.txt")
self.f2 = self.f2_list[0]
self.f3_list = self.loader.load("ascii_test_3.txt")
self.f3 = self.f3_list[0]
self.f4_list = self.loader.load("ascii_test_4.abs")
self.f4 = self.f4_list[0]
self.f5_list = self.loader.load("ascii_test_5.txt")
self.f5 = self.f5_list[0]
def test_checkdata(self):
"""
Test .ABS file loaded as ascii
"""
# The length of the data is 10
self.assertEqual(len(self.f1.x), 10)
self.assertEqual(self.f1.x[0],0.002618)
self.assertEqual(self.f1.x[9],0.0497)
self.assertTrue(self.f1.x_unit == 'A^{-1}')
self.assertTrue(self.f1.y_unit == 'cm^{-1}')
self.assertEqual(self.f1.meta_data['loader'],"ASCII")
def test_truncated_1(self):
"""
Test an ascii file with header and a
comment line in the middle of the data section.
The business rule says that we should stop
reading at the first comment once the data
section has started (and treat the comment
as though it were the start of a footer).
"""
# The length of the data is 5
self.assertEqual(len(self.f2.x), 5)
self.assertEqual(self.f2.x[0],0.002618)
self.assertEqual(self.f2.x[4],0.02356)
def test_truncated_2(self):
"""
Test a 6-col ascii file with header and a
line with only 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the first inconsitent line.
"""
# The length of the data is 5
self.assertEqual(len(self.f3.x), 5)
self.assertEqual(self.f3.x[0],0.002618)
self.assertEqual(self.f3.x[4],0.02356)
def test_truncated_3(self):
"""
Test a 6-col ascii file with complex header and
many lines with 2 or 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the last line of header.
"""
# The length of the data is 5
self.assertEqual(len(self.f4.x), 5)
self.assertEqual(self.f4.x[0],0.012654)
self.assertEqual(self.f4.x[4],0.02654)
def test_truncated_4(self):
"""
Test mix of 6-col and 2-col.
Only the last 5 2-col lines should be read.
"""
# The length of the data is 5
self.assertEqual(len(self.f5.x), 5)
self.assertEqual(self.f5.x[0],0.02879)
self.assertEqual(self.f5.x[4],0.0497)
def test_truncated_5(self):
"""
Test a 6-col ascii file with complex header where one of them has a
letter and many lines with 2 or 2 columns in the middle of the data
section. Will be rejected because fewer than 5 lines.
"""
# Test .ABS file loaded as ascii
f = None
try:
f = self.loader.load("ascii_test_6.txt")
# The length of the data is 5
except:
self.assertEqual(f, None)
class cansas_reader_hdf5(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.datafile_basic = find("test_data" + os.sep +
"simpleexamplefile.h5")
self.datafile_nodi = find("test_data" + os.sep + "x25000_no_di.h5")
self.datafile_multiplesasentry = find(
"test_data" + os.sep + "nxcansas_1Dand2D_multisasentry.h5")
self.datafile_multiplesasdata = find(
"test_data" + os.sep + "nxcansas_1Dand2D_multisasdata.h5")
self.datafile_multiplesasdata_multiplesasentry = find(
"test_data" + os.sep +
"nxcansas_1Dand2D_multisasentry_multisasdata.h5")
def test_real_data(self):
self.data = self.loader.load(self.datafile_basic)
self._check_example_data(self.data[0])
def test_no_di(self):
self.data = self.loader.load(self.datafile_nodi)
self.assertTrue(self.data is not None)
self.assertTrue(len(self.data) == 1)
dataset = self.data[0]
self.assertTrue(dataset.err_data is None)
self.assertTrue(dataset.q_data is not None)
def test_multiple_sasentries(self):
self.data = self.loader.load(self.datafile_multiplesasentry)
self.assertTrue(len(self.data) == 2)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
if isinstance(self.data[0], Data1D):
self._check_1d_data(self.data[0])
self._check_2d_data(self.data[1])
else:
self._check_1d_data(self.data[1])
self._check_2d_data(self.data[0])
def test_multiple_sasdatas(self):
self.data = self.loader.load(self.datafile_multiplesasdata)
self.assertTrue(len(self.data) == 2)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
if isinstance(self.data[0], Data1D):
self._check_1d_data(self.data[0])
self._check_2d_data(self.data[1])
else:
self._check_1d_data(self.data[1])
self._check_2d_data(self.data[0])
def test_multiple_sasentries_multiplesasdatas(self):
self.data = self.loader.load(
self.datafile_multiplesasdata_multiplesasentry)
self.assertTrue(len(self.data) == 4)
self._check_multiple_data(self.data[0])
self._check_multiple_data(self.data[1])
self._check_multiple_data(self.data[2])
self._check_multiple_data(self.data[3])
for data in self.data:
if isinstance(data, Data1D):
self._check_1d_data(data)
else:
self._check_2d_data(data)
def _check_multiple_data(self, data):
self.assertEqual(data.title, "MH4_5deg_16T_SLOW")
self.assertEqual(data.run[0], '33837')
self.assertEqual(len(data.run), 1)
self.assertEqual(data.instrument, "SANS2D")
self.assertEqual(data.source.radiation, "Spallation Neutron Source")
self.assertEqual(len(data.detector), 2)
self.assertTrue(data.detector[0].name == "rear-detector"
or data.detector[1].name == "rear-detector")
self.assertTrue(data.detector[0].name == "front-detector"
or data.detector[1].name == "front-detector")
self.assertAlmostEqual(
data.detector[0].distance + data.detector[1].distance, 7230.54, 2)
self.assertEqual(data.detector[0].distance_unit, 'mm')
self.assertEqual(len(data.trans_spectrum), 1)
def _check_1d_data(self, data):
self.assertEqual(len(data.x), 66)
self.assertEqual(len(data.x), len(data.y))
self.assertAlmostEqual(data.dy[10], 0.207214)
self.assertAlmostEqual(data.y[10], 24.1939)
self.assertAlmostEqual(data.x[10], 0.00898113)
def _check_2d_data(self, data):
self.assertTrue(isinstance(data, Data2D))
self.assertEqual(len(data.q_data), 150 * 150)
self.assertEqual(len(data.q_data), len(data.data))
self.assertAlmostEqual(data.err_data[10], 0.186723989418)
self.assertAlmostEqual(data.data[10], 0.465181)
self.assertAlmostEqual(data.qx_data[10], -0.129)
self.assertAlmostEqual(data.qy_data[10], -0.149)
def _check_example_data(self, data):
self.assertEqual(data.title, "")
self.assertEqual(data.x.size, 100)
self.assertEqual(data._xunit, "A^{-1}")
self.assertEqual(data._yunit, "cm^{-1}")
self.assertEqual(data.y.size, 100)
self.assertAlmostEqual(data.y[40], 0.952749011516985)
self.assertAlmostEqual(data.x[40], 0.3834415188257777)
self.assertAlmostEqual(len(data.meta_data), 0)
class GenericFileReaderTests(unittest.TestCase):
def setUp(self):
self.reader = TestFileReader()
self.bad_file = find("ACB123.txt")
self.good_file = find("123ABC.txt")
self.generic_reader = Loader()
self.deprecated_file_type = find("FEB18012.ASC")
def test_bad_file_path(self):
self.assertRaises(NoKnownLoaderException, self.reader.read,
self.bad_file)
def test_good_file_path(self):
f = open(self.good_file, 'w')
f.write('123ABC exists!')
f.close()
output = self.reader.read(self.good_file)
self.assertEqual(len(output), 1)
self.assertEqual(output[0].meta_data["blah"], '123ABC exists!')
def test_old_file_types(self):
f = self.generic_reader.load(self.deprecated_file_type)
last_f = f[0]
if hasattr(last_f, "errors"):
self.assertEqual(len(last_f.errors), 1)
else:
self.fail("Errors did not propogate to the file properly.")
def test_same_file_unknown_extensions(self):
# Five files, all with the same content, but different file extensions
no_ext = find("TestExtensions")
not_xml = find("TestExtensions.notxml")
# Deprecated extensions
asc_dep = find("TestExtensions.asc")
nxs_dep = find("TestExtensions.nxs")
# Native extension as a baseline
xml_native = find("TestExtensions.xml")
# Load the files and check contents
no_ext_load = self.generic_reader.load(no_ext)
asc_load = self.generic_reader.load(asc_dep)
nxs_load = self.generic_reader.load(nxs_dep)
not_xml_load = self.generic_reader.load(not_xml)
xml_load = self.generic_reader.load(xml_native)
self.check_unknown_extension(no_ext_load[0])
self.check_unknown_extension(asc_load[0])
self.check_unknown_extension(nxs_load[0])
self.check_unknown_extension(not_xml_load[0])
self.check_unknown_extension(xml_load[0])
# Be sure the deprecation warning is passed with the file
self.assertEqual(len(asc_load[0].errors), 1)
self.assertEqual(len(nxs_load[0].errors), 0)
def check_unknown_extension(self, data):
self.assertTrue(isinstance(data, Data1D))
self.assertEqual(len(data.x), 138)
self.assertEqual(data.sample.ID, "TK49 c10_SANS")
self.assertEqual(data.meta_data["loader"], "CanSAS XML 1D")
def tearDown(self):
if os.path.isfile(self.bad_file):
os.remove(self.bad_file)
if os.path.isfile(self.good_file):
os.remove(self.good_file)
class cansas_reader_xml(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.cansas_reader = Reader()
self.xml_reader = XMLreader()
self.xml_valid = find("cansas_test_modified.xml")
self.xml_invalid = find("cansas_test.xml")
self.cansas1d_badunits = find("cansas1d_badunits.xml")
self.cansas1d = find("cansas1d.xml")
self.cansas1d_slit = find("cansas1d_slit.xml")
self.cansas1d_units = find("cansas1d_units.xml")
self.cansas1d_notitle = find("cansas1d_notitle.xml")
self.isis_1_0 = find("ISIS_1_0.xml")
self.isis_1_1 = find("ISIS_1_1.xml")
self.isis_1_1_notrans = find("ISIS_1_1_notrans.xml")
self.isis_1_1_doubletrans = find("ISIS_1_1_doubletrans.xml")
self.schema_1_0 = find("cansas1d_v1_0.xsd")
self.schema_1_1 = find("cansas1d_v1_1.xsd")
self.write_1_0_filename = find("isis_1_0_write_test.xml")
self.write_1_1_filename = find("isis_1_1_write_test.xml")
self.write_filename = find("write_test.xml")
def get_number_of_entries(self, dictionary, name, i):
if dictionary.get(name) is not None:
i += 1
name = name.split("_")[0]
name += "_{0}".format(i)
name = self.get_number_of_entries(dictionary, name, i)
return name
def test_invalid_xml(self):
"""
Should fail gracefully and send a message to logger.info()
"""
invalid = StringIO('<a><c></b></a>')
self.assertRaises(XMLSyntaxError, lambda: XMLreader(invalid))
def test_xml_validate(self):
string = "<xsd:schema xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">\n"
string += "\t<xsd:element name=\"a\" type=\"AType\"/>\n"
string += "\t<xsd:complexType name=\"AType\">\n"
string += "\t\t<xsd:sequence>\n"
string += "\t\t\t<xsd:element name=\"b\" type=\"xsd:string\" />\n"
string += "\t\t</xsd:sequence>\n"
string += "\t</xsd:complexType>\n"
string += "</xsd:schema>"
f = StringIO(string)
xmlschema_doc = etree.parse(f)
xmlschema = etree.XMLSchema(xmlschema_doc)
valid = etree.parse(StringIO('<a><b></b></a>'))
invalid = etree.parse(StringIO('<a><c></c></a>'))
self.assertTrue(xmlschema.validate(valid))
self.assertFalse(xmlschema.validate(invalid))
def test_real_xml(self):
reader = XMLreader(self.xml_valid, self.schema_1_0)
valid = reader.validate_xml()
self.assertTrue(valid)
def _check_data(self, data):
self.assertTrue(data.title == "TK49 c10_SANS")
self.assertTrue(data.x.size == 138)
self.assertTrue(len(data.meta_data) == 3)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[1].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "HAB")
self.assertTrue(data.detector[1].name == "main-detector-bank")
self.assertTrue(data.detector[0].distance == 575.0)
self.assertAlmostEqual(data.detector[1].distance, 4145.02)
self.assertTrue(
data.process[0].name == "Mantid generated CanSAS1D XML")
self.assertTrue(data.meta_data["xmlpreprocess"] is not None)
def _check_data_1_1(self, data):
spectrum = data.trans_spectrum[0]
self.assertTrue(len(spectrum.wavelength) == 138)
def test_cansas_xml(self):
self.xml_reader.set_xml_file(self.isis_1_1)
self.xml_reader.set_schema(self.schema_1_1)
self.xml_reader.set_processing_instructions()
self.assertTrue(self.xml_reader.validate_xml())
dataloader = self.loader.load(self.isis_1_1)
cansasreader = self.cansas_reader.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
self.loader.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
def test_double_trans_spectra(self):
self.xml_reader.set_xml_file(self.cansas1d)
self.xml_reader.set_schema(self.schema_1_0)
self.assertTrue(self.xml_reader.validate_xml())
data = self.loader.load(self.isis_1_1_doubletrans)
for item in data:
self._check_data(item)
def test_entry_name_recurse(self):
test_values = [1, 2, 3, 4, 5, 6]
base_key = "key"
d = {}
for value in test_values:
new_key = self.get_number_of_entries(d, base_key, i=0)
d[new_key] = value
self.assertTrue(len(d) == 6)
def test_load_cansas_file(self):
reader1 = XMLreader(self.xml_valid, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
reader2 = XMLreader(self.xml_invalid, self.schema_1_0)
self.assertFalse(reader2.validate_xml())
reader3 = XMLreader(self.xml_valid, self.schema_1_1)
self.assertFalse(reader3.validate_xml())
reader4 = XMLreader(self.xml_invalid, self.schema_1_1)
self.assertFalse(reader4.validate_xml())
reader5 = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(reader5.validate_xml())
reader6 = XMLreader(self.isis_1_1, self.schema_1_1)
self.assertTrue(reader6.validate_xml())
reader7 = XMLreader(self.isis_1_1, self.schema_1_0)
self.assertFalse(reader7.validate_xml())
def test_invalid_cansas(self):
list = self.loader.load(self.cansas1d_notitle)
data = list[0]
self.assertTrue(data.x.size == 2)
self.assertTrue(len(data.meta_data) == 2)
self.assertTrue(len(data.errors) == 1)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "fictional hybrid")
self.assertTrue(data.detector[0].distance == 4150)
def test_old_cansas_files(self):
self.xml_reader.set_xml_file(self.cansas1d)
self.xml_reader.set_schema(self.schema_1_0)
self.assertTrue(self.xml_reader.validate_xml())
self.loader.load(self.cansas1d)
reader2 = XMLreader(self.cansas1d_units, self.schema_1_0)
self.assertTrue(reader2.validate_xml())
reader3 = XMLreader(self.cansas1d_badunits, self.schema_1_0)
self.assertTrue(reader3.validate_xml())
reader4 = XMLreader(self.cansas1d_slit, self.schema_1_0)
self.assertTrue(reader4.validate_xml())
def test_save_cansas_v1_0(self):
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
dataloader = self.loader.load(self.isis_1_0)
cansasreader = self.cansas_reader.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
self.loader.save(self.write_1_0_filename, dataloader[i], None)
reader2 = Reader()
self.assertTrue(os.path.isfile(self.write_1_0_filename))
return_data = reader2.read(self.write_1_0_filename)
written_data = return_data[0]
xmlreader = XMLreader(self.write_1_0_filename, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
self._check_data(written_data)
if os.path.isfile(self.write_1_0_filename):
os.remove(self.write_1_0_filename)
def test_processing_instructions(self):
self.xml_reader.set_xml_file(self.isis_1_1)
self.xml_reader.set_schema(self.schema_1_1)
if self.xml_reader.validate_xml():
# find the processing instructions and make into a dictionary
dic = self.get_processing_instructions(self.xml_reader)
self.assertEqual(
dic,
{'xml-stylesheet': 'type="text/xsl" href="cansas1d.xsl" '})
xml = "<test><a><b><c></c></b></a></test>"
xmldoc = minidom.parseString(xml)
# take the processing instructions and put them back in
xmldoc = self.set_processing_instructions(xmldoc, dic)
xmldoc.toprettyxml()
def set_processing_instructions(self, minidom_object, dic):
xmlroot = minidom_object.firstChild
for item in dic:
pi = minidom_object.createProcessingInstruction(item, dic[item])
minidom_object.insertBefore(pi, xmlroot)
return minidom_object
def get_processing_instructions(self, xml_reader_object):
dict = {}
pi = xml_reader_object.xmlroot.getprevious()
i = 0
while pi is not None:
attr = {}
pi_name = ""
pi_string = decode(etree.tostring(pi))
if isinstance(pi_string, str):
pi_string = pi_string.replace("<?", "").replace("?>", "")
split = pi_string.split(" ", 1)
pi_name = split[0]
attr = split[1]
dict[pi_name] = attr
pi = pi.getprevious()
return dict
def test_generic_loader(self):
# the generic loader should work as well
data = self.loader.load(find("cansas1d.xml"))
self.assertEqual(len(data), 1)
self.assertEqual(data[0].meta_data['loader'], "CanSAS XML 1D")
def test_cansas_checkdata(self):
self.data_list = self.loader.load(find("cansas1d.xml"))
self.data = self.data_list[0]
self.assertEqual(self.data.filename, "cansas1d.xml")
self._checkdata()
def _checkdata(self):
"""
Check the data content to see whether
it matches the specific file we loaded.
Check the units too to see whether the
Data1D defaults changed. Otherwise the
tests won't pass
"""
self.assertEqual(len(self.data_list), 1)
self.assertEqual(self.data.run[0], "1234")
self.assertEqual(self.data.meta_data['loader'], "CanSAS XML 1D")
# Data
self.assertEqual(len(self.data.x), 2)
self.assertEqual(self.data.x_unit, 'A^{-1}')
self.assertEqual(self.data.y_unit, 'cm^{-1}')
self.assertAlmostEqual(self.data.x[0], 0.02, 6)
self.assertAlmostEqual(self.data.y[0], 1000, 6)
self.assertAlmostEqual(self.data.dx[0], 0.01, 6)
self.assertAlmostEqual(self.data.dy[0], 3, 6)
self.assertAlmostEqual(self.data.x[1], 0.03, 6)
self.assertAlmostEqual(self.data.y[1], 1001.0)
self.assertAlmostEqual(self.data.dx[1], 0.02, 6)
self.assertAlmostEqual(self.data.dy[1], 4, 6)
self.assertEqual(self.data.run_name['1234'], 'run name')
self.assertEqual(self.data.title, "Test title")
# Sample info
self.assertEqual(self.data.sample.ID, "SI600-new-long")
self.assertEqual(self.data.sample.name, "my sample")
self.assertEqual(self.data.sample.thickness_unit, 'mm')
self.assertAlmostEqual(self.data.sample.thickness, 1.03)
self.assertAlmostEqual(self.data.sample.transmission, 0.327)
self.assertEqual(self.data.sample.temperature_unit, 'C')
self.assertEqual(self.data.sample.temperature, 0)
self.assertEqual(self.data.sample.position_unit, 'mm')
self.assertEqual(self.data.sample.position.x, 10)
self.assertEqual(self.data.sample.position.y, 0)
self.assertEqual(self.data.sample.orientation_unit, 'degree')
self.assertAlmostEqual(self.data.sample.orientation.x, 22.5, 6)
self.assertAlmostEqual(self.data.sample.orientation.y, 0.02, 6)
self.assertEqual(
self.data.sample.details[0],
"http://chemtools.chem.soton.ac.uk/projects/blog/blogs.php/bit_id/2720"
)
self.assertEqual(self.data.sample.details[1], "Some text here")
# Instrument info
self.assertEqual(self.data.instrument, "canSAS instrument")
# Source
self.assertEqual(self.data.source.radiation, "neutron")
self.assertEqual(self.data.source.beam_size_unit, "mm")
self.assertEqual(self.data.source.beam_size_name, "bm")
self.assertEqual(self.data.source.beam_size.x, 12)
self.assertEqual(self.data.source.beam_size.y, 13)
self.assertEqual(self.data.source.beam_shape, "disc")
self.assertEqual(self.data.source.wavelength_unit, "A")
self.assertEqual(self.data.source.wavelength, 6)
self.assertEqual(self.data.source.wavelength_max_unit, "nm")
self.assertAlmostEqual(self.data.source.wavelength_max, 1.0)
self.assertEqual(self.data.source.wavelength_min_unit, "nm")
self.assertAlmostEqual(self.data.source.wavelength_min, 0.22)
self.assertEqual(self.data.source.wavelength_spread_unit, "percent")
self.assertEqual(self.data.source.wavelength_spread, 14.3)
# Collimation
_found1 = False
_found2 = False
self.assertEqual(self.data.collimation[0].length, 123.)
self.assertEqual(self.data.collimation[0].name, 'test coll name')
for item in self.data.collimation[0].aperture:
self.assertEqual(item.size_unit, 'mm')
self.assertEqual(item.distance_unit, 'mm')
if item.size.x == 50 \
and item.distance == 11000.0 \
and item.name == 'source' \
and item.type == 'radius':
_found1 = True
elif item.size.x == 1.0 \
and item.name == 'sample' \
and item.type == 'radius':
_found2 = True
if not _found1 or not _found2:
raise RuntimeError("Could not find all data %s %s" %
(_found1, _found2))
# Detector
self.assertEqual(self.data.detector[0].name, "fictional hybrid")
self.assertEqual(self.data.detector[0].distance_unit, "mm")
self.assertEqual(self.data.detector[0].distance, 4150)
self.assertEqual(self.data.detector[0].orientation_unit, "degree")
self.assertAlmostEqual(self.data.detector[0].orientation.x, 1.0, 6)
self.assertEqual(self.data.detector[0].orientation.y, 0.0)
self.assertEqual(self.data.detector[0].orientation.z, 0.0)
self.assertEqual(self.data.detector[0].offset_unit, "m")
self.assertEqual(self.data.detector[0].offset.x, .001)
self.assertEqual(self.data.detector[0].offset.y, .002)
self.assertEqual(self.data.detector[0].offset.z, None)
self.assertEqual(self.data.detector[0].beam_center_unit, "mm")
self.assertEqual(self.data.detector[0].beam_center.x, 322.64)
self.assertEqual(self.data.detector[0].beam_center.y, 327.68)
self.assertEqual(self.data.detector[0].beam_center.z, None)
self.assertEqual(self.data.detector[0].pixel_size_unit, "mm")
self.assertEqual(self.data.detector[0].pixel_size.x, 5)
self.assertEqual(self.data.detector[0].pixel_size.y, 5)
self.assertEqual(self.data.detector[0].pixel_size.z, None)
# Process
_found_term1 = False
_found_term2 = False
for item in self.data.process:
self.assertTrue(item.name in ['NCNR-IGOR', 'spol'])
self.assertTrue(
item.date in ['04-Sep-2007 18:35:02', '03-SEP-2006 11:42:47'])
for t in item.term:
if (t['name'] == "ABS:DSTAND" and t['unit'] == 'mm'
and float(t['value']) == 1.0):
_found_term2 = True
elif (t['name'] == "radialstep" and t['unit'] == 'mm'
and float(t['value']) == 10.0):
_found_term1 = True
if not _found_term1 or not _found_term2:
raise RuntimeError("Could not find all process terms %s %s" %
(_found_term1, _found_term2))
def test_writer(self):
filename = "write_test.xml"
data_list = self.loader.load(self.cansas1d)
data = data_list[0]
self.cansas_reader.write(self.write_filename, data)
self.data_list = self.loader.load(self.write_filename)
self.data = self.data_list[0]
self.assertEqual(len(self.data_list), 1)
self.assertEqual(len(self.data_list), len(self.data_list))
self.assertEqual(self.data.filename, filename)
self._checkdata()
if os.path.isfile(self.write_filename):
os.remove(self.write_filename)
def test_units(self):
"""
Check units.
Note that not all units are available.
"""
filename = "cansas1d_units.xml"
self.data_list = self.cansas_reader.read(find(filename))
self.data = self.data_list[0]
self.assertEqual(len(self.data_list), 1)
self.assertEqual(self.data.filename, filename)
self._checkdata()
def test_badunits(self):
"""
Check units.
Note that not all units are available.
"""
filename = "cansas1d_badunits.xml"
self.data_list = self.cansas_reader.read(find(filename))
self.data = self.data_list[0]
self.assertEqual(len(self.data_list), 1)
self.assertEqual(self.data.filename, filename)
# The followed should not have been loaded
self.assertAlmostEqual(self.data.sample.thickness, 0.00103)
# This one should
self.assertAlmostEqual(self.data.sample.transmission, 0.327)
self.assertEqual(self.data.meta_data['loader'], "CanSAS XML 1D")
self.assertEqual(len(self.data.errors), 0)
def test_slits(self):
"""
Check slit data
"""
filename = "cansas1d_slit.xml"
self.data_list = self.cansas_reader.read(find(filename))
self.data = self.data_list[0]
self.assertEqual(len(self.data_list), 1)
self.assertEqual(self.data.filename, filename)
self.assertEqual(self.data.run[0], "1234")
# Data
self.assertEqual(len(self.data.x), 2)
self.assertEqual(self.data.x_unit, 'A^{-1}')
self.assertEqual(self.data.y_unit, 'cm^{-1}')
self.assertEqual(self.data.x[0], 0.02)
self.assertEqual(self.data.y[0], 1000)
self.assertEqual(self.data.dxl[0], 0.005)
self.assertEqual(self.data.dxw[0], 0.001)
self.assertEqual(self.data.dy[0], 3)
self.assertEqual(self.data.x[1], 0.03)
self.assertAlmostEqual(self.data.y[1], 1001.0)
self.assertEqual(self.data.dxl[1], 0.005)
self.assertEqual(self.data.dxw[1], 0.001)
self.assertEqual(self.data.dy[1], 4)
self.assertEqual(self.data.run_name['1234'], 'run name')
self.assertEqual(self.data.title, "Test title")
class GenericFileReaderTests(unittest.TestCase):
def setUp(self):
self.reader = TestFileReader()
self.bad_file = find("ACB123.txt")
self.good_file = find("123ABC.txt")
self.generic_reader = Loader()
self.deprecated_file_type = find("FEB18012.ASC")
def test_bad_file_path(self):
self.assertRaises(NoKnownLoaderException, self.reader.read,
self.bad_file)
def test_good_file_path(self):
f = open(self.good_file, 'w')
f.write('123ABC exists!')
f.close()
output = self.reader.read(self.good_file)
self.assertEqual(len(output), 1)
self.assertEqual(output[0].meta_data["blah"], '123ABC exists!')
def test_old_file_types(self):
f = self.generic_reader.load(self.deprecated_file_type)
last_f = f[0]
if hasattr(last_f, "errors"):
self.assertEqual(len(last_f.errors), 1)
else:
self.fail("Errors did not propogate to the file properly.")
def test_same_file_unknown_extensions(self):
# Five files, all with the same content, but different file extensions
no_ext = find("test_data" + os.sep + "TestExtensions")
not_xml = find("test_data" + os.sep + "TestExtensions.notxml")
# Deprecated extensions
asc_dep = find("test_data" + os.sep + "TestExtensions.asc")
nxs_dep = find("test_data" + os.sep + "TestExtensions.nxs")
# Native extension as a baseline
xml_native = find("test_data" + os.sep + "TestExtensions.xml")
# Load the files and check contents
no_ext_load = self.generic_reader.load(no_ext)
asc_load = self.generic_reader.load(asc_dep)
nxs_load = self.generic_reader.load(nxs_dep)
not_xml_load = self.generic_reader.load(not_xml)
xml_load = self.generic_reader.load(xml_native)
self.check_unknown_extension(no_ext_load[0])
self.check_unknown_extension(asc_load[0])
self.check_unknown_extension(nxs_load[0])
self.check_unknown_extension(not_xml_load[0])
self.check_unknown_extension(xml_load[0])
# Be sure the deprecation warning is passed with the file
self.assertEqual(len(asc_load[0].errors), 1)
self.assertEqual(len(nxs_load[0].errors), 0)
def check_unknown_extension(self, data):
self.assertTrue(isinstance(data, Data1D))
self.assertEqual(len(data.x), 138)
self.assertEqual(data.sample.ID, "TK49 c10_SANS")
self.assertEqual(data.meta_data["loader"], "CanSAS XML 1D")
def tearDown(self):
if os.path.isfile(self.bad_file):
os.remove(self.bad_file)
if os.path.isfile(self.good_file):
os.remove(self.good_file)
self.msg_txt.SetEditable(False)
self.msg_txt.SetValue('No message available')
self.sizer.Add(self.msg_txt, 1, wx.EXPAND | wx.ALL, 10)
if self._data is not None:
self.set_message(msg=self._data.__str__())
self.SetSizer(self.sizer)
def set_manager(self, manager):
"""
Set the manager of this window
"""
self._manager = manager
def set_message(self, msg=""):
"""
Display the message received
"""
self.msg_txt.SetValue(str(msg))
if __name__ == "__main__":
app = wx.App()
# Instantiate a loader
loader = Loader()
# Load data
test_data = loader.load("MAR07232_rest.ASC")
dlg = ConsoleDialog(data=test_data)
dlg.ShowModal()
app.MainLoop()
class cansas_reader_xml(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.xml_valid = find("cansas_test_modified.xml")
self.xml_invalid = find("cansas_test.xml")
self.cansas1d_badunits = find("cansas1d_badunits.xml")
self.cansas1d = find("cansas1d.xml")
self.cansas1d_slit = find("cansas1d_slit.xml")
self.cansas1d_units = find("cansas1d_units.xml")
self.cansas1d_notitle = find("cansas1d_notitle.xml")
self.isis_1_0 = find("ISIS_1_0.xml")
self.isis_1_1 = find("ISIS_1_1.xml")
self.isis_1_1_notrans = find("ISIS_1_1_notrans.xml")
self.isis_1_1_doubletrans = find("ISIS_1_1_doubletrans.xml")
self.schema_1_0 = find("cansas1d_v1_0.xsd")
self.schema_1_1 = find("cansas1d_v1_1.xsd")
self.write_1_0_filename = find("isis_1_0_write_test.xml")
self.write_1_1_filename = find("isis_1_1_write_test.xml")
def get_number_of_entries(self, dictionary, name, i):
if dictionary.get(name) is not None:
i += 1
name = name.split("_")[0]
name += "_{0}".format(i)
name = self.get_number_of_entries(dictionary, name, i)
return name
def test_invalid_xml(self):
"""
Should fail gracefully and send a message to logger.info()
"""
invalid = StringIO('<a><c></b></a>')
self.assertRaises(XMLSyntaxError, lambda: XMLreader(invalid))
def test_xml_validate(self):
string = "<xsd:schema xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">\n"
string += "\t<xsd:element name=\"a\" type=\"AType\"/>\n"
string += "\t<xsd:complexType name=\"AType\">\n"
string += "\t\t<xsd:sequence>\n"
string += "\t\t\t<xsd:element name=\"b\" type=\"xsd:string\" />\n"
string += "\t\t</xsd:sequence>\n"
string += "\t</xsd:complexType>\n"
string += "</xsd:schema>"
f = StringIO(string)
xmlschema_doc = etree.parse(f)
xmlschema = etree.XMLSchema(xmlschema_doc)
valid = etree.parse(StringIO('<a><b></b></a>'))
invalid = etree.parse(StringIO('<a><c></c></a>'))
self.assertTrue(xmlschema.validate(valid))
self.assertFalse(xmlschema.validate(invalid))
def test_real_xml(self):
reader = XMLreader(self.xml_valid, self.schema_1_0)
valid = reader.validate_xml()
self.assertTrue(valid)
def _check_data(self, data):
self.assertTrue(data.title == "TK49 c10_SANS")
self.assertTrue(data.x.size == 138)
self.assertTrue(len(data.meta_data) == 3)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[1].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "HAB")
self.assertTrue(data.detector[1].name == "main-detector-bank")
self.assertTrue(data.detector[0].distance == 575.0)
self.assertAlmostEqual(data.detector[1].distance, 4145.02)
self.assertTrue(data.process[0].name == "Mantid generated CanSAS1D XML")
self.assertTrue(data.meta_data["xmlpreprocess"] is not None)
def _check_data_1_1(self, data):
spectrum = data.trans_spectrum[0]
self.assertTrue(len(spectrum.wavelength) == 138)
def test_cansas_xml(self):
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
def test_double_trans_spectra(self):
xmlreader = XMLreader(self.isis_1_1_doubletrans, self.schema_1_1)
self.assertTrue(xmlreader.validate_xml())
reader = Loader()
data = reader.load(self.isis_1_1_doubletrans)
for item in data:
self._check_data(item)
def test_entry_name_recurse(self):
test_values = [1,2,3,4,5,6]
base_key = "key"
d = {}
for value in test_values:
new_key = self.get_number_of_entries(d, base_key, i = 0)
d[new_key] = value
self.assertTrue(len(d) == 6)
def test_load_cansas_file(self):
reader1 = XMLreader(self.xml_valid, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
reader2 = XMLreader(self.xml_invalid, self.schema_1_0)
self.assertFalse(reader2.validate_xml())
reader3 = XMLreader(self.xml_valid, self.schema_1_1)
self.assertFalse(reader3.validate_xml())
reader4 = XMLreader(self.xml_invalid, self.schema_1_1)
self.assertFalse(reader4.validate_xml())
reader5 = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(reader5.validate_xml())
reader6 = XMLreader(self.isis_1_1, self.schema_1_1)
self.assertTrue(reader6.validate_xml())
reader7 = XMLreader(self.isis_1_1, self.schema_1_0)
self.assertFalse(reader7.validate_xml())
def test_invalid_cansas(self):
list = self.loader.load(self.cansas1d_notitle)
data = list[0]
self.assertTrue(data.x.size == 2)
self.assertTrue(len(data.meta_data) == 2)
self.assertTrue(len(data.errors) == 1)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "fictional hybrid")
self.assertTrue(data.detector[0].distance == 4150)
def test_old_cansas_files(self):
reader1 = XMLreader(self.cansas1d, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
file_loader = Loader()
file_loader.load(self.cansas1d)
reader2 = XMLreader(self.cansas1d_units, self.schema_1_0)
self.assertTrue(reader2.validate_xml())
reader3 = XMLreader(self.cansas1d_badunits, self.schema_1_0)
self.assertTrue(reader3.validate_xml())
reader4 = XMLreader(self.cansas1d_slit, self.schema_1_0)
self.assertTrue(reader4.validate_xml())
def test_save_cansas_v1_0(self):
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(self.write_1_0_filename, dataloader[i], None)
reader2 = Reader()
self.assertTrue(os.path.isfile(self.write_1_0_filename))
return_data = reader2.read(self.write_1_0_filename)
written_data = return_data[0]
xmlreader = XMLreader(self.write_1_0_filename, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
self._check_data(written_data)
if os.path.isfile(self.write_1_0_filename):
os.remove(self.write_1_0_filename)
def test_processing_instructions(self):
reader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = reader.validate_xml()
if valid:
# find the processing instructions and make into a dictionary
dic = self.get_processing_instructions(reader)
self.assertEqual(dic, {'xml-stylesheet':
'type="text/xsl" href="cansas1d.xsl" '})
xml = "<test><a><b><c></c></b></a></test>"
xmldoc = minidom.parseString(xml)
# take the processing instructions and put them back in
xmldoc = self.set_processing_instructions(xmldoc, dic)
xmldoc.toprettyxml()
def set_processing_instructions(self, minidom_object, dic):
xmlroot = minidom_object.firstChild
for item in dic:
pi = minidom_object.createProcessingInstruction(item, dic[item])
minidom_object.insertBefore(pi, xmlroot)
return minidom_object
def get_processing_instructions(self, xml_reader_object):
dict = {}
pi = xml_reader_object.xmlroot.getprevious()
i = 0
while pi is not None:
attr = {}
pi_name = ""
pi_string = decode(etree.tostring(pi))
if isinstance(pi_string, str):
pi_string = pi_string.replace("<?", "").replace("?>", "")
split = pi_string.split(" ", 1)
pi_name = split[0]
attr = split[1]
dict[pi_name] = attr
pi = pi.getprevious()
return dict
class abs_reader(unittest.TestCase):
def setUp(self):
self.loader = Loader()
def test_checkdata(self):
"""
Test .ABS file loaded as ascii
"""
f = self.loader.load("ascii_test_1.txt")
# The length of the data is 10
self.assertEqual(len(f.x), 10)
self.assertEqual(f.x[0],0.002618)
self.assertEqual(f.x[9],0.0497)
self.assertEqual(f.x_unit, '1/A')
self.assertEqual(f.y_unit, '1/cm')
self.assertEqual(f.meta_data['loader'],"ASCII")
def test_truncated_1(self):
"""
Test an ascii file with header and a
comment line in the middle of the data section.
The business rule says that we should stop
reading at the first comment once the data
section has started (and treat the comment
as though it were the start of a footer).
"""
# Test .ABS file loaded as ascii
f = self.loader.load("ascii_test_2.txt")
# The length of the data is 10
self.assertEqual(len(f.x), 5)
self.assertEqual(f.x[0],0.002618)
self.assertEqual(f.x[4],0.02356)
def test_truncated_2(self):
"""
Test a 6-col ascii file with header and a
line with only 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the first inconsitent line.
"""
# Test .ABS file loaded as ascii
f = self.loader.load("ascii_test_3.txt")
# The length of the data is 5
self.assertEqual(len(f.x), 5)
self.assertEqual(f.x[0],0.002618)
self.assertEqual(f.x[4],0.02356)
def test_truncated_3(self):
"""
Test a 6-col ascii file with complex header and
many lines with 2 or 2 columns in the middle of the data section.
The business rule says that we should stop
reading at the last line of header.
"""
# Test .ABS file loaded as ascii
f = self.loader.load("ascii_test_4.abs")
# The length of the data is 5
self.assertEqual(len(f.x), 5)
self.assertEqual(f.x[0],0.012654)
self.assertEqual(f.x[4],0.02654)
def test_truncated_4(self):
"""
Test mix of 6-col and 2-col.
Only the last 5 2-col lines should be read.
"""
# Test .ABS file loaded as ascii
f = self.loader.load("ascii_test_5.txt")
# The length of the data is 5
self.assertEqual(len(f.x), 5)
self.assertEqual(f.x[0],0.02879)
self.assertEqual(f.x[4],0.0497)
def test_truncated_5(self):
"""
Test a 6-col ascii file with complex header where one of them has a letter and
many lines with 2 or 2 columns in the middle of the data section.
Only last four lines should be read.
"""
# Test .ABS file loaded as ascii
f = None
try:
f = self.loader.load("ascii_test_6.txt")
# The length of the data is 5
except:
self.assertEqual(f, None)
class cansas_reader_xml(unittest.TestCase):
def setUp(self):
self.loader = Loader()
self.xml_valid = find("cansas_test_modified.xml")
self.xml_invalid = find("cansas_test.xml")
self.cansas1d_badunits = find("cansas1d_badunits.xml")
self.cansas1d = find("cansas1d.xml")
self.cansas1d_slit = find("cansas1d_slit.xml")
self.cansas1d_units = find("cansas1d_units.xml")
self.cansas1d_notitle = find("cansas1d_notitle.xml")
self.isis_1_0 = find("ISIS_1_0.xml")
self.isis_1_1 = find("ISIS_1_1.xml")
self.isis_1_1_notrans = find("ISIS_1_1_notrans.xml")
self.isis_1_1_doubletrans = find("ISIS_1_1_doubletrans.xml")
self.schema_1_0 = find("cansas1d_v1_0.xsd")
self.schema_1_1 = find("cansas1d_v1_1.xsd")
self.write_1_0_filename = find("isis_1_0_write_test.xml")
self.write_1_1_filename = find("isis_1_1_write_test.xml")
def get_number_of_entries(self, dictionary, name, i):
if dictionary.get(name) is not None:
i += 1
name = name.split("_")[0]
name += "_{0}".format(i)
name = self.get_number_of_entries(dictionary, name, i)
return name
def test_invalid_xml(self):
"""
Should fail gracefully and send a message to logger.info()
"""
invalid = StringIO('<a><c></b></a>')
self.assertRaises(XMLSyntaxError, lambda: XMLreader(invalid))
def test_xml_validate(self):
string = "<xsd:schema xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">\n"
string += "\t<xsd:element name=\"a\" type=\"AType\"/>\n"
string += "\t<xsd:complexType name=\"AType\">\n"
string += "\t\t<xsd:sequence>\n"
string += "\t\t\t<xsd:element name=\"b\" type=\"xsd:string\" />\n"
string += "\t\t</xsd:sequence>\n"
string += "\t</xsd:complexType>\n"
string += "</xsd:schema>"
f = StringIO(string)
xmlschema_doc = etree.parse(f)
xmlschema = etree.XMLSchema(xmlschema_doc)
valid = etree.parse(StringIO('<a><b></b></a>'))
invalid = etree.parse(StringIO('<a><c></c></a>'))
self.assertTrue(xmlschema.validate(valid))
self.assertFalse(xmlschema.validate(invalid))
def test_real_xml(self):
reader = XMLreader(self.xml_valid, self.schema_1_0)
valid = reader.validate_xml()
self.assertTrue(valid)
def _check_data(self, data):
self.assertTrue(data.title == "TK49 c10_SANS")
self.assertTrue(data.x.size == 138)
self.assertTrue(len(data.meta_data) == 3)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[1].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "HAB")
self.assertTrue(data.detector[1].name == "main-detector-bank")
self.assertTrue(data.detector[0].distance == 575.0)
self.assertAlmostEqual(data.detector[1].distance, 4145.02)
self.assertTrue(
data.process[0].name == "Mantid generated CanSAS1D XML")
self.assertTrue(data.meta_data["xmlpreprocess"] is not None)
def _check_data_1_1(self, data):
spectrum = data.trans_spectrum[0]
self.assertTrue(len(spectrum.wavelength) == 138)
def test_cansas_xml(self):
xmlreader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = xmlreader.validate_xml()
xmlreader.set_processing_instructions()
self.assertTrue(valid)
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_1)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_1)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data_1_1(dataloader[i])
self._check_data(cansasreader[i])
self._check_data_1_1(cansasreader[i])
reader_generic.save(self.write_1_1_filename, dataloader[i], None)
reader2 = Loader()
self.assertTrue(os.path.isfile(self.write_1_1_filename))
return_data = reader2.load(self.write_1_1_filename)
written_data = return_data[0]
self._check_data(written_data)
if os.path.isfile(self.write_1_1_filename):
os.remove(self.write_1_1_filename)
def test_double_trans_spectra(self):
xmlreader = XMLreader(self.isis_1_1_doubletrans, self.schema_1_1)
self.assertTrue(xmlreader.validate_xml())
reader = Loader()
data = reader.load(self.isis_1_1_doubletrans)
for item in data:
self._check_data(item)
def test_entry_name_recurse(self):
test_values = [1, 2, 3, 4, 5, 6]
base_key = "key"
d = {}
for value in test_values:
new_key = self.get_number_of_entries(d, base_key, i=0)
d[new_key] = value
self.assertTrue(len(d) == 6)
def test_load_cansas_file(self):
reader1 = XMLreader(self.xml_valid, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
reader2 = XMLreader(self.xml_invalid, self.schema_1_0)
self.assertFalse(reader2.validate_xml())
reader3 = XMLreader(self.xml_valid, self.schema_1_1)
self.assertFalse(reader3.validate_xml())
reader4 = XMLreader(self.xml_invalid, self.schema_1_1)
self.assertFalse(reader4.validate_xml())
reader5 = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(reader5.validate_xml())
reader6 = XMLreader(self.isis_1_1, self.schema_1_1)
self.assertTrue(reader6.validate_xml())
reader7 = XMLreader(self.isis_1_1, self.schema_1_0)
self.assertFalse(reader7.validate_xml())
def test_invalid_cansas(self):
list = self.loader.load(self.cansas1d_notitle)
data = list[0]
self.assertTrue(data.x.size == 2)
self.assertTrue(len(data.meta_data) == 2)
self.assertTrue(len(data.errors) == 1)
self.assertTrue(data.detector[0].distance_unit == "mm")
self.assertTrue(data.detector[0].name == "fictional hybrid")
self.assertTrue(data.detector[0].distance == 4150)
def test_old_cansas_files(self):
reader1 = XMLreader(self.cansas1d, self.schema_1_0)
self.assertTrue(reader1.validate_xml())
file_loader = Loader()
file_loader.load(self.cansas1d)
reader2 = XMLreader(self.cansas1d_units, self.schema_1_0)
self.assertTrue(reader2.validate_xml())
reader3 = XMLreader(self.cansas1d_badunits, self.schema_1_0)
self.assertTrue(reader3.validate_xml())
reader4 = XMLreader(self.cansas1d_slit, self.schema_1_0)
self.assertTrue(reader4.validate_xml())
def test_save_cansas_v1_0(self):
xmlreader = XMLreader(self.isis_1_0, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
reader_generic = Loader()
dataloader = reader_generic.load(self.isis_1_0)
reader_cansas = Reader()
cansasreader = reader_cansas.read(self.isis_1_0)
for i in range(len(dataloader)):
self._check_data(dataloader[i])
self._check_data(cansasreader[i])
reader_generic.save(self.write_1_0_filename, dataloader[i], None)
reader2 = Reader()
self.assertTrue(os.path.isfile(self.write_1_0_filename))
return_data = reader2.read(self.write_1_0_filename)
written_data = return_data[0]
xmlreader = XMLreader(self.write_1_0_filename, self.schema_1_0)
self.assertTrue(xmlreader.validate_xml())
self._check_data(written_data)
if os.path.isfile(self.write_1_0_filename):
os.remove(self.write_1_0_filename)
def test_processing_instructions(self):
reader = XMLreader(self.isis_1_1, self.schema_1_1)
valid = reader.validate_xml()
if valid:
# find the processing instructions and make into a dictionary
dic = self.get_processing_instructions(reader)
self.assertEqual(
dic,
{'xml-stylesheet': 'type="text/xsl" href="cansas1d.xsl" '})
xml = "<test><a><b><c></c></b></a></test>"
xmldoc = minidom.parseString(xml)
# take the processing instructions and put them back in
xmldoc = self.set_processing_instructions(xmldoc, dic)
xmldoc.toprettyxml()
def set_processing_instructions(self, minidom_object, dic):
xmlroot = minidom_object.firstChild
for item in dic:
pi = minidom_object.createProcessingInstruction(item, dic[item])
minidom_object.insertBefore(pi, xmlroot)
return minidom_object
def get_processing_instructions(self, xml_reader_object):
dict = {}
pi = xml_reader_object.xmlroot.getprevious()
i = 0
while pi is not None:
attr = {}
pi_name = ""
pi_string = decode(etree.tostring(pi))
if isinstance(pi_string, str):
pi_string = pi_string.replace("<?", "").replace("?>", "")
split = pi_string.split(" ", 1)
pi_name = split[0]
attr = split[1]
dict[pi_name] = attr
pi = pi.getprevious()
return dict
class testLoader(unittest.TestCase):
logging.debug("Inside testLoad module")
""" test fitting """
def setUp(self):
"""
Set up the initial conditions before _each_ test
so that they all start from the same well-defined state.
"""
#Creating a loader
self.L=Loader()
def testLoad0(self):
"""test reading empty file"""
self.assertRaises(RuntimeError, self.L.load, 'empty.txt')
def testLoad1(self):
"""test reading 2 columns"""
#Testing loading a txt file of 2 columns, the only reader should be read1
output=self.L.load('test_2_columns.txt')
x=[2.83954,0.204082,0.408163,0.612245,0.816327,1.02041,1.22449,1.42857,1.63265]
y=[0.6,3.44938, 5.82026,5.27591,5.2781,5.22531,7.47487,7.85852,10.2278]
dx=[]
dy=[]
self.assertEqual(len(output.x),len(x))
self.assertEqual(len(output.y),len(y))
for i in range(len(x)):
self.assertEqual(output.x[i],x[i])
self.assertEqual(output.y[i],y[i])
def testLoad2(self):
"""Testing loading a txt file of 3 columns"""
output= self.L.load('test_3_columns.txt')
x=[0,0.204082,0.408163,0.612245,0.816327,1.02041,1.22449]
y=[2.83954,3.44938,5.82026,5.27591,5.2781,5.22531,7.47487]
dx=[]
dy=[0.6,0.676531,0.753061,0.829592,0.906122,0.982653,1.05918]
self.assertEqual(len(output.x),len(x))
self.assertEqual(len(output.y),len(y))
self.assertEqual(len(output.dy),len(dy))
for i in range(len(x)):
self.assertEqual(output.x[i],x[i])
self.assertEqual(output.y[i],y[i])
self.assertEqual(output.dy[i],dy[i])
def testLoad2_uppercase(self):
"""Testing loading a txt file of 3 columns"""
output= self.L.load('test_3_columns.TXT')
x=[0,0.204082,0.408163,0.612245,0.816327,1.02041,1.22449]
y=[2.83954,3.44938,5.82026,5.27591,5.2781,5.22531,7.47487]
dx=[]
dy=[0.6,0.676531,0.753061,0.829592,0.906122,0.982653,1.05918]
self.assertEqual(len(output.x),len(x))
self.assertEqual(len(output.y),len(y))
self.assertEqual(len(output.dy),len(dy))
for i in range(len(x)):
self.assertEqual(output.x[i],x[i])
self.assertEqual(output.y[i],y[i])
self.assertEqual(output.dy[i],dy[i])
def testload3(self):
""" Testing loading Igor data"""
#tested good file.asc
output= self.L.load('MAR07232_rest.ASC')
self.assertEqual(output.xmin,-0.018558945804750416)
self.assertEqual(output.xmax, 0.016234058202440633,)
self.assertEqual(output.ymin,-0.01684257151702391)
self.assertEqual(output.ymax,0.017950440578015116)
#tested corrupted file.asc
try:self.L.load('AR07232_rest.ASC')
except ValueError,msg:
#logging.log(10,str(msg))
logging.error(str(msg))
class Plugin(PluginBase):
def __init__(self):
PluginBase.__init__(self, name="DataLoader")
# Default location
self._default_save_location = DEFAULT_OPEN_FOLDER
self.loader = Loader()
self._data_menu = None
def populate_file_menu(self):
"""
get a menu item and append it under file menu of the application
add load file menu item and load folder item
"""
# menu for data files
data_file_hint = "load one or more data in the application"
menu_list = [('&Load Data File(s)', data_file_hint, self.load_data)]
gui_style = self.parent.get_style()
style = gui_style & GUIFRAME.MULTIPLE_APPLICATIONS
if style == GUIFRAME.MULTIPLE_APPLICATIONS:
# menu for data from folder
data_folder_hint = "load multiple data in the application"
menu_list.append(('&Load Data Folder', data_folder_hint,
self._load_folder))
return menu_list
def load_data(self, event):
"""
Load data
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location is None:
self._default_save_location = os.getcwd()
cards = self.loader.get_wildcards()
temp = [APPLICATION_WLIST] + PLUGINS_WLIST
for item in temp:
if item in cards:
cards.remove(item)
wlist = '|'.join(cards)
style = wx.OPEN | wx.FD_MULTIPLE
dlg = wx.FileDialog(self.parent,
"Choose a file",
self._default_save_location, "",
wlist,
style=style)
if dlg.ShowModal() == wx.ID_OK:
file_list = dlg.GetPaths()
if len(file_list) >= 0 and file_list[0] is not None:
self._default_save_location = os.path.dirname(file_list[0])
path = self._default_save_location
dlg.Destroy()
if path is None or not file_list or file_list[0] is None:
return
self.parent._default_save_location = self._default_save_location
self.get_data(file_list)
def can_load_data(self):
"""
if return True, then call handler to load data
"""
return True
def _load_folder(self, event):
"""
Load entire folder
"""
path = None
self._default_save_location = self.parent._default_save_location
if self._default_save_location is None:
self._default_save_location = os.getcwd()
dlg = wx.DirDialog(self.parent, "Choose a directory",
self._default_save_location,
style=wx.DD_DEFAULT_STYLE)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
self._default_save_location = path
dlg.Destroy()
if path is not None:
self._default_save_location = os.path.dirname(path)
else:
return
file_list = self.get_file_path(path)
self.get_data(file_list)
self.parent._default_save_location = self._default_save_location
def load_error(self, error=None):
"""
Pop up an error message.
:param error: details error message to be displayed
"""
if error is not None or str(error).strip() != "":
dial = wx.MessageDialog(self.parent, str(error),
'Error Loading File',
wx.OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
def get_file_path(self, path):
"""
Receive a list containing folder then return a list of file
"""
if os.path.isdir(path):
return [os.path.join(os.path.abspath(path), filename) for filename
in os.listdir(path)]
def _process_data_and_errors(self, item, p_file, output, message):
"""
Check to see if data set loaded with any errors. If so, append to
error message to be sure user knows the issue.
"""
data_error = False
if hasattr(item, 'errors'):
for error_data in item.errors:
data_error = True
message += "\tError: {0}\n".format(error_data)
else:
logger.error("Loader returned an invalid object:\n %s" % str(item))
data_error = True
data = self.parent.create_gui_data(item, p_file)
output[data.id] = data
return output, message, data_error
def get_data(self, path, format=None):
"""
"""
file_errors = {}
output = {}
exception_occurred = False
for p_file in path:
basename = os.path.basename(p_file)
# Skip files that start with a period
if basename.startswith("."):
msg = "The folder included a potential hidden file - %s." \
% basename
msg += " Do you wish to load this file as data?"
msg_box = wx.MessageDialog(None, msg, 'Warning',
wx.OK | wx.CANCEL)
if msg_box.ShowModal() == wx.ID_CANCEL:
continue
_, extension = os.path.splitext(basename)
if extension.lower() in EXTENSIONS:
log_msg = "Data Loader cannot "
log_msg += "load: {}\n".format(str(p_file))
log_msg += "Please try to open that file from \"open project\""
log_msg += "or \"open analysis\" menu."
logger.info(log_msg)
file_errors[basename] = [log_msg]
continue
try:
message = "Loading {}...\n".format(p_file)
self.load_update(message=message, info="info")
temp = self.loader.load(p_file, format)
if not isinstance(temp, list):
temp = [temp]
for item in temp:
error_message = ""
output, error_message, data_error = \
self._process_data_and_errors(item,
p_file,
output,
error_message)
if data_error:
if basename in file_errors.keys():
file_errors[basename] += [error_message]
else:
file_errors[basename] = [error_message]
self.load_update(message="Loaded {}\n".format(p_file),
info="info")
except NoKnownLoaderException as e:
exception_occurred = True
error_message = "Loading data failed!\n" + e.message
file_errors[basename] = [error_message]
except Exception as e:
exception_occurred = True
file_err = "The Data file you selected could not be "
file_err += "loaded.\nMake sure the content of your file"
file_err += " is properly formatted.\n"
file_err += "When contacting the SasView team, mention the"
file_err += " following:\n"
file_err += e.message
file_errors[basename] = [file_err]
if len(file_errors) > 0:
error_message = ""
for filename, error_array in file_errors.iteritems():
error_message += "The following issues were found whilst "
error_message += "loading {}:\n".format(filename)
for message in error_array:
error_message += message + "\n"
error_message = error_message[:-1]
self.load_complete(output=output,
message=error_message,
info="error")
elif not exception_occurred: # Everything loaded as expected
self.load_complete(output=output, message="Loading data complete!",
info="info")
def load_update(self, message="", info="warning"):
"""
print update on the status bar
"""
if message != "":
wx.PostEvent(self.parent, StatusEvent(status=message,
info=info,
type="progress"))
def load_complete(self, output, message="", info="warning"):
"""
post message to status bar and return list of data
"""
wx.PostEvent(self.parent, StatusEvent(status=message,
info=info,
type="stop"))
if output is not None:
self.parent.add_data(data_list=output)