def test_full_write(self):
formatter = GNUPlotFormat()
location = self.locations[0]
data = DataSet1D(name="test_full_write", location=location)
formatter.write(data, data.io, data.location)
with open(location + '/x_set.dat') as f:
self.assertEqual(f.read(), file_1d())
# check that we can add comment lines randomly into the file
# as long as it's after the first three lines, which are comments
# with well-defined meaning,
# and that we can un-quote the labels
lines = file_1d().split('\n')
lines[1] = lines[1].replace('"', '')
lines[3:3] = ['# this data is awesome!']
lines[6:6] = ['# the next point is my favorite.']
with open(location + '/x_set.dat', 'w') as f:
f.write('\n'.join(lines))
# normally this would be just done by data2 = load_data(location)
# but we want to work directly with the Formatter interface here
data2 = DataSet(location=location)
formatter.read(data2)
self.checkArraysEqual(data2.x_set, data.x_set)
self.checkArraysEqual(data2.y, data.y)
# data has been saved
self.assertEqual(data.y.last_saved_index, 4)
# data2 has been read back in, should show the same
# last_saved_index
self.assertEqual(data2.y.last_saved_index, 4)
# while we're here, check some errors on bad reads
# first: trying to read into a dataset that already has the
# wrong size
x = DataArray(name='x_set', label='X', preset_data=(1., 2.))
y = DataArray(name='y',
label='Y',
preset_data=(3., 4.),
set_arrays=(x, ))
data3 = new_data(arrays=(x, y), location=location + 'XX')
# initially give it a different location so we can make it without
# error, then change back to the location we want.
data3.location = location
with LogCapture() as logs:
formatter.read(data3)
self.assertTrue('ValueError' in logs.value, logs.value)
# no problem reading again if only data has changed, it gets
# overwritten with the disk copy
data2.x_set[2] = 42
data2.y[2] = 99
formatter.read(data2)
self.assertEqual(data2.x_set[2], 3)
self.assertEqual(data2.y[2], 5)
def test_format_options(self):
formatter = GNUPlotFormat(extension='.splat', terminator='\r',
separator=' ', comment='?:',
number_format='5.2f')
location = self.locations[0]
data = DataSet1D(location)
formatter.write(data, data.io, data.location)
# TODO - Python3 uses universal newlines for read and write...
# which means '\n' gets converted on write to the OS standard
# (os.linesep) and all of the options we support get converted
# back to '\n' on read. So I'm tempted to just take out terminator
# as an option rather than turn this feature off.
odd_format = '\n'.join([
'?:x_set y',
'?:"X" "Y"',
'?:5',
' 1.00 3.00',
' 2.00 4.00',
' 3.00 5.00',
' 4.00 6.00',
' 5.00 7.00', ''])
with open(location + '/x_set.splat', 'r') as f:
self.assertEqual(f.read(), odd_format)
def test_incremental_write(self):
formatter = GNUPlotFormat()
location = self.locations[0]
location2 = self.locations[1] # use 2nd location for reading back in
data = DataSet1D(location)
path = location + '/x_set.dat'
data_copy = DataSet1D(False)
# empty the data and mark it as unmodified
data.x_set[:] = float('nan')
data.y[:] = float('nan')
data.x_set.modified_range = None
data.y.modified_range = None
# simulate writing after every value comes in, even within
# one row (x comes first, it's the setpoint)
# we'll add a '*' after each write and check that they're
# in the right places afterward, ie we don't write any given
# row until it's done and we never totally rewrite the file
self.stars_before_write = 0
for i, (x, y) in enumerate(zip(data_copy.x_set, data_copy.y)):
data.x_set[i] = x
formatter.write(data, data.io, data.location)
formatter.write(data, data.io, location2)
self.add_star(path)
data.y[i] = y
formatter.write(data, data.io, data.location)
data.x_set.clear_save()
data.y.clear_save()
formatter.write(data, data.io, location2)
self.add_star(path)
# we wrote to a second location without the stars, so we can read
# back in and make sure that we get the right last_saved_index
# for the amount of data we've read.
reread_data = load_data(location=location2, data_manager=False,
formatter=formatter, io=data.io)
self.assertEqual(repr(reread_data.x_set.tolist()),
repr(data.x_set.tolist()))
self.assertEqual(repr(reread_data.y.tolist()),
repr(data.y.tolist()))
self.assertEqual(reread_data.x_set.last_saved_index, i)
self.assertEqual(reread_data.y.last_saved_index, i)
starred_file = '\n'.join([
'# x_set\ty',
'# "X"\t"Y"',
'# 5',
'1\t3',
'**2\t4',
'**3\t5',
'**4\t6',
'**5\t7', '*'])
with open(path, 'r') as f:
self.assertEqual(f.read(), starred_file)
self.assertEqual(self.stars_before_write, 1)
def test_constructor_errors(self):
with self.assertRaises(AttributeError):
# extension must be a string
GNUPlotFormat(extension=5)
with self.assertRaises(ValueError):
# terminator must be \r, \n, or \r\n
GNUPlotFormat(terminator='\n\r')
with self.assertRaises(ValueError):
# this is not CSV - separator must be whitespace
GNUPlotFormat(separator=',')
with self.assertRaises(ValueError):
GNUPlotFormat(comment=' \r\n\t ')
def test_read_errors(self):
formatter = GNUPlotFormat()
# non-comment line at the beginning
location = self.locations[0]
data = DataSet(location=location)
os.makedirs(location, exist_ok=True)
with open(location + '/x_set.dat', 'w') as f:
f.write('1\t2\n' + file_1d())
with LogCapture() as logs:
formatter.read(data)
self.assertTrue('ValueError' in logs.value, logs.value)
# same data array in 2 files
location = self.locations[1]
data = DataSet(location=location)
os.makedirs(location, exist_ok=True)
with open(location + '/x_set.dat', 'w') as f:
f.write('\n'.join(
['# x_set\ty', '# "X"\t"Y"', '# 2', '1\t2', '3\t4']))
with open(location + '/q.dat', 'w') as f:
f.write('\n'.join(['# q\ty', '# "Q"\t"Y"', '# 2', '1\t2', '3\t4']))
with LogCapture() as logs:
formatter.read(data)
self.assertTrue('ValueError' in logs.value, logs.value)
def test_multifile(self):
formatter = GNUPlotFormat()
location = self.locations[1]
data = DataSetCombined(location)
formatter.write(data, data.io, data.location)
filex, filexy = files_combined()
with open(location + '/x_set.dat') as f:
self.assertEqual(f.read(), filex)
with open(location + '/x_set_y_set.dat') as f:
self.assertEqual(f.read(), filexy)
data2 = DataSet(location=location)
formatter.read(data2)
for array_id in ('x_set', 'y1', 'y2', 'y_set', 'z1', 'z2'):
self.checkArraysEqual(data2.arrays[array_id],
data.arrays[array_id])
def load_dataset(location, io=None, verbose=0):
""" Load a dataset from storage
An attempt is made to automatically detect the formatter. Supported are currently GNUPlotFormat and HDF5Format
Args:
location (str): either the relative or full location
io (None or qcodes.DiskIO):
Returns:
dataset (DataSet or None)
"""
if io is None:
io = qcodes.DataSet.default_io
formatters = [qcodes.DataSet.default_formatter]
try:
from qcodes.data.hdf5_format_hickle import HDF5FormatHickle
formatters += [HDF5FormatHickle()]
except:
pass
from qcodes.data.hdf5_format import HDF5Format
formatters += [HDF5Format()]
from qcodes.data.gnuplot_format import GNUPlotFormat
formatters += [GNUPlotFormat()]
data = None
for ii, hformatter in enumerate(formatters):
try:
if verbose:
print('%d: %s' % (ii, hformatter))
data = qcodes.load_data(location, formatter=hformatter, io=io)
if len(data.arrays) == 0:
data = None
raise Exception(
'empty dataset, probably a HDF5 format misread by GNUPlotFormat'
)
logging.debug('load_data: loaded %s with %s' %
(location, hformatter))
except Exception as ex:
logging.info(
'load_data: location %s: failed for formatter %d: %s' %
(location, ii, hformatter))
if verbose:
print(ex)
pass
finally:
if data is not None:
if isinstance(hformatter, GNUPlotFormat):
# workaround for bug in GNUPlotFormat not saving the units
if '__dataset_metadata' in data.metadata:
dataset_meta = data.metadata['__dataset_metadata']
for key, array_metadata in dataset_meta[
'arrays'].items():
if key in data.arrays:
if data.arrays[key].unit is None:
if verbose:
print(
'load_dataset: updating unit for %s'
% key)
data.arrays[key].unit = array_metadata[
'unit']
if verbose:
print('succes with formatter %s' % hformatter)
break
if verbose:
if data is None:
print('could not load data from %s, returning None' % location)
return data
def load_dataset(location, io=None, verbose=0):
""" Load a dataset from storage
An attempt is made to automatically detect the formatter. Supported are currently qcodes GNUPlotFormat,
qcodes HDF5Format and json format.
Args:
location (str): either the relative or full location
io (None or qcodes.data.io.DiskIO):
Returns:
dataset (DataSet or None)
"""
if io is None:
io = DataSet.default_io
formatters = [DataSet.default_formatter]
from qcodes.data.hdf5_format import HDF5FormatMetadata
formatters += [HDF5FormatMetadata()]
try:
from qcodes.data.hdf5_format_hickle import HDF5FormatHickle
formatters += [HDF5FormatHickle()]
except ImportError as ex:
logging.info(f'HDF5FormatHickle not available {ex}')
from qcodes.data.hdf5_format import HDF5Format
formatters += [HDF5Format()]
from qcodes.data.gnuplot_format import GNUPlotFormat
formatters += [GNUPlotFormat()]
data = None
if location.endswith('.json'):
dataset_dictionary = qtt.utilities.json_serializer.load_json(location)
data = qtt.data.dictionary_to_dataset(dataset_dictionary)
else:
# assume we have a QCoDeS dataset
for ii, hformatter in enumerate(formatters):
try:
if verbose:
print('%d: %s' % (ii, hformatter))
data = qcodes.data.data_set.load_data(location,
formatter=hformatter,
io=io)
if len(data.arrays) == 0:
data = None
raise Exception(
'empty dataset, probably a HDF5 format misread by GNUPlotFormat'
)
logging.debug('load_data: loaded %s with %s' %
(location, hformatter))
except Exception as ex:
logging.info(
'load_data: location %s: failed for formatter %d: %s' %
(location, ii, hformatter))
if verbose:
print(ex)
finally:
if data is not None:
if isinstance(hformatter, GNUPlotFormat):
# workaround for bug in GNUPlotFormat not saving the units
if '__dataset_metadata' in data.metadata:
dataset_meta = data.metadata['__dataset_metadata']
for key, array_metadata in dataset_meta[
'arrays'].items():
if key in data.arrays:
if data.arrays[key].unit is None:
if verbose:
print(
'load_dataset: updating unit for %s'
% key)
data.arrays[key].unit = array_metadata[
'unit']
if verbose:
print('succes with formatter %s' % hformatter)
break
if verbose:
if data is None:
print('could not load data from %s, returning None' % location)
return data
