def test_full_write(self): formatter = GNUPlotFormat() location = self.locations[0] data = DataSet1D(name="test_full_write", location=location) formatter.write(data, data.io, data.location) with open(location + '/x_set.dat') as f: self.assertEqual(f.read(), file_1d()) # check that we can add comment lines randomly into the file # as long as it's after the first three lines, which are comments # with well-defined meaning, # and that we can un-quote the labels lines = file_1d().split('\n') lines[1] = lines[1].replace('"', '') lines[3:3] = ['# this data is awesome!'] lines[6:6] = ['# the next point is my favorite.'] with open(location + '/x_set.dat', 'w') as f: f.write('\n'.join(lines)) # normally this would be just done by data2 = load_data(location) # but we want to work directly with the Formatter interface here data2 = DataSet(location=location) formatter.read(data2) self.checkArraysEqual(data2.x_set, data.x_set) self.checkArraysEqual(data2.y, data.y) # data has been saved self.assertEqual(data.y.last_saved_index, 4) # data2 has been read back in, should show the same # last_saved_index self.assertEqual(data2.y.last_saved_index, 4) # while we're here, check some errors on bad reads # first: trying to read into a dataset that already has the # wrong size x = DataArray(name='x_set', label='X', preset_data=(1., 2.)) y = DataArray(name='y', label='Y', preset_data=(3., 4.), set_arrays=(x, )) data3 = new_data(arrays=(x, y), location=location + 'XX') # initially give it a different location so we can make it without # error, then change back to the location we want. data3.location = location with LogCapture() as logs: formatter.read(data3) self.assertTrue('ValueError' in logs.value, logs.value) # no problem reading again if only data has changed, it gets # overwritten with the disk copy data2.x_set[2] = 42 data2.y[2] = 99 formatter.read(data2) self.assertEqual(data2.x_set[2], 3) self.assertEqual(data2.y[2], 5)
def test_format_options(self): formatter = GNUPlotFormat(extension='.splat', terminator='\r', separator=' ', comment='?:', number_format='5.2f') location = self.locations[0] data = DataSet1D(location) formatter.write(data, data.io, data.location) # TODO - Python3 uses universal newlines for read and write... # which means '\n' gets converted on write to the OS standard # (os.linesep) and all of the options we support get converted # back to '\n' on read. So I'm tempted to just take out terminator # as an option rather than turn this feature off. odd_format = '\n'.join([ '?:x_set y', '?:"X" "Y"', '?:5', ' 1.00 3.00', ' 2.00 4.00', ' 3.00 5.00', ' 4.00 6.00', ' 5.00 7.00', '']) with open(location + '/x_set.splat', 'r') as f: self.assertEqual(f.read(), odd_format)
def test_incremental_write(self): formatter = GNUPlotFormat() location = self.locations[0] location2 = self.locations[1] # use 2nd location for reading back in data = DataSet1D(location) path = location + '/x_set.dat' data_copy = DataSet1D(False) # empty the data and mark it as unmodified data.x_set[:] = float('nan') data.y[:] = float('nan') data.x_set.modified_range = None data.y.modified_range = None # simulate writing after every value comes in, even within # one row (x comes first, it's the setpoint) # we'll add a '*' after each write and check that they're # in the right places afterward, ie we don't write any given # row until it's done and we never totally rewrite the file self.stars_before_write = 0 for i, (x, y) in enumerate(zip(data_copy.x_set, data_copy.y)): data.x_set[i] = x formatter.write(data, data.io, data.location) formatter.write(data, data.io, location2) self.add_star(path) data.y[i] = y formatter.write(data, data.io, data.location) data.x_set.clear_save() data.y.clear_save() formatter.write(data, data.io, location2) self.add_star(path) # we wrote to a second location without the stars, so we can read # back in and make sure that we get the right last_saved_index # for the amount of data we've read. reread_data = load_data(location=location2, data_manager=False, formatter=formatter, io=data.io) self.assertEqual(repr(reread_data.x_set.tolist()), repr(data.x_set.tolist())) self.assertEqual(repr(reread_data.y.tolist()), repr(data.y.tolist())) self.assertEqual(reread_data.x_set.last_saved_index, i) self.assertEqual(reread_data.y.last_saved_index, i) starred_file = '\n'.join([ '# x_set\ty', '# "X"\t"Y"', '# 5', '1\t3', '**2\t4', '**3\t5', '**4\t6', '**5\t7', '*']) with open(path, 'r') as f: self.assertEqual(f.read(), starred_file) self.assertEqual(self.stars_before_write, 1)
def test_constructor_errors(self): with self.assertRaises(AttributeError): # extension must be a string GNUPlotFormat(extension=5) with self.assertRaises(ValueError): # terminator must be \r, \n, or \r\n GNUPlotFormat(terminator='\n\r') with self.assertRaises(ValueError): # this is not CSV - separator must be whitespace GNUPlotFormat(separator=',') with self.assertRaises(ValueError): GNUPlotFormat(comment=' \r\n\t ')
def test_read_errors(self): formatter = GNUPlotFormat() # non-comment line at the beginning location = self.locations[0] data = DataSet(location=location) os.makedirs(location, exist_ok=True) with open(location + '/x_set.dat', 'w') as f: f.write('1\t2\n' + file_1d()) with LogCapture() as logs: formatter.read(data) self.assertTrue('ValueError' in logs.value, logs.value) # same data array in 2 files location = self.locations[1] data = DataSet(location=location) os.makedirs(location, exist_ok=True) with open(location + '/x_set.dat', 'w') as f: f.write('\n'.join( ['# x_set\ty', '# "X"\t"Y"', '# 2', '1\t2', '3\t4'])) with open(location + '/q.dat', 'w') as f: f.write('\n'.join(['# q\ty', '# "Q"\t"Y"', '# 2', '1\t2', '3\t4'])) with LogCapture() as logs: formatter.read(data) self.assertTrue('ValueError' in logs.value, logs.value)
def test_multifile(self): formatter = GNUPlotFormat() location = self.locations[1] data = DataSetCombined(location) formatter.write(data, data.io, data.location) filex, filexy = files_combined() with open(location + '/x_set.dat') as f: self.assertEqual(f.read(), filex) with open(location + '/x_set_y_set.dat') as f: self.assertEqual(f.read(), filexy) data2 = DataSet(location=location) formatter.read(data2) for array_id in ('x_set', 'y1', 'y2', 'y_set', 'z1', 'z2'): self.checkArraysEqual(data2.arrays[array_id], data.arrays[array_id])
def load_dataset(location, io=None, verbose=0): """ Load a dataset from storage An attempt is made to automatically detect the formatter. Supported are currently GNUPlotFormat and HDF5Format Args: location (str): either the relative or full location io (None or qcodes.DiskIO): Returns: dataset (DataSet or None) """ if io is None: io = qcodes.DataSet.default_io formatters = [qcodes.DataSet.default_formatter] try: from qcodes.data.hdf5_format_hickle import HDF5FormatHickle formatters += [HDF5FormatHickle()] except: pass from qcodes.data.hdf5_format import HDF5Format formatters += [HDF5Format()] from qcodes.data.gnuplot_format import GNUPlotFormat formatters += [GNUPlotFormat()] data = None for ii, hformatter in enumerate(formatters): try: if verbose: print('%d: %s' % (ii, hformatter)) data = qcodes.load_data(location, formatter=hformatter, io=io) if len(data.arrays) == 0: data = None raise Exception( 'empty dataset, probably a HDF5 format misread by GNUPlotFormat' ) logging.debug('load_data: loaded %s with %s' % (location, hformatter)) except Exception as ex: logging.info( 'load_data: location %s: failed for formatter %d: %s' % (location, ii, hformatter)) if verbose: print(ex) pass finally: if data is not None: if isinstance(hformatter, GNUPlotFormat): # workaround for bug in GNUPlotFormat not saving the units if '__dataset_metadata' in data.metadata: dataset_meta = data.metadata['__dataset_metadata'] for key, array_metadata in dataset_meta[ 'arrays'].items(): if key in data.arrays: if data.arrays[key].unit is None: if verbose: print( 'load_dataset: updating unit for %s' % key) data.arrays[key].unit = array_metadata[ 'unit'] if verbose: print('succes with formatter %s' % hformatter) break if verbose: if data is None: print('could not load data from %s, returning None' % location) return data
def load_dataset(location, io=None, verbose=0): """ Load a dataset from storage An attempt is made to automatically detect the formatter. Supported are currently qcodes GNUPlotFormat, qcodes HDF5Format and json format. Args: location (str): either the relative or full location io (None or qcodes.data.io.DiskIO): Returns: dataset (DataSet or None) """ if io is None: io = DataSet.default_io formatters = [DataSet.default_formatter] from qcodes.data.hdf5_format import HDF5FormatMetadata formatters += [HDF5FormatMetadata()] try: from qcodes.data.hdf5_format_hickle import HDF5FormatHickle formatters += [HDF5FormatHickle()] except ImportError as ex: logging.info(f'HDF5FormatHickle not available {ex}') from qcodes.data.hdf5_format import HDF5Format formatters += [HDF5Format()] from qcodes.data.gnuplot_format import GNUPlotFormat formatters += [GNUPlotFormat()] data = None if location.endswith('.json'): dataset_dictionary = qtt.utilities.json_serializer.load_json(location) data = qtt.data.dictionary_to_dataset(dataset_dictionary) else: # assume we have a QCoDeS dataset for ii, hformatter in enumerate(formatters): try: if verbose: print('%d: %s' % (ii, hformatter)) data = qcodes.data.data_set.load_data(location, formatter=hformatter, io=io) if len(data.arrays) == 0: data = None raise Exception( 'empty dataset, probably a HDF5 format misread by GNUPlotFormat' ) logging.debug('load_data: loaded %s with %s' % (location, hformatter)) except Exception as ex: logging.info( 'load_data: location %s: failed for formatter %d: %s' % (location, ii, hformatter)) if verbose: print(ex) finally: if data is not None: if isinstance(hformatter, GNUPlotFormat): # workaround for bug in GNUPlotFormat not saving the units if '__dataset_metadata' in data.metadata: dataset_meta = data.metadata['__dataset_metadata'] for key, array_metadata in dataset_meta[ 'arrays'].items(): if key in data.arrays: if data.arrays[key].unit is None: if verbose: print( 'load_dataset: updating unit for %s' % key) data.arrays[key].unit = array_metadata[ 'unit'] if verbose: print('succes with formatter %s' % hformatter) break if verbose: if data is None: print('could not load data from %s, returning None' % location) return data