def test_write_samefile(self):
with tempfile.TemporaryDirectory() as tmpdirname:
exp = SweptTestExperiment()
wr1 = WriteToFile(tmpdirname + "/test_write_samefile.auspex",
groupname="group1")
wr2 = WriteToFile(tmpdirname + "/test_write_samefile.auspex",
groupname="group2")
edges = [(exp.voltage, wr1.sink), (exp.current, wr2.sink)]
exp.set_graph(edges)
exp.add_sweep(exp.field, np.linspace(0, 100.0, 4))
exp.add_sweep(exp.freq, np.linspace(0, 10.0, 3))
exp.run_sweeps()
container = AuspexDataContainer(tmpdirname +
"/test_write_samefile-0000.auspex")
data1, desc1 = container.open_dataset('group1', 'data')
data2, desc2 = container.open_dataset('group2', 'data')
self.assertTrue(
os.path.exists(tmpdirname +
"/test_write_samefile-0000.auspex"))
self.assertTrue(0.0 not in data1)
self.assertTrue(0.0 not in data2)
self.assertTrue(np.all(desc1['field'] == np.linspace(0, 100.0, 4)))
self.assertTrue(np.all(desc1['freq'] == np.linspace(0, 10.0, 3)))
self.assertTrue(np.all(desc1['samples'] == np.linspace(0, 4, 5)))
self.assertTrue(np.all(desc2['field'] == np.linspace(0, 100.0, 4)))
self.assertTrue(np.all(desc2['freq'] == np.linspace(0, 10.0, 3)))
self.assertTrue(desc1.axis('freq').unit == "Hz")
self.assertTrue(desc1.axis('freq').unit == "Hz")
def test_write_complex(self):
with tempfile.TemporaryDirectory() as tmpdirname:
exp = SweptTestExperiment()
exp.is_complex = True
wr = WriteToFile(tmpdirname + "/test_write_complex.auspex")
edges = [(exp.voltage, wr.sink)]
exp.set_graph(edges)
exp.voltage.descriptor.dtype = np.complex128
exp.update_descriptors()
exp.add_sweep(exp.field, np.linspace(0, 100.0, 4))
exp.add_sweep(exp.freq, np.linspace(0, 10.0, 3))
exp.run_sweeps()
self.assertTrue(
os.path.exists(tmpdirname + "/test_write_complex-0000.auspex"))
container = AuspexDataContainer(tmpdirname +
"/test_write_complex-0000.auspex")
data, desc = container.open_dataset('main', 'data')
self.assertTrue(0.0 not in data)
self.assertTrue(np.all(desc['field'] == np.linspace(0, 100.0, 4)))
self.assertTrue(np.all(desc['freq'] == np.linspace(0, 10.0, 3)))
self.assertTrue(np.all(desc['samples'] == np.linspace(0, 4, 5)))
self.assertTrue(desc.axis('freq').unit == "Hz")
self.assertTrue(data.dtype.type is np.complex128)
def test_write_metadata(self):
with tempfile.TemporaryDirectory() as tmpdirname:
exp = SweptTestExperimentMetadata()
wr = WriteToFile(tmpdirname + "/test_write_metadata.auspex")
edges = [(exp.voltage, wr.sink)]
exp.set_graph(edges)
exp.add_sweep(exp.field, np.linspace(0, 100.0, 4))
exp.add_sweep(exp.freq, np.linspace(0, 10.0, 3))
exp.run_sweeps()
self.assertTrue(
os.path.exists(tmpdirname +
"/test_write_metadata-0000.auspex"))
container = AuspexDataContainer(tmpdirname +
"/test_write_metadata-0000.auspex")
data, desc = container.open_dataset('main', 'data')
self.assertTrue(0.0 not in data)
self.assertTrue(np.all(desc['field'] == np.linspace(0, 100.0, 4)))
self.assertTrue(np.all(desc['freq'] == np.linspace(0, 10.0, 3)))
self.assertTrue(np.all(desc['samples'][:3] == [0.0, 1.0, 2.0]))
self.assertTrue(np.all(np.isnan(desc['samples'][3:])))
self.assertTrue(
np.all(
desc.axis('samples').metadata ==
["data", "data", "data", "0", "1"]))
class WriteToFile(Filter):
"""Writes data to file using the Auspex container type, which is a simple directory structure
with subdirectories, binary datafiles, and json meta files that store the axis descriptors
and other information."""
sink = InputConnector()
filename = FilenameParameter()
groupname = Parameter(default='main')
def __init__(self,
filename=None,
groupname=None,
datasetname='data',
**kwargs):
super(WriteToFile, self).__init__(**kwargs)
if filename:
self.filename.value = filename
if groupname:
self.groupname.value = groupname
if datasetname:
self.datasetname = datasetname
self.ret_queue = None # MP queue For returning data
def final_init(self):
assert self.filename.value, "Filename never supplied to writer."
assert self.groupname.value, "Groupname never supplied to writer."
assert self.datasetname, "Dataset name never supplied to writer."
self.descriptor = self.sink.input_streams[0].descriptor
self.container = AuspexDataContainer(self.filename.value)
self.group = self.container.new_group(self.groupname.value)
self.mmap = self.container.new_dataset(self.groupname.value,
self.datasetname,
self.descriptor)
self.w_idx = 0
self.points_taken = 0
def get_data_while_running(self, return_queue):
"""Return data to the main thread or user as requested. Use a MP queue to transmit."""
assert not self.done.is_set(), Exception(
"Experiment is over and filter done. Please use get_data")
self.return_queue.put(np.array(self.mmap))
def get_data(self):
assert self.done.is_set(), Exception(
"Experiment is still running. Please use get_data_while_running")
container = AuspexDataContainer(self.filename.value)
return container.open_dataset(self.groupname.value, self.datasetname)
def process_data(self, data):
# Write the data
self.mmap[self.w_idx:self.w_idx + data.size] = data
self.w_idx += data.size
self.points_taken = self.w_idx
def final_init(self):
assert self.filename.value, "Filename never supplied to writer."
assert self.groupname.value, "Groupname never supplied to writer."
assert self.datasetname, "Dataset name never supplied to writer."
self.descriptor = self.sink.input_streams[0].descriptor
self.container = AuspexDataContainer(self.filename.value)
self.group = self.container.new_group(self.groupname.value)
self.mmap = self.container.new_dataset(self.groupname.value,
self.datasetname,
self.descriptor)
self.w_idx = 0
self.points_taken = 0
def open_data(num=None, folder=None, groupname="main", datasetname="data", date=None):
"""Convenience Load data from an `AuspexDataContainer` given a file number and folder.
Assumes that files are named with the convention `ExperimentName-NNNNN.auspex`
Parameters:
num (int)
File number to be loaded.
folder (string)
Base folder where file is stored. If the `date` parameter is not None, assumes file is a dated folder. If no folder is specified, open a dialogue box. Open the folder with the desired ExperimentName-NNNN.auspex, then press OK
groupname (string)
Group name of data to be loaded.
datasetname (string, optional)
Data set name to be loaded. Default is "data".
date (string, optional)
Date folder from which data is to be loaded. Format is "YYMMDD" Defaults to today's date.
Returns:
data (numpy.array)
Data loaded from file.
desc (DataSetDescriptor)
Dataset descriptor loaded from file.
Examples:
Loading a data container
>>> data, desc = open_data(42, '/path/to/my/data', "q1-main", date="190301")
"""
if num is None or folder is None:
# pull up dialog box
data_file = get_file_name()
folder = ""
else:
if date == None:
date = datetime.date.today().strftime('%y%m%d')
folder = path.join(folder, date)
assert path.isdir(folder), f"Could not find data folder: {folder}"
p = re.compile(r".+-(\d+).auspex")
files = [x.name for x in os.scandir(folder) if x.is_dir()]
data_file = [x for x in files if p.match(x) and int(p.match(x).groups()[0]) == num]
if len(data_file) == 0:
raise ValueError("Could not find file!")
elif len(data_file) > 1:
raise ValueError(f"Ambiguous file information: found {data_file}")
data_container = AuspexDataContainer(path.join(folder, data_file[0]))
return data_container.open_dataset(groupname, datasetname)
def get_data(self):
assert self.done.is_set(), Exception(
"Experiment is still running. Please use get_data_while_running")
container = AuspexDataContainer(self.filename.value)
return container.open_dataset(self.groupname.value, self.datasetname)