def test_histogram_with_ui(qtbot):
dataset = _make_testdata()
assert dataset.validate()
hist, edges = histogram(dataset.data_vals('noise'), axis=1, bins=15)
Histogrammer.useUi = True
fc = linearFlowchart(('h', Histogrammer))
win = AutoPlotMainWindow(fc, loaderName=None, monitor=False)
win.show()
qtbot.addWidget(win)
fc.setInput(dataIn=dataset)
hnode = fc.nodes()['h']
# emit signal manually right now, since that's what's connected.
# setting value alone won't emit the change.
hnode.ui.widget.nbins.setValue(15)
hnode.ui.widget.nbins.editingFinished.emit()
hnode.ui.widget.combo.setCurrentText('y')
assert fc.outputValues()['dataOut'].dependents() == ['noise_count']
assert fc.outputValues()['dataOut'].axes('noise_count') == \
['x', 'z', 'noise']
assert arrays_equal(fc.outputValues()['dataOut']['noise_count']['values'],
hist)
def main():
fc = linearFlowchart(('dummy', DummyNode))
node = fc.nodes()['dummy']
dialog = widgetDialog(node.ui, title='dummy node')
data = testdata.get_2d_scalar_cos_data(2, 2, 1)
fc.setInput(dataIn=data)
return dialog, fc
def test_data_extraction(qtbot):
"""
Test whether extraction of one dependent gives the right data back.
"""
DataSelector.useUi = False
data = testdata.three_compatible_3d_sets()
data_name = data.dependents()[0]
field_names = [data_name] + data.axes(data_name)
fc = linearFlowchart(('selector', DataSelector))
node = fc.nodes()['selector']
fc.setInput(dataIn=data)
node.selectedData = data_name
out = fc.output()['dataOut']
assert out.dependents() == [data_name]
for d, _ in out.data_items():
assert d in field_names
assert np.all(np.isclose(
data.data_vals(data_name), out.data_vals(data_name),
atol=1e-15
))
def test_average_subtraction(qtbot):
"""Test the subtract average filter node"""
SubtractAverage.useUi = False
SubtractAverage.uiClass = None
fc = linearFlowchart(('Subtract Average', SubtractAverage), )
node = fc.nodes()['Subtract Average']
x = np.arange(11) - 5.
y = np.linspace(0, 10, 51)
xx, yy = np.meshgrid(x, y, indexing='ij')
zz = np.sin(yy) + xx
zz_ref_avg_y = np.sin(yy) - np.sin(yy).mean()
data = MeshgridDataDict(x=dict(values=xx),
y=dict(values=yy),
z=dict(values=zz, axes=['x', 'y']))
assert data.validate()
fc.setInput(dataIn=data)
assert num.arrays_equal(zz, fc.outputValues()['dataOut'].data_vals('z'))
node.averagingAxis = 'y'
assert num.arrays_equal(
zz_ref_avg_y,
fc.outputValues()['dataOut'].data_vals('z'),
)
def test_qcodes_flow_shaped_data(qtbot, dataset_with_shape):
fc = linearFlowchart(
('Data loader', QCodesDSLoader),
('Data selection', DataSelector),
('Grid', DataGridder),
)
loader = fc.nodes()['Data loader']
selector = fc.nodes()['Data selection']
selector.selectedData = 'z_0'
gridder = fc.nodes()['Grid']
gridder.grid = (GridOption.metadataShape, {})
loader.pathAndId = dataset_with_shape.path_to_db, dataset_with_shape.run_id
loader.update()
expected_shape = dataset_with_shape.description.shapes['z_0']
datadict = fc.output()['dataOut']
for key in ('x', 'y', 'z_0'):
assert datadict[key]['values'].shape == expected_shape
assert datadict.shapes()[key] == expected_shape
assert_array_equal(datadict[key]['values'],
dataset_with_shape.get_parameter_data()['z_0'][key])
assert datadict.shape() == expected_shape
def test_set_grid_with_order(qtbot):
"""Test making meshgrid when the internal axis order needs to be fixed."""
DataGridder.useUi = False
DataGridder.uiClass = None
fc = linearFlowchart(('grid', DataGridder))
node = fc.nodes()['grid']
x = np.arange(5.0)
y = np.linspace(0, 1, 5)
z = np.arange(4.0, 6.0, 1.0)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vv = xx * yy * zz
x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten()
v1d = vv.flatten()
# construct data dict, with axes for vals not conforming to the
# correct order with which we've generated the data
data = DataDict(x=dict(values=x1d),
y=dict(values=y1d),
z=dict(values=z1d),
vals=dict(values=v1d, axes=['y', 'z', 'x']))
assert data.validate()
# in the 1-d data, nothing unusual should happen
fc.setInput(dataIn=data)
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
v1d,
)
# guessing the grid should work, and fix the wrong order
node.grid = GridOption.guessShape, dict()
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
vv.transpose((1, 2, 0)),
)
assert fc.outputValues()['dataOut']['vals']['axes'] == ['y', 'z', 'x']
# finally, specify manually. omitting inner shape doesn't work
node.grid = GridOption.specifyShape, dict(shape=(5, 2, 5))
assert fc.outputValues()['dataOut'].data_vals('vals').shape == (5, 2, 5)
assert not num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
vv.transpose((1, 2, 0)),
)
# but using the right inner axis order should do it
node.grid = GridOption.specifyShape, dict(order=['x', 'y', 'z'],
shape=(5, 5, 2))
assert fc.outputValues()['dataOut'].data_vals('vals').shape == (5, 2, 5)
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
vv.transpose((1, 2, 0)),
)
def test_update_qcloader(qtbot):
qc.config.core.db_location = DBPATH
initialise_database()
exp = load_or_create_experiment('2d_softsweep', sample_name='no sample')
# define test data
x = np.linspace(0, 1., 5)
y = np.linspace(0, 1., 5)
xx, yy = np.meshgrid(x, y, indexing='ij')
zz = np.random.rand(*xx.shape)
def get_2dsoftsweep_results():
for x, y, z in zip(xx.reshape(-1), yy.reshape(-1), zz.reshape(-1)):
yield dict(x=x, y=y, z=z)
# create data set
_ds = new_data_set('2d_softsweep', exp_id=exp.exp_id,
specs=[ParamSpec('x', 'numeric', unit='A'),
ParamSpec('y', 'numeric', unit='B'),
ParamSpec('z', 'numeric', unit='C',
depends_on=['x', 'y']), ], )
run_id = _ds.run_id
results = get_2dsoftsweep_results()
# setting up the flowchart
fc = linearFlowchart(('loader', QCodesDSLoader))
loader = fc.nodes()['loader']
loader.pathAndId = DBPATH, run_id
def check():
nresults = _ds.number_of_results
loader.update()
ddict = fc.output()['dataOut']
if ddict is not None and nresults > 0:
z_in = zz.reshape(-1)[:nresults]
z_out = ddict.data_vals('z')
if z_out is not None:
assert z_in.size == z_out.size
assert np.allclose(z_in, z_out, atol=1e-15)
# insert data in small chunks, and check
while True:
try:
ninsertions = np.random.randint(0, 5)
for n in range(ninsertions):
_ds.add_result(next(results))
except StopIteration:
_ds.mark_complete()
break
check()
check()
def test_loader_node(qtbot):
dds.DDH5Loader.useUi = False
x = np.arange(3)
y = np.repeat(np.linspace(0, 1, 5).reshape(1, -1), 3, 0)
z = np.arange(y.size).reshape(y.shape)
data = dd.DataDict(
x=dict(values=x,
unit='A',
__info__='careful!',
__moreinfo__='more words in here'),
y=dict(values=y, unit='B'),
z=dict(values=z, axes=['x', 'y'], unit='C'),
__desc__='some description',
)
assert data.validate()
dds.datadict_to_hdf5(data, str(FILEPATH), append_mode=dds.AppendMode.new)
assert _clean_from_file(dds.datadict_from_hdf5(str(FILEPATH))) == data
fc = linearFlowchart(('loader', dds.DDH5Loader))
node = fc.nodes()['loader']
assert fc.outputValues()['dataOut'] is None
with qtbot.waitSignal(node.loadingWorker.dataLoaded,
timeout=1000) as blocker:
node.filepath = str(FILEPATH)
out = fc.outputValues()['dataOut'].copy()
out.pop('__title__')
assert _clean_from_file(out) == data
data.add_data(x=[3],
y=np.linspace(0, 1, 5).reshape(1, -1),
z=np.arange(5).reshape(1, -1))
dds.datadict_to_hdf5(data, str(FILEPATH), append_mode=dds.AppendMode.new)
assert _clean_from_file(dds.datadict_from_hdf5(str(FILEPATH))) == data
out = fc.outputValues()['dataOut'].copy()
out.pop('__title__')
assert not _clean_from_file(out) == data
with qtbot.waitSignal(node.loadingWorker.dataLoaded,
timeout=1000) as blocker:
node.update()
out = fc.outputValues()['dataOut'].copy()
out.pop('__title__')
assert _clean_from_file(out) == data
FILEPATH.unlink()
def xySelection(interactive=False):
if not interactive:
app = QtGui.QApplication([])
fc = linearFlowchart(('xysel', XYSelector))
selector = fc.nodes()['xysel']
dialog = widgetDialog(selector.ui, 'xysel')
data = datadict_to_meshgrid(testdata.three_compatible_3d_sets(4, 4, 4))
fc.setInput(dataIn=data)
if not interactive:
app.exec_()
else:
return dialog, fc
def dimReduction(interactive=False):
if not interactive:
app = QtGui.QApplication([])
fc = linearFlowchart(('reducer', DimensionReducer))
reducer = fc.nodes()['reducer']
dialog = widgetDialog(reducer.ui, 'reducer')
data = datadict_to_meshgrid(testdata.three_compatible_3d_sets(2, 2, 2))
fc.setInput(dataIn=data)
if not interactive:
app.exec_()
else:
return dialog, fc
def test_xy_selector_with_roles(qtbot):
"""Testing XY selector using the roles 'meta' property."""
XYSelector.uiClass = None
fc = linearFlowchart(('xysel', XYSelector))
node = fc.nodes()['xysel']
x = np.arange(5.0)
y = np.linspace(0, 1, 5)
z = np.arange(4.0, 6.0, 1.0)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vals = xx * yy * zz
data = MeshgridDataDict(x=dict(values=xx),
y=dict(values=yy),
z=dict(values=zz),
vals=dict(values=vals, axes=['x', 'y', 'z']))
assert data.validate()
fc.setInput(dataIn=data)
# this should return None, because no x/y axes were set.
assert fc.outputValues()['dataOut'] is None
# now select two axes, and test that the other one is correctly selected
node.xyAxes = ('x', 'y')
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals[:, :, 0])
assert node.dimensionRoles == {
'x': 'x-axis',
'y': 'y-axis',
'z': (ReductionMethod.elementSelection, [], {
'index': 0,
'axis': 2
})
}
# now set the role directly through the meta property
node.dimensionRoles = {
'x': 'y-axis',
'y': (ReductionMethod.average, [], {}),
'z': 'x-axis',
}
assert node.xyAxes == ('z', 'x')
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals[:, :, :].mean(axis=1).transpose((1, 0)))
def loader_node(interactive=False):
def cb(*vals):
print(vals)
if not interactive:
app = QtWidgets.QApplication([])
fc = linearFlowchart(('loader', dds.DDH5Loader))
loader = fc.nodes()['loader']
dialog = widgetDialog(loader.ui, 'loader')
if not interactive:
loader.newDataStructure.connect(cb)
app.exec_()
else:
return dialog, fc
def main(pathAndId):
app = QtGui.QApplication([])
# flowchart and window
fc = linearFlowchart(
('Dataset loader', QCodesDSLoader),
('Data selection', DataSelector),
('Grid', DataGridder),
('Dimension assignment', XYSelector),
('Sine fit', sinefit),
('plot', PlotNode),
)
win = QCAutoPlotMainWindow(fc, pathAndId=pathAndId)
win.show()
return app.exec_()
def test_basic_flowchart_and_nodes(qtbot):
fc = flowchart()
node = Node(name='node')
fc.addNode(node, name=node.name())
fc.connectTerminals(fc['dataIn'], node['dataIn'])
fc.connectTerminals(node['dataOut'], fc['dataOut'])
fc.setInput(dataIn='abcdef')
assert fc.outputValues() == dict(dataOut='abcdef')
for i in range(3):
lst = [(f'node{j}', Node) for j in range(i)]
fc = linearFlowchart(*lst)
fc.setInput(dataIn='abcdef')
assert fc.outputValues() == dict(dataOut='abcdef')
def test_real_histogram(qtbot):
dataset = _make_testdata()
assert dataset.validate()
hist, edges = histogram(dataset.data_vals('noise'), axis=0, bins=10)
Histogrammer.useUi = False
fc = linearFlowchart(('h', Histogrammer))
fc.setInput(dataIn=dataset)
assert fc.outputValues()['dataOut'] == dataset
fc.nodes()['h'].nbins = 10
fc.nodes()['h'].histogramAxis = 'x'
assert fc.outputValues()['dataOut'].dependents() == ['noise_count']
assert fc.outputValues()['dataOut'].axes('noise_count') == \
['y', 'z', 'noise']
assert arrays_equal(fc.outputValues()['dataOut']['noise_count']['values'],
hist)
def test_incompatible_sets(qtbot):
"""
Test that selecting incompatible data sets give None output.
"""
DataSelector.useUi = False
data = testdata.three_incompatible_3d_sets()
fc = linearFlowchart(('selector', DataSelector))
node = fc.nodes()['selector']
fc.setInput(dataIn=data)
node.selectedData = data.dependents()[0], data.dependents()[1]
assert fc.output()['dataOut'] == None
node.selectedData = data.dependents()[0]
assert fc.output()['dataOut'].dependents() == [data.dependents()[0]]
node.selectedData = data.dependents()[1]
assert fc.output()['dataOut'].dependents() == [data.dependents()[1]]
def test_data_selector():
fc = linearFlowchart(('selector', DataSelector))
selector = fc.nodes()['selector']
dialog = widgetDialog(selector.ui, title='selector')
data = testdata.three_incompatible_3d_sets(2, 2, 2)
fc.setInput(dataIn=data)
selector.selectedData = ['data']
# for testing purposes, insert differently structured data
data2 = testdata.two_compatible_noisy_2d_sets()
fc.setInput(dataIn=data2)
# ... and go back.
fc.setInput(dataIn=data)
selector.selectedData = ['data']
return dialog, fc
def gridder(interactive=False):
def cb(val):
print(val)
if not interactive:
app = QtGui.QApplication([])
fc = linearFlowchart(('grid', DataGridder))
gridder = fc.nodes()['grid']
dialog = widgetDialog(gridder.ui, 'gridder')
data = testdata.three_compatible_3d_sets(2, 2, 2)
fc.setInput(dataIn=data)
if not interactive:
gridder.shapeDetermined.connect(cb)
app.exec_()
else:
return dialog, fc
def test_update_qcloader(qtbot, empty_db_path):
db_path = empty_db_path
exp = load_or_create_experiment('2d_softsweep', sample_name='no sample')
N = 2
m = qc.Measurement(exp=exp)
m.register_custom_parameter('x')
m.register_custom_parameter('y')
dd_expected = DataDict(x=dict(values=np.array([])),
y=dict(values=np.array([])))
for n in range(N):
m.register_custom_parameter(f'z_{n}', setpoints=['x', 'y'])
dd_expected[f'z_{n}'] = dict(values=np.array([]), axes=['x', 'y'])
dd_expected.validate()
# setting up the flowchart
fc = linearFlowchart(('loader', QCodesDSLoader))
loader = fc.nodes()['loader']
def check():
nresults = ds.number_of_results
loader.update()
ddict = fc.output()['dataOut']
if ddict is not None and nresults > 0:
z_in = dd_expected.data_vals('z_1')
z_out = ddict.data_vals('z_1')
if z_out is not None:
assert z_in.size == z_out.size
assert np.allclose(z_in, z_out, atol=1e-15)
with m.run() as datasaver:
ds = datasaver.dataset
run_id = datasaver.dataset.captured_run_id
loader.pathAndId = db_path, run_id
for result in testdata.generate_2d_scalar_simple(3, 3, N):
row = [(k, v) for k, v in result.items()]
datasaver.add_result(*row)
dd_expected.add_data(**result)
check()
check()
def test_GridNode():
def cb(val):
print(val)
fc = linearFlowchart(('grid', DataGridder))
gridder = fc.nodes()['grid']
dialog = widgetDialog(gridder.ui, 'gridder')
data = testdata.three_compatible_3d_sets(2, 2, 2)
fc.setInput(dataIn=data)
gridder.shapeDetermined.connect(cb)
gridder.grid = GridOption.guessShape, {}
gridder.grid = GridOption.specifyShape, \
dict(order=['x', 'y', 'z'], shape=(2,2,3))
gridder.grid = GridOption.guessShape, {}
gridder.grid = GridOption.specifyShape, \
dict(order=['x', 'y', 'z'], shape=(2,2,3))
return dialog, fc
def test_basic_gridding(qtbot):
"""Test simple gridding tasks"""
DataGridder.useUi = False
DataGridder.uiClass = None
fc = linearFlowchart(('grid', DataGridder))
node = fc.nodes()['grid']
x = np.arange(5.0)
y = np.linspace(0, 1, 5)
z = np.arange(4.0, 6.0, 1.0)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vv = xx * yy * zz
x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten()
v1d = vv.flatten()
data = DataDict(x=dict(values=x1d),
y=dict(values=y1d),
z=dict(values=z1d),
vals=dict(values=v1d, axes=['x', 'y', 'z']))
assert data.validate()
fc.setInput(dataIn=data)
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
v1d,
)
node.grid = GridOption.guessShape, dict()
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
vv,
)
node.grid = GridOption.specifyShape, dict(shape=(5, 5, 2))
assert num.arrays_equal(
fc.outputValues()['dataOut'].data_vals('vals'),
vv,
)
def test_reduction(qtbot):
"""Test basic dimension reduction."""
DimensionReducer.uiClass = None
fc = linearFlowchart(('dim_red', DimensionReducer))
node = fc.nodes()['dim_red']
x = np.arange(5.0)
y = np.linspace(0, 1, 5)
z = np.arange(4.0, 6.0, 1.0)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vals = xx * yy * zz
data = MeshgridDataDict(x=dict(values=xx),
y=dict(values=yy),
z=dict(values=zz),
vals=dict(values=vals, axes=['x', 'y', 'z']))
assert data.validate()
fc.setInput(dataIn=data)
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals)
node.reductions = {'y': (np.mean, [], {})}
out = fc.outputValues()['dataOut']
assert num.arrays_equal(vals.mean(axis=1), out.data_vals('vals'))
assert out.axes('vals') == ['x', 'z']
node.reductions = {
'y': (ReductionMethod.elementSelection, [], {
'index': 0
}),
'z': (ReductionMethod.average, )
}
out = fc.outputValues()['dataOut']
assert num.arrays_equal(vals[:, 0, :].mean(axis=-1), out.data_vals('vals'))
assert out.axes('vals') == ['x']
def test_xy_selector(qtbot):
"""Basic XY selector node test."""
XYSelector.uiClass = None
fc = linearFlowchart(('xysel', XYSelector))
node = fc.nodes()['xysel']
x = np.arange(5.0)
y = np.linspace(0, 1, 5)
z = np.arange(4.0, 6.0, 1.0)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vals = xx * yy * zz
data = MeshgridDataDict(x=dict(values=xx),
y=dict(values=yy),
z=dict(values=zz),
vals=dict(values=vals, axes=['x', 'y', 'z']))
assert data.validate()
fc.setInput(dataIn=data)
# this should return None, because no x/y axes were set.
assert fc.outputValues()['dataOut'] is None
# now select two axes, and test that the other one is correctly selected
node.xyAxes = ('x', 'y')
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals[:, :, 0])
# try a different reduction on the third axis
node.reductions = {'z': (ReductionMethod.average, [], {})}
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals.mean(axis=-1))
# Test transposing the data by flipping x/y
node.xyAxes = ('y', 'x')
assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'),
vals.mean(axis=-1).transpose((1, 0)))
def test_data_selector(interactive=True):
if not interactive:
app = QtGui.QApplication([])
fc = linearFlowchart(('selector', DataSelector))
selector = fc.nodes()['selector']
dialog = widgetDialog(selector.ui, 'selector')
data = testdata.three_incompatible_3d_sets(2, 2, 2)
fc.setInput(dataIn=data)
selector.selectedData = ['data']
# for testing purposes, insert differently structured data
data2 = testdata.two_compatible_noisy_2d_sets()
fc.setInput(dataIn=data2)
# ... and go back.
fc.setInput(dataIn=data)
selector.selectedData = ['data']
if not interactive:
app.exec_()
else:
return dialog, fc
def test_basic_scale_units(qtbot):
ScaleUnits.useUi = False
ScaleUnits.uiClass = None
fc = linearFlowchart(('scale_units', ScaleUnits))
node = fc.nodes()['scale_units']
x = np.arange(0, 5.0e-9, 1.0e-9)
y = np.linspace(0, 1e9, 5)
z = np.arange(4.0e6, 6.0e6, 1.0e6)
xx, yy, zz = np.meshgrid(x, y, z, indexing='ij')
vv = xx * yy * zz
x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten()
v1d = vv.flatten()
data = DataDict(x=dict(values=x1d, unit='V'),
y=dict(values=y1d, unit="A"),
z=dict(values=z1d, unit="Foobar"),
vals=dict(values=v1d, axes=['x', 'y', 'z']))
assert data.validate()
fc.setInput(dataIn=data)
output = fc.outputValues()['dataOut']
assert output['x']['unit'] == 'nV'
assert_allclose(output['x']["values"], (xx * 1e9).ravel())
assert output['y']['unit'] == 'GA'
assert_allclose(output['y']["values"], (yy / 1e9).ravel())
assert output['z']["unit"] == '$10^{6}$ Foobar'
assert_allclose(output['z']["values"], (zz / 1e6).ravel())
assert output['vals']['unit'] == ''
assert_allclose(output['vals']['values'], vv.flatten())
