def test_histogram_with_ui(qtbot): dataset = _make_testdata() assert dataset.validate() hist, edges = histogram(dataset.data_vals('noise'), axis=1, bins=15) Histogrammer.useUi = True fc = linearFlowchart(('h', Histogrammer)) win = AutoPlotMainWindow(fc, loaderName=None, monitor=False) win.show() qtbot.addWidget(win) fc.setInput(dataIn=dataset) hnode = fc.nodes()['h'] # emit signal manually right now, since that's what's connected. # setting value alone won't emit the change. hnode.ui.widget.nbins.setValue(15) hnode.ui.widget.nbins.editingFinished.emit() hnode.ui.widget.combo.setCurrentText('y') assert fc.outputValues()['dataOut'].dependents() == ['noise_count'] assert fc.outputValues()['dataOut'].axes('noise_count') == \ ['x', 'z', 'noise'] assert arrays_equal(fc.outputValues()['dataOut']['noise_count']['values'], hist)
def main(): fc = linearFlowchart(('dummy', DummyNode)) node = fc.nodes()['dummy'] dialog = widgetDialog(node.ui, title='dummy node') data = testdata.get_2d_scalar_cos_data(2, 2, 1) fc.setInput(dataIn=data) return dialog, fc
def test_data_extraction(qtbot): """ Test whether extraction of one dependent gives the right data back. """ DataSelector.useUi = False data = testdata.three_compatible_3d_sets() data_name = data.dependents()[0] field_names = [data_name] + data.axes(data_name) fc = linearFlowchart(('selector', DataSelector)) node = fc.nodes()['selector'] fc.setInput(dataIn=data) node.selectedData = data_name out = fc.output()['dataOut'] assert out.dependents() == [data_name] for d, _ in out.data_items(): assert d in field_names assert np.all(np.isclose( data.data_vals(data_name), out.data_vals(data_name), atol=1e-15 ))
def test_average_subtraction(qtbot): """Test the subtract average filter node""" SubtractAverage.useUi = False SubtractAverage.uiClass = None fc = linearFlowchart(('Subtract Average', SubtractAverage), ) node = fc.nodes()['Subtract Average'] x = np.arange(11) - 5. y = np.linspace(0, 10, 51) xx, yy = np.meshgrid(x, y, indexing='ij') zz = np.sin(yy) + xx zz_ref_avg_y = np.sin(yy) - np.sin(yy).mean() data = MeshgridDataDict(x=dict(values=xx), y=dict(values=yy), z=dict(values=zz, axes=['x', 'y'])) assert data.validate() fc.setInput(dataIn=data) assert num.arrays_equal(zz, fc.outputValues()['dataOut'].data_vals('z')) node.averagingAxis = 'y' assert num.arrays_equal( zz_ref_avg_y, fc.outputValues()['dataOut'].data_vals('z'), )
def test_qcodes_flow_shaped_data(qtbot, dataset_with_shape): fc = linearFlowchart( ('Data loader', QCodesDSLoader), ('Data selection', DataSelector), ('Grid', DataGridder), ) loader = fc.nodes()['Data loader'] selector = fc.nodes()['Data selection'] selector.selectedData = 'z_0' gridder = fc.nodes()['Grid'] gridder.grid = (GridOption.metadataShape, {}) loader.pathAndId = dataset_with_shape.path_to_db, dataset_with_shape.run_id loader.update() expected_shape = dataset_with_shape.description.shapes['z_0'] datadict = fc.output()['dataOut'] for key in ('x', 'y', 'z_0'): assert datadict[key]['values'].shape == expected_shape assert datadict.shapes()[key] == expected_shape assert_array_equal(datadict[key]['values'], dataset_with_shape.get_parameter_data()['z_0'][key]) assert datadict.shape() == expected_shape
def test_set_grid_with_order(qtbot): """Test making meshgrid when the internal axis order needs to be fixed.""" DataGridder.useUi = False DataGridder.uiClass = None fc = linearFlowchart(('grid', DataGridder)) node = fc.nodes()['grid'] x = np.arange(5.0) y = np.linspace(0, 1, 5) z = np.arange(4.0, 6.0, 1.0) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vv = xx * yy * zz x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten() v1d = vv.flatten() # construct data dict, with axes for vals not conforming to the # correct order with which we've generated the data data = DataDict(x=dict(values=x1d), y=dict(values=y1d), z=dict(values=z1d), vals=dict(values=v1d, axes=['y', 'z', 'x'])) assert data.validate() # in the 1-d data, nothing unusual should happen fc.setInput(dataIn=data) assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), v1d, ) # guessing the grid should work, and fix the wrong order node.grid = GridOption.guessShape, dict() assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), vv.transpose((1, 2, 0)), ) assert fc.outputValues()['dataOut']['vals']['axes'] == ['y', 'z', 'x'] # finally, specify manually. omitting inner shape doesn't work node.grid = GridOption.specifyShape, dict(shape=(5, 2, 5)) assert fc.outputValues()['dataOut'].data_vals('vals').shape == (5, 2, 5) assert not num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), vv.transpose((1, 2, 0)), ) # but using the right inner axis order should do it node.grid = GridOption.specifyShape, dict(order=['x', 'y', 'z'], shape=(5, 5, 2)) assert fc.outputValues()['dataOut'].data_vals('vals').shape == (5, 2, 5) assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), vv.transpose((1, 2, 0)), )
def test_update_qcloader(qtbot): qc.config.core.db_location = DBPATH initialise_database() exp = load_or_create_experiment('2d_softsweep', sample_name='no sample') # define test data x = np.linspace(0, 1., 5) y = np.linspace(0, 1., 5) xx, yy = np.meshgrid(x, y, indexing='ij') zz = np.random.rand(*xx.shape) def get_2dsoftsweep_results(): for x, y, z in zip(xx.reshape(-1), yy.reshape(-1), zz.reshape(-1)): yield dict(x=x, y=y, z=z) # create data set _ds = new_data_set('2d_softsweep', exp_id=exp.exp_id, specs=[ParamSpec('x', 'numeric', unit='A'), ParamSpec('y', 'numeric', unit='B'), ParamSpec('z', 'numeric', unit='C', depends_on=['x', 'y']), ], ) run_id = _ds.run_id results = get_2dsoftsweep_results() # setting up the flowchart fc = linearFlowchart(('loader', QCodesDSLoader)) loader = fc.nodes()['loader'] loader.pathAndId = DBPATH, run_id def check(): nresults = _ds.number_of_results loader.update() ddict = fc.output()['dataOut'] if ddict is not None and nresults > 0: z_in = zz.reshape(-1)[:nresults] z_out = ddict.data_vals('z') if z_out is not None: assert z_in.size == z_out.size assert np.allclose(z_in, z_out, atol=1e-15) # insert data in small chunks, and check while True: try: ninsertions = np.random.randint(0, 5) for n in range(ninsertions): _ds.add_result(next(results)) except StopIteration: _ds.mark_complete() break check() check()
def test_loader_node(qtbot): dds.DDH5Loader.useUi = False x = np.arange(3) y = np.repeat(np.linspace(0, 1, 5).reshape(1, -1), 3, 0) z = np.arange(y.size).reshape(y.shape) data = dd.DataDict( x=dict(values=x, unit='A', __info__='careful!', __moreinfo__='more words in here'), y=dict(values=y, unit='B'), z=dict(values=z, axes=['x', 'y'], unit='C'), __desc__='some description', ) assert data.validate() dds.datadict_to_hdf5(data, str(FILEPATH), append_mode=dds.AppendMode.new) assert _clean_from_file(dds.datadict_from_hdf5(str(FILEPATH))) == data fc = linearFlowchart(('loader', dds.DDH5Loader)) node = fc.nodes()['loader'] assert fc.outputValues()['dataOut'] is None with qtbot.waitSignal(node.loadingWorker.dataLoaded, timeout=1000) as blocker: node.filepath = str(FILEPATH) out = fc.outputValues()['dataOut'].copy() out.pop('__title__') assert _clean_from_file(out) == data data.add_data(x=[3], y=np.linspace(0, 1, 5).reshape(1, -1), z=np.arange(5).reshape(1, -1)) dds.datadict_to_hdf5(data, str(FILEPATH), append_mode=dds.AppendMode.new) assert _clean_from_file(dds.datadict_from_hdf5(str(FILEPATH))) == data out = fc.outputValues()['dataOut'].copy() out.pop('__title__') assert not _clean_from_file(out) == data with qtbot.waitSignal(node.loadingWorker.dataLoaded, timeout=1000) as blocker: node.update() out = fc.outputValues()['dataOut'].copy() out.pop('__title__') assert _clean_from_file(out) == data FILEPATH.unlink()
def xySelection(interactive=False): if not interactive: app = QtGui.QApplication([]) fc = linearFlowchart(('xysel', XYSelector)) selector = fc.nodes()['xysel'] dialog = widgetDialog(selector.ui, 'xysel') data = datadict_to_meshgrid(testdata.three_compatible_3d_sets(4, 4, 4)) fc.setInput(dataIn=data) if not interactive: app.exec_() else: return dialog, fc
def dimReduction(interactive=False): if not interactive: app = QtGui.QApplication([]) fc = linearFlowchart(('reducer', DimensionReducer)) reducer = fc.nodes()['reducer'] dialog = widgetDialog(reducer.ui, 'reducer') data = datadict_to_meshgrid(testdata.three_compatible_3d_sets(2, 2, 2)) fc.setInput(dataIn=data) if not interactive: app.exec_() else: return dialog, fc
def test_xy_selector_with_roles(qtbot): """Testing XY selector using the roles 'meta' property.""" XYSelector.uiClass = None fc = linearFlowchart(('xysel', XYSelector)) node = fc.nodes()['xysel'] x = np.arange(5.0) y = np.linspace(0, 1, 5) z = np.arange(4.0, 6.0, 1.0) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vals = xx * yy * zz data = MeshgridDataDict(x=dict(values=xx), y=dict(values=yy), z=dict(values=zz), vals=dict(values=vals, axes=['x', 'y', 'z'])) assert data.validate() fc.setInput(dataIn=data) # this should return None, because no x/y axes were set. assert fc.outputValues()['dataOut'] is None # now select two axes, and test that the other one is correctly selected node.xyAxes = ('x', 'y') assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals[:, :, 0]) assert node.dimensionRoles == { 'x': 'x-axis', 'y': 'y-axis', 'z': (ReductionMethod.elementSelection, [], { 'index': 0, 'axis': 2 }) } # now set the role directly through the meta property node.dimensionRoles = { 'x': 'y-axis', 'y': (ReductionMethod.average, [], {}), 'z': 'x-axis', } assert node.xyAxes == ('z', 'x') assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals[:, :, :].mean(axis=1).transpose((1, 0)))
def loader_node(interactive=False): def cb(*vals): print(vals) if not interactive: app = QtWidgets.QApplication([]) fc = linearFlowchart(('loader', dds.DDH5Loader)) loader = fc.nodes()['loader'] dialog = widgetDialog(loader.ui, 'loader') if not interactive: loader.newDataStructure.connect(cb) app.exec_() else: return dialog, fc
def main(pathAndId): app = QtGui.QApplication([]) # flowchart and window fc = linearFlowchart( ('Dataset loader', QCodesDSLoader), ('Data selection', DataSelector), ('Grid', DataGridder), ('Dimension assignment', XYSelector), ('Sine fit', sinefit), ('plot', PlotNode), ) win = QCAutoPlotMainWindow(fc, pathAndId=pathAndId) win.show() return app.exec_()
def test_basic_flowchart_and_nodes(qtbot): fc = flowchart() node = Node(name='node') fc.addNode(node, name=node.name()) fc.connectTerminals(fc['dataIn'], node['dataIn']) fc.connectTerminals(node['dataOut'], fc['dataOut']) fc.setInput(dataIn='abcdef') assert fc.outputValues() == dict(dataOut='abcdef') for i in range(3): lst = [(f'node{j}', Node) for j in range(i)] fc = linearFlowchart(*lst) fc.setInput(dataIn='abcdef') assert fc.outputValues() == dict(dataOut='abcdef')
def test_real_histogram(qtbot): dataset = _make_testdata() assert dataset.validate() hist, edges = histogram(dataset.data_vals('noise'), axis=0, bins=10) Histogrammer.useUi = False fc = linearFlowchart(('h', Histogrammer)) fc.setInput(dataIn=dataset) assert fc.outputValues()['dataOut'] == dataset fc.nodes()['h'].nbins = 10 fc.nodes()['h'].histogramAxis = 'x' assert fc.outputValues()['dataOut'].dependents() == ['noise_count'] assert fc.outputValues()['dataOut'].axes('noise_count') == \ ['y', 'z', 'noise'] assert arrays_equal(fc.outputValues()['dataOut']['noise_count']['values'], hist)
def test_incompatible_sets(qtbot): """ Test that selecting incompatible data sets give None output. """ DataSelector.useUi = False data = testdata.three_incompatible_3d_sets() fc = linearFlowchart(('selector', DataSelector)) node = fc.nodes()['selector'] fc.setInput(dataIn=data) node.selectedData = data.dependents()[0], data.dependents()[1] assert fc.output()['dataOut'] == None node.selectedData = data.dependents()[0] assert fc.output()['dataOut'].dependents() == [data.dependents()[0]] node.selectedData = data.dependents()[1] assert fc.output()['dataOut'].dependents() == [data.dependents()[1]]
def test_data_selector(): fc = linearFlowchart(('selector', DataSelector)) selector = fc.nodes()['selector'] dialog = widgetDialog(selector.ui, title='selector') data = testdata.three_incompatible_3d_sets(2, 2, 2) fc.setInput(dataIn=data) selector.selectedData = ['data'] # for testing purposes, insert differently structured data data2 = testdata.two_compatible_noisy_2d_sets() fc.setInput(dataIn=data2) # ... and go back. fc.setInput(dataIn=data) selector.selectedData = ['data'] return dialog, fc
def gridder(interactive=False): def cb(val): print(val) if not interactive: app = QtGui.QApplication([]) fc = linearFlowchart(('grid', DataGridder)) gridder = fc.nodes()['grid'] dialog = widgetDialog(gridder.ui, 'gridder') data = testdata.three_compatible_3d_sets(2, 2, 2) fc.setInput(dataIn=data) if not interactive: gridder.shapeDetermined.connect(cb) app.exec_() else: return dialog, fc
def test_update_qcloader(qtbot, empty_db_path): db_path = empty_db_path exp = load_or_create_experiment('2d_softsweep', sample_name='no sample') N = 2 m = qc.Measurement(exp=exp) m.register_custom_parameter('x') m.register_custom_parameter('y') dd_expected = DataDict(x=dict(values=np.array([])), y=dict(values=np.array([]))) for n in range(N): m.register_custom_parameter(f'z_{n}', setpoints=['x', 'y']) dd_expected[f'z_{n}'] = dict(values=np.array([]), axes=['x', 'y']) dd_expected.validate() # setting up the flowchart fc = linearFlowchart(('loader', QCodesDSLoader)) loader = fc.nodes()['loader'] def check(): nresults = ds.number_of_results loader.update() ddict = fc.output()['dataOut'] if ddict is not None and nresults > 0: z_in = dd_expected.data_vals('z_1') z_out = ddict.data_vals('z_1') if z_out is not None: assert z_in.size == z_out.size assert np.allclose(z_in, z_out, atol=1e-15) with m.run() as datasaver: ds = datasaver.dataset run_id = datasaver.dataset.captured_run_id loader.pathAndId = db_path, run_id for result in testdata.generate_2d_scalar_simple(3, 3, N): row = [(k, v) for k, v in result.items()] datasaver.add_result(*row) dd_expected.add_data(**result) check() check()
def test_GridNode(): def cb(val): print(val) fc = linearFlowchart(('grid', DataGridder)) gridder = fc.nodes()['grid'] dialog = widgetDialog(gridder.ui, 'gridder') data = testdata.three_compatible_3d_sets(2, 2, 2) fc.setInput(dataIn=data) gridder.shapeDetermined.connect(cb) gridder.grid = GridOption.guessShape, {} gridder.grid = GridOption.specifyShape, \ dict(order=['x', 'y', 'z'], shape=(2,2,3)) gridder.grid = GridOption.guessShape, {} gridder.grid = GridOption.specifyShape, \ dict(order=['x', 'y', 'z'], shape=(2,2,3)) return dialog, fc
def test_basic_gridding(qtbot): """Test simple gridding tasks""" DataGridder.useUi = False DataGridder.uiClass = None fc = linearFlowchart(('grid', DataGridder)) node = fc.nodes()['grid'] x = np.arange(5.0) y = np.linspace(0, 1, 5) z = np.arange(4.0, 6.0, 1.0) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vv = xx * yy * zz x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten() v1d = vv.flatten() data = DataDict(x=dict(values=x1d), y=dict(values=y1d), z=dict(values=z1d), vals=dict(values=v1d, axes=['x', 'y', 'z'])) assert data.validate() fc.setInput(dataIn=data) assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), v1d, ) node.grid = GridOption.guessShape, dict() assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), vv, ) node.grid = GridOption.specifyShape, dict(shape=(5, 5, 2)) assert num.arrays_equal( fc.outputValues()['dataOut'].data_vals('vals'), vv, )
def test_reduction(qtbot): """Test basic dimension reduction.""" DimensionReducer.uiClass = None fc = linearFlowchart(('dim_red', DimensionReducer)) node = fc.nodes()['dim_red'] x = np.arange(5.0) y = np.linspace(0, 1, 5) z = np.arange(4.0, 6.0, 1.0) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vals = xx * yy * zz data = MeshgridDataDict(x=dict(values=xx), y=dict(values=yy), z=dict(values=zz), vals=dict(values=vals, axes=['x', 'y', 'z'])) assert data.validate() fc.setInput(dataIn=data) assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals) node.reductions = {'y': (np.mean, [], {})} out = fc.outputValues()['dataOut'] assert num.arrays_equal(vals.mean(axis=1), out.data_vals('vals')) assert out.axes('vals') == ['x', 'z'] node.reductions = { 'y': (ReductionMethod.elementSelection, [], { 'index': 0 }), 'z': (ReductionMethod.average, ) } out = fc.outputValues()['dataOut'] assert num.arrays_equal(vals[:, 0, :].mean(axis=-1), out.data_vals('vals')) assert out.axes('vals') == ['x']
def test_xy_selector(qtbot): """Basic XY selector node test.""" XYSelector.uiClass = None fc = linearFlowchart(('xysel', XYSelector)) node = fc.nodes()['xysel'] x = np.arange(5.0) y = np.linspace(0, 1, 5) z = np.arange(4.0, 6.0, 1.0) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vals = xx * yy * zz data = MeshgridDataDict(x=dict(values=xx), y=dict(values=yy), z=dict(values=zz), vals=dict(values=vals, axes=['x', 'y', 'z'])) assert data.validate() fc.setInput(dataIn=data) # this should return None, because no x/y axes were set. assert fc.outputValues()['dataOut'] is None # now select two axes, and test that the other one is correctly selected node.xyAxes = ('x', 'y') assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals[:, :, 0]) # try a different reduction on the third axis node.reductions = {'z': (ReductionMethod.average, [], {})} assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals.mean(axis=-1)) # Test transposing the data by flipping x/y node.xyAxes = ('y', 'x') assert num.arrays_equal(fc.outputValues()['dataOut'].data_vals('vals'), vals.mean(axis=-1).transpose((1, 0)))
def test_data_selector(interactive=True): if not interactive: app = QtGui.QApplication([]) fc = linearFlowchart(('selector', DataSelector)) selector = fc.nodes()['selector'] dialog = widgetDialog(selector.ui, 'selector') data = testdata.three_incompatible_3d_sets(2, 2, 2) fc.setInput(dataIn=data) selector.selectedData = ['data'] # for testing purposes, insert differently structured data data2 = testdata.two_compatible_noisy_2d_sets() fc.setInput(dataIn=data2) # ... and go back. fc.setInput(dataIn=data) selector.selectedData = ['data'] if not interactive: app.exec_() else: return dialog, fc
def test_basic_scale_units(qtbot): ScaleUnits.useUi = False ScaleUnits.uiClass = None fc = linearFlowchart(('scale_units', ScaleUnits)) node = fc.nodes()['scale_units'] x = np.arange(0, 5.0e-9, 1.0e-9) y = np.linspace(0, 1e9, 5) z = np.arange(4.0e6, 6.0e6, 1.0e6) xx, yy, zz = np.meshgrid(x, y, z, indexing='ij') vv = xx * yy * zz x1d, y1d, z1d = xx.flatten(), yy.flatten(), zz.flatten() v1d = vv.flatten() data = DataDict(x=dict(values=x1d, unit='V'), y=dict(values=y1d, unit="A"), z=dict(values=z1d, unit="Foobar"), vals=dict(values=v1d, axes=['x', 'y', 'z'])) assert data.validate() fc.setInput(dataIn=data) output = fc.outputValues()['dataOut'] assert output['x']['unit'] == 'nV' assert_allclose(output['x']["values"], (xx * 1e9).ravel()) assert output['y']['unit'] == 'GA' assert_allclose(output['y']["values"], (yy / 1e9).ravel()) assert output['z']["unit"] == '$10^{6}$ Foobar' assert_allclose(output['z']["values"], (zz / 1e6).ravel()) assert output['vals']['unit'] == '' assert_allclose(output['vals']['values'], vv.flatten())