def test_load_from_netcdf_and_write_metadata_to_db(empty_temp_db):
netcdf_file_path = (Path(__file__).parent / "fixtures" / "db_files" /
"netcdf" / "qcodes_2.nc")
if not os.path.exists(str(netcdf_file_path)):
pytest.skip("No netcdf fixtures found.")
ds = DataSetInMem._load_from_netcdf(netcdf_file_path)
ds.write_metadata_to_db()
loaded_ds = load_by_run_spec(captured_run_id=ds.captured_run_id)
assert isinstance(loaded_ds, DataSetInMem)
assert loaded_ds.captured_run_id == ds.captured_run_id
assert loaded_ds.captured_counter == ds.captured_counter
assert loaded_ds.run_timestamp_raw == ds.run_timestamp_raw
assert loaded_ds.completed_timestamp_raw == ds.completed_timestamp_raw
compare_datasets(ds, loaded_ds)
# now we attempt to write again. This should be a noop so everything should
# stay the same
ds.write_metadata_to_db()
loaded_ds = load_by_run_spec(captured_run_id=ds.captured_run_id)
assert isinstance(loaded_ds, DataSetInMem)
assert loaded_ds.captured_run_id == ds.captured_run_id
assert loaded_ds.captured_counter == ds.captured_counter
assert loaded_ds.run_timestamp_raw == ds.run_timestamp_raw
assert loaded_ds.completed_timestamp_raw == ds.completed_timestamp_raw
compare_datasets(ds, loaded_ds)
def test_timestamps(experiment):
ds = DataSetInMem._create_new_run(name="foo")
assert ds.run_timestamp() is None
assert ds.run_timestamp_raw is None
assert ds.completed_timestamp() is None
assert ds.completed_timestamp_raw is None
pss: List[ParamSpecBase] = []
for n in range(3):
pss.append(ParamSpecBase(f"ps{n}", paramtype="numeric"))
idps = InterDependencies_(dependencies={pss[0]: (pss[1], pss[2])})
ds.prepare(interdeps=idps, snapshot={})
assert ds.run_timestamp() is not None
assert ds.run_timestamp_raw is not None
assert ds.completed_timestamp() is None
assert ds.completed_timestamp_raw is None
ds.mark_completed()
assert ds.run_timestamp() is not None
assert ds.run_timestamp_raw is not None
assert ds.completed_timestamp() is not None
assert ds.completed_timestamp_raw is not None
ds.mark_completed()
def test_dataset_load_from_netcdf_and_db(meas_with_registered_param, DMM, DAC,
tmp_path):
with meas_with_registered_param.run(
dataset_class=DataSetType.DataSetInMem) as datasaver:
for set_v in np.linspace(0, 25, 10):
DAC.ch1.set(set_v)
get_v = DMM.v1()
datasaver.add_result((DAC.ch1, set_v), (DMM.v1, get_v))
with meas_with_registered_param.run(
dataset_class=DataSetType.DataSetInMem) as datasaver:
for set_v in np.linspace(0, 25, 10):
DAC.ch1.set(set_v)
get_v = DMM.v1()
datasaver.add_result((DAC.ch1, set_v), (DMM.v1, get_v))
path_to_db = datasaver.dataset._path_to_db
ds = datasaver.dataset
ds.add_metadata("mymetadatatag", 42)
assert ds.run_id == 2
assert isinstance(ds, DataSetInMem)
ds.export(export_type="netcdf", path=str(tmp_path))
loaded_ds = DataSetInMem._load_from_netcdf(
tmp_path / f"qcodes_{ds.captured_run_id}_{ds.guid}.nc",
path_to_db=path_to_db)
assert isinstance(loaded_ds, DataSetInMem)
assert loaded_ds.run_id == ds.run_id
compare_datasets(ds, loaded_ds)
def test_mark_pristine_completed_raises(experiment):
ds = DataSetInMem._create_new_run(name="foo")
with pytest.raises(
RuntimeError, match="Can not mark a dataset as complete before it"
):
ds.mark_completed()
def test_load_from_netcdf_no_db_file(non_created_db):
netcdf_file_path = (Path(__file__).parent / "fixtures" / "db_files" /
"netcdf" / "qcodes_2.nc")
if not os.path.exists(str(netcdf_file_path)):
pytest.skip("No netcdf fixtures found.")
ds = DataSetInMem._load_from_netcdf(netcdf_file_path)
ds.write_metadata_to_db()
loaded_ds = load_by_run_spec(captured_run_id=ds.captured_run_id)
assert isinstance(loaded_ds, DataSetInMem)
compare_datasets(ds, loaded_ds)
def test_empty_ds_parameters(experiment):
ds = DataSetInMem._create_new_run(name="foo")
assert ds._parameters is None
ds._perform_start_actions()
assert ds._parameters is None
ds.mark_completed()
assert ds._parameters is None
def test_create_dataset_in_memory_explicit_db(empty_temp_db):
default_db_location = qc.config["core"]["db_location"]
extra_db_location = str(Path(default_db_location).parent / "extra.db")
load_or_create_experiment(
conn=connect(extra_db_location), experiment_name="myexp", sample_name="mysample"
)
ds = DataSetInMem._create_new_run(name="foo", path_to_db=str(extra_db_location))
assert ds.path_to_db == extra_db_location
assert default_db_location != extra_db_location
def test_prepare_twice_raises(experiment):
ds = DataSetInMem._create_new_run(name="foo")
pss: List[ParamSpecBase] = []
for n in range(3):
pss.append(ParamSpecBase(f"ps{n}", paramtype="numeric"))
idps = InterDependencies_(dependencies={pss[0]: (pss[1], pss[2])})
ds.prepare(interdeps=idps, snapshot={})
with pytest.raises(
RuntimeError, match="Cannot prepare a dataset that is not pristine."
):
ds.prepare(interdeps=idps, snapshot={})
def test_write_metadata_to_explicit_db(empty_temp_db):
default_db_location = qc.config["core"]["db_location"]
extra_db_location = str(Path(default_db_location).parent / "extra.db")
load_or_create_experiment(experiment_name="myexp", sample_name="mysample")
load_or_create_experiment(
conn=connect(extra_db_location), experiment_name="myexp", sample_name="mysample"
)
ds = DataSetInMem._create_new_run(name="foo")
assert ds._parameters is None
assert ds.path_to_db == default_db_location
ds.export("netcdf")
ds.write_metadata_to_db(path_to_db=extra_db_location)
loaded_ds = load_by_guid(ds.guid, conn=connect(extra_db_location))
ds.the_same_dataset_as(loaded_ds)
def test_load_from_netcdf_legacy_version(non_created_db):
# Qcodes 0.26 exported netcdf files did not contain
# the parent dataset links and used a different engine to write data
# check that it still loads correctly
netcdf_file_path = (Path(__file__).parent / "fixtures" / "db_files" /
"netcdf" / "qcodes_v26.nc")
if not os.path.exists(str(netcdf_file_path)):
pytest.skip("No netcdf fixtures found.")
ds = DataSetInMem._load_from_netcdf(netcdf_file_path)
ds.write_metadata_to_db()
loaded_ds = load_by_run_spec(captured_run_id=ds.captured_run_id)
assert isinstance(loaded_ds, DataSetInMem)
compare_datasets(ds, loaded_ds)
def test_dataset_in_memory_reload_from_netcdf_complex(
meas_with_registered_param_complex, DAC, complex_num_instrument,
tmp_path):
with meas_with_registered_param_complex.run(
dataset_class=DataSetType.DataSetInMem) as datasaver:
for set_v in np.linspace(0, 25, 10):
DAC.ch1.set(set_v)
get_v = complex_num_instrument.complex_num()
datasaver.add_result((DAC.ch1, set_v),
(complex_num_instrument.complex_num, get_v))
ds = datasaver.dataset
ds.add_metadata("mymetadatatag", 42)
ds.add_metadata("someothermetadatatag", 42)
ds.export(export_type="netcdf", path=str(tmp_path))
assert isinstance(ds, DataSetInMem)
loaded_ds = DataSetInMem._load_from_netcdf(
tmp_path / f"qcodes_{ds.captured_run_id}_{ds.guid}.nc")
assert isinstance(loaded_ds, DataSetInMem)
compare_datasets(ds, loaded_ds)
def __enter__(self) -> DataSaver:
# TODO: should user actions really precede the dataset?
# first do whatever bootstrapping the user specified
for func, args in self.enteractions:
func(*args)
dataset_class: Type[DataSetProtocol]
# next set up the "datasaver"
if self.experiment is not None:
exp_id: Optional[int] = self.experiment.exp_id
path_to_db: Optional[str] = self.experiment.path_to_db
conn: Optional["ConnectionPlus"] = self.experiment.conn
else:
exp_id = None
path_to_db = None
conn = None
if self._dataset_class is DataSetType.DataSet:
self.ds = DataSet(
name=self.name,
exp_id=exp_id,
conn=conn,
in_memory_cache=self._in_memory_cache,
)
elif self._dataset_class is DataSetType.DataSetInMem:
if self._in_memory_cache is False:
raise RuntimeError("Cannot disable the in memory cache for a "
"dataset that is only in memory.")
self.ds = DataSetInMem._create_new_run(
name=self.name,
exp_id=exp_id,
path_to_db=path_to_db,
)
else:
raise RuntimeError("Does not support any other dataset classes")
# .. and give the dataset a snapshot as metadata
if self.station is None:
station = Station.default
else:
station = self.station
if station is not None:
snapshot = station.snapshot()
else:
snapshot = {}
self.ds.prepare(
snapshot=snapshot,
interdeps=self._interdependencies,
write_in_background=self._write_in_background,
shapes=self._shapes,
parent_datasets=self._parent_datasets,
)
# register all subscribers
if isinstance(self.ds, DataSet):
for (callble, state) in self.subscribers:
# We register with minimal waiting time.
# That should make all subscribers be called when data is flushed
# to the database
log.debug(f"Subscribing callable {callble} with state {state}")
self.ds.subscribe(callble,
min_wait=0,
min_count=1,
state=state)
print(f"Starting experimental run with id: {self.ds.captured_run_id}."
f" {self._extra_log_info}")
log.info(f"Starting measurement with guid: {self.ds.guid}, "
f'sample_name: "{self.ds.sample_name}", '
f'exp_name: "{self.ds.exp_name}", '
f'ds_name: "{self.ds.name}". '
f"{self._extra_log_info}")
log.info(f"Using background writing: {self._write_in_background}")
self.datasaver = DataSaver(dataset=self.ds,
write_period=self.write_period,
interdeps=self._interdependencies)
return self.datasaver
def test_no_interdeps_raises_in_prepare(experiment):
ds = DataSetInMem._create_new_run(name="foo")
with pytest.raises(RuntimeError, match="No parameters supplied"):
ds.prepare(interdeps=InterDependencies_(), snapshot={})
def test_load_from_non_existing_guid(experiment):
guid = "This is not a guid"
with pytest.raises(
RuntimeError, match="Could not find the requested run with GUID"
):
_ = DataSetInMem._load_from_db(conn=experiment.conn, guid=guid)