def test_atomic_creation(experiment):
""""
Test that dataset creation is atomic. Test for
https://github.com/QCoDeS/Qcodes/issues/1444
"""
def just_throw(*args):
raise RuntimeError("This breaks adding metadata")
# first we patch add_data_to_dynamic_columns to throw an exception
# if create_data is not atomic this would create a partial
# run in the db. Causing the next create_run to fail
with patch("qcodes.dataset.sqlite.queries.add_data_to_dynamic_columns",
new=just_throw):
x = ParamSpec("x", "numeric")
t = ParamSpec("t", "numeric")
y = ParamSpec("y", "numeric", depends_on=["x", "t"])
with pytest.raises(
RuntimeError,
match="Rolling back due to unhandled exception") as e:
mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
parameters=[x, t, y],
metadata={"a": 1},
)
assert error_caused_by(e, "This breaks adding metadata")
# since we are starting from an empty database and the above transaction
# should be rolled back there should be no runs in the run table
runs = mut_conn.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut_conn.transaction(new_conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
# if the above was not correctly rolled back we
# expect the next creation of a run to fail
mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y],
metadata={'a': 1})
runs = mut_conn.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut_conn.transaction(new_conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
def test_get_dependents(experiment):
# more parameters, more complicated dependencies
x = ParamSpecBase("x", "numeric")
t = ParamSpecBase("t", "numeric")
y = ParamSpecBase("y", "numeric")
x_raw = ParamSpecBase("x_raw", "numeric")
x_cooked = ParamSpecBase("x_cooked", "numeric")
z = ParamSpecBase("z", "numeric")
deps_param_tree = {y: (x, t), z: (x_cooked, )}
inferred_param_tree = {x_cooked: (x_raw, )}
interdeps = InterDependencies_(dependencies=deps_param_tree,
inferences=inferred_param_tree)
description = RunDescriber(interdeps=interdeps)
(_, run_id, _) = mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
description=description,
)
deps = mut_queries._get_dependents(experiment.conn, run_id)
expected_deps = [
mut_queries._get_layout_id(experiment.conn, 'y', run_id),
mut_queries._get_layout_id(experiment.conn, 'z', run_id)
]
assert deps == expected_deps
def _add_run_to_runs_table(
dataset: DataSetProtocol,
target_conn: ConnectionPlus,
target_exp_id: int,
create_run_table: bool = True,
) -> Optional[str]:
metadata = dataset.metadata
snapshot_raw = dataset._snapshot_raw
captured_run_id = dataset.captured_run_id
captured_counter = dataset.captured_counter
parent_dataset_links = links_to_str(dataset.parent_dataset_links)
_, target_run_id, target_table_name = create_run(
target_conn,
target_exp_id,
name=dataset.name,
guid=dataset.guid,
metadata=metadata,
captured_run_id=captured_run_id,
captured_counter=captured_counter,
parent_dataset_links=parent_dataset_links,
create_run_table=create_run_table,
snapshot_raw=snapshot_raw,
description=dataset.description,
)
mark_run_complete(target_conn, target_run_id)
_rewrite_timestamps(
target_conn,
target_run_id,
dataset.run_timestamp_raw,
dataset.completed_timestamp_raw,
)
return target_table_name
def _extract_single_dataset_into_db(dataset: DataSet,
target_conn: ConnectionPlus,
target_exp_id: int) -> None:
"""
NB: This function should only be called from within
:meth:extract_runs_into_db
Insert the given dataset into the specified database file as the latest
run.
Trying to insert a run already in the DB is a NOOP.
Args:
dataset: A dataset representing the run to be copied
target_conn: connection to the DB. Must be atomically guarded
target_exp_id: The exp_id of the (target DB) experiment in which to
insert the run
"""
if not dataset.completed:
raise ValueError('Dataset not completed. An incomplete dataset '
'can not be copied. The incomplete dataset has '
f'GUID: {dataset.guid} and run_id: {dataset.run_id}')
source_conn = dataset.conn
run_id = get_runid_from_guid(target_conn, dataset.guid)
if run_id != -1:
return
if dataset.parameters is not None:
param_names = dataset.parameters.split(',')
else:
param_names = []
parspecs_dict = {
p.name: p
for p in new_to_old(dataset._interdeps).paramspecs
}
parspecs = [parspecs_dict[p] for p in param_names]
metadata = dataset.metadata
snapshot_raw = dataset.snapshot_raw
_, target_run_id, target_table_name = create_run(target_conn,
target_exp_id,
name=dataset.name,
guid=dataset.guid,
parameters=parspecs,
metadata=metadata)
_populate_results_table(source_conn, target_conn, dataset.table_name,
target_table_name)
mark_run_complete(target_conn, target_run_id)
_rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw,
dataset.completed_timestamp_raw)
if snapshot_raw is not None:
add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def test_get_dependents(experiment):
x = ParamSpec('x', 'numeric')
t = ParamSpec('t', 'numeric')
y = ParamSpec('y', 'numeric', depends_on=['x', 't'])
# Make a dataset
(_, run_id, _) = mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y])
deps = mut_queries._get_dependents(experiment.conn, run_id)
layout_id = mut_queries._get_layout_id(experiment.conn, 'y', run_id)
assert deps == [layout_id]
# more parameters, more complicated dependencies
x_raw = ParamSpec('x_raw', 'numeric')
x_cooked = ParamSpec('x_cooked', 'numeric', inferred_from=['x_raw'])
z = ParamSpec('z', 'numeric', depends_on=['x_cooked'])
(_, run_id,
_) = mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, x_raw, x_cooked, y, z])
deps = mut_queries._get_dependents(experiment.conn, run_id)
expected_deps = [
mut_queries._get_layout_id(experiment.conn, 'y', run_id),
mut_queries._get_layout_id(experiment.conn, 'z', run_id)
]
assert deps == expected_deps
def test_get_dependents_simple(experiment, simple_run_describer):
(_, run_id, _) = mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
description=simple_run_describer,
)
deps = mut_queries._get_dependents(experiment.conn, run_id)
layout_id = mut_queries._get_layout_id(experiment.conn, "y", run_id)
assert deps == [layout_id]
def _create_new_run(
cls,
name: str,
path_to_db: Optional[Union[Path, str]] = None,
exp_id: Optional[int] = None,
) -> DataSetInMem:
if path_to_db is not None:
path_to_db = str(path_to_db)
with contextlib.closing(
conn_from_dbpath_or_conn(conn=None,
path_to_db=path_to_db)) as conn:
if exp_id is None:
exp_id = get_default_experiment_id(conn)
name = name or "dataset"
sample_name = get_sample_name_from_experiment_id(conn, exp_id)
exp_name = get_experiment_name_from_experiment_id(conn, exp_id)
guid = generate_guid()
run_counter, run_id, _ = create_run(conn,
exp_id,
name,
guid=guid,
parameters=None,
create_run_table=False)
ds = cls(
run_id=run_id,
captured_run_id=run_id,
counter=run_counter,
captured_counter=run_counter,
name=name,
exp_id=exp_id,
exp_name=exp_name,
sample_name=sample_name,
guid=guid,
path_to_db=conn.path_to_dbfile,
run_timestamp_raw=None,
completed_timestamp_raw=None,
metadata=None,
)
return ds
def __init__(self,
path_to_db: str = None,
run_id: Optional[int] = None,
conn: Optional[ConnectionPlus] = None,
exp_id=None,
name: str = None,
specs: Optional[SpecsOrInterDeps] = None,
values=None,
metadata=None) -> None:
"""
Create a new DataSet object. The object can either hold a new run or
an already existing run. If a run_id is provided, then an old run is
looked up, else a new run is created.
Args:
path_to_db: path to the sqlite file on disk. If not provided, the
path will be read from the config.
run_id: provide this when loading an existing run, leave it
as None when creating a new run
conn: connection to the DB; if provided and `path_to_db` is
provided as well, then a ValueError is raised (this is to
prevent the possibility of providing a connection to a DB
file that is different from `path_to_db`)
exp_id: the id of the experiment in which to create a new run.
Ignored if run_id is provided.
name: the name of the dataset. Ignored if run_id is provided.
specs: paramspecs belonging to the dataset. Ignored if run_id is
provided.
values: values to insert into the dataset. Ignored if run_id is
provided.
metadata: metadata to insert into the dataset. Ignored if run_id
is provided.
"""
if path_to_db is not None and conn is not None:
raise ValueError("Both `path_to_db` and `conn` arguments have "
"been passed together with non-None values. "
"This is not allowed.")
self._path_to_db = path_to_db or get_DB_location()
self.conn = make_connection_plus_from(conn) if conn is not None else \
connect(self.path_to_db)
self._run_id = run_id
self._debug = False
self.subscribers: Dict[str, _Subscriber] = {}
self._interdeps: InterDependencies_
if run_id is not None:
if not run_exists(self.conn, run_id):
raise ValueError(f"Run with run_id {run_id} does not exist in "
f"the database")
self._completed = completed(self.conn, self.run_id)
run_desc = self._get_run_description_from_db()
if run_desc._old_style_deps:
# TODO: what if the old run had invalid interdep.s?
old_idps: InterDependencies = cast(InterDependencies,
run_desc.interdeps)
self._interdeps = old_to_new(old_idps)
else:
new_idps: InterDependencies_ = cast(InterDependencies_,
run_desc.interdeps)
self._interdeps = new_idps
self._metadata = get_metadata_from_run_id(self.conn, run_id)
self._started = self.run_timestamp_raw is not None
else:
# Actually perform all the side effects needed for the creation
# of a new dataset. Note that a dataset is created (in the DB)
# with no parameters; they are written to disk when the dataset
# is marked as started
if exp_id is None:
if len(get_experiments(self.conn)) > 0:
exp_id = get_last_experiment(self.conn)
else:
raise ValueError("No experiments found."
"You can start a new one with:"
" new_experiment(name, sample_name)")
name = name or "dataset"
_, run_id, __ = create_run(self.conn,
exp_id,
name,
generate_guid(),
parameters=None,
values=values,
metadata=metadata)
# this is really the UUID (an ever increasing count in the db)
self._run_id = run_id
self._completed = False
self._started = False
if isinstance(specs, InterDependencies_):
self._interdeps = specs
elif specs is not None:
self._interdeps = old_to_new(InterDependencies(*specs))
else:
self._interdeps = InterDependencies_()
self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
