def test_atomic_creation(experiment):
""""
Test that dataset creation is atomic. Test for
https://github.com/QCoDeS/Qcodes/issues/1444
"""
def just_throw(*args):
raise RuntimeError("This breaks adding metadata")
# first we patch add_meta_data to throw an exception
# if create_data is not atomic this would create a partial
# run in the db. Causing the next create_run to fail
with patch('qcodes.dataset.sqlite_base.add_meta_data', new=just_throw):
x = ParamSpec('x', 'numeric')
t = ParamSpec('t', 'numeric')
y = ParamSpec('y', 'numeric', depends_on=['x', 't'])
with pytest.raises(
RuntimeError,
match="Rolling back due to unhandled exception") as e:
mut.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y],
metadata={'a': 1})
assert error_caused_by(e, "This breaks adding metadata")
# since we are starting from an empty database and the above transaction
# should be rolled back there should be no runs in the run table
runs = mut.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut.transaction(new_conn, 'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
# if the above was not correctly rolled back we
# expect the next creation of a run to fail
mut.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y],
metadata={'a': 1})
runs = mut.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut.transaction(new_conn, 'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
def _extract_single_dataset_into_db(dataset: DataSet,
target_conn: ConnectionPlus,
target_exp_id: int) -> None:
"""
NB: This function should only be called from within
:meth:extract_runs_into_db
Insert the given dataset into the specified database file as the latest
run.
Trying to insert a run already in the DB is a NOOP.
Args:
dataset: A dataset representing the run to be copied
target_conn: connection to the DB. Must be atomically guarded
target_exp_id: The exp_id of the (target DB) experiment in which to
insert the run
"""
if not dataset.completed:
raise ValueError('Dataset not completed. An incomplete dataset '
'can not be copied. The incomplete dataset has '
f'GUID: {dataset.guid} and run_id: {dataset.run_id}')
source_conn = dataset.conn
run_id = get_runid_from_guid(target_conn, dataset.guid)
if run_id != -1:
return
if dataset.parameters is not None:
param_names = dataset.parameters.split(',')
else:
param_names = []
parspecs_dict = {
p.name: p
for p in new_to_old(dataset._interdeps).paramspecs
}
parspecs = [parspecs_dict[p] for p in param_names]
metadata = dataset.metadata
snapshot_raw = dataset.snapshot_raw
_, target_run_id, target_table_name = create_run(target_conn,
target_exp_id,
name=dataset.name,
guid=dataset.guid,
parameters=parspecs,
metadata=metadata)
_populate_results_table(source_conn, target_conn, dataset.table_name,
target_table_name)
mark_run_complete(target_conn, target_run_id)
_rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw,
dataset.completed_timestamp_raw)
if snapshot_raw is not None:
add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def _new(self, name, exp_id, specs: SPECS = None, values=None,
metadata=None) -> None:
"""
Actually perform all the side effects needed for
the creation of a new dataset.
"""
_, run_id, __ = create_run(self.conn, exp_id, name,
specs, values, metadata)
# this is really the UUID (an ever increasing count in the db)
self.run_id = run_id
self._completed = False
def test_get_dependents(experiment):
x = ParamSpec('x', 'numeric')
t = ParamSpec('t', 'numeric')
y = ParamSpec('y', 'numeric', depends_on=['x', 't'])
# Make a dataset
(_, run_id, _) = mut.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y])
deps = mut.get_dependents(experiment.conn, run_id)
layout_id = mut.get_layout_id(experiment.conn, 'y', run_id)
assert deps == [layout_id]
# more parameters, more complicated dependencies
x_raw = ParamSpec('x_raw', 'numeric')
x_cooked = ParamSpec('x_cooked', 'numeric', inferred_from=['x_raw'])
z = ParamSpec('z', 'numeric', depends_on=['x_cooked'])
(_, run_id, _) = mut.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, x_raw, x_cooked, y, z])
deps = mut.get_dependents(experiment.conn, run_id)
expected_deps = [
mut.get_layout_id(experiment.conn, 'y', run_id),
mut.get_layout_id(experiment.conn, 'z', run_id)
]
assert deps == expected_deps
def __init__(self,
path_to_db: str = None,
run_id: Optional[int] = None,
conn: Optional[ConnectionPlus] = None,
exp_id=None,
name: str = None,
specs: Optional[SpecsOrInterDeps] = None,
values=None,
metadata=None) -> None:
"""
Create a new DataSet object. The object can either hold a new run or
an already existing run. If a run_id is provided, then an old run is
looked up, else a new run is created.
Args:
path_to_db: path to the sqlite file on disk. If not provided, the
path will be read from the config.
run_id: provide this when loading an existing run, leave it
as None when creating a new run
conn: connection to the DB; if provided and `path_to_db` is
provided as well, then a ValueError is raised (this is to
prevent the possibility of providing a connection to a DB
file that is different from `path_to_db`)
exp_id: the id of the experiment in which to create a new run.
Ignored if run_id is provided.
name: the name of the dataset. Ignored if run_id is provided.
specs: paramspecs belonging to the dataset. Ignored if run_id is
provided.
values: values to insert into the dataset. Ignored if run_id is
provided.
metadata: metadata to insert into the dataset. Ignored if run_id
is provided.
"""
if path_to_db is not None and conn is not None:
raise ValueError("Both `path_to_db` and `conn` arguments have "
"been passed together with non-None values. "
"This is not allowed.")
self._path_to_db = path_to_db or get_DB_location()
self.conn = make_connection_plus_from(conn) if conn is not None else \
connect(self.path_to_db)
self._run_id = run_id
self._debug = False
self.subscribers: Dict[str, _Subscriber] = {}
self._interdeps: InterDependencies_
if run_id is not None:
if not run_exists(self.conn, run_id):
raise ValueError(f"Run with run_id {run_id} does not exist in "
f"the database")
self._completed = completed(self.conn, self.run_id)
run_desc = self._get_run_description_from_db()
if run_desc._old_style_deps:
# TODO: what if the old run had invalid interdep.s?
old_idps: InterDependencies = cast(InterDependencies,
run_desc.interdeps)
self._interdeps = old_to_new(old_idps)
else:
new_idps: InterDependencies_ = cast(InterDependencies_,
run_desc.interdeps)
self._interdeps = new_idps
self._metadata = get_metadata_from_run_id(self.conn, run_id)
self._started = self.run_timestamp_raw is not None
else:
# Actually perform all the side effects needed for the creation
# of a new dataset. Note that a dataset is created (in the DB)
# with no parameters; they are written to disk when the dataset
# is marked as started
if exp_id is None:
if len(get_experiments(self.conn)) > 0:
exp_id = get_last_experiment(self.conn)
else:
raise ValueError("No experiments found."
"You can start a new one with:"
" new_experiment(name, sample_name)")
name = name or "dataset"
_, run_id, __ = create_run(self.conn,
exp_id,
name,
generate_guid(),
parameters=None,
values=values,
metadata=metadata)
# this is really the UUID (an ever increasing count in the db)
self._run_id = run_id
self._completed = False
self._started = False
if isinstance(specs, InterDependencies_):
self._interdeps = specs
elif specs is not None:
self._interdeps = old_to_new(InterDependencies(*specs))
else:
self._interdeps = InterDependencies_()
self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
def __init__(self, path_to_db: str=None,
run_id: Optional[int]=None,
conn=None,
exp_id=None,
name: str=None,
specs: SPECS=None,
values=None,
metadata=None) -> None:
"""
Create a new DataSet object. The object can either hold a new run or
an already existing run. If a run_id is provided, then an old run is
looked up, else a new run is created.
Args:
path_to_db: path to the sqlite file on disk. If not provided, the
path will be read from the config.
run_id: provide this when loading an existing run, leave it
as None when creating a new run
conn: connection to the DB
exp_id: the id of the experiment in which to create a new run.
Ignored if run_id is provided.
name: the name of the dataset. Ignored if run_id is provided.
specs: paramspecs belonging to the dataset. Ignored if run_id is
provided.
values: values to insert into the dataset. Ignored if run_id is
provided.
metadata: metadata to insert into the dataset. Ignored if run_id
is provided.
"""
# TODO: handle fail here by defaulting to
# a standard db
self.path_to_db = path_to_db or get_DB_location()
if conn is None:
self.conn = connect(self.path_to_db)
else:
self.conn = conn
self.run_id = run_id
self._debug = False
self.subscribers: Dict[str, _Subscriber] = {}
if run_id:
if not run_exists(self.conn, run_id):
raise ValueError(f"Run with run_id {run_id} does not exist in "
f"the database")
self._completed = completed(self.conn, self.run_id)
else:
if exp_id is None:
if len(get_experiments(self.conn)) > 0:
exp_id = get_last_experiment(self.conn)
else:
raise ValueError("No experiments found."
"You can start a new one with:"
" new_experiment(name, sample_name)")
# Actually perform all the side effects needed for
# the creation of a new dataset.
name = name or "dataset"
_, run_id, __ = create_run(self.conn, exp_id, name,
generate_guid(),
specs, values, metadata)
# this is really the UUID (an ever increasing count in the db)
self.run_id = run_id
self._completed = False
