def generate_DB_file_with_some_runs(version=VERSION):
"""
Generate a .db-file with a handful of runs with some interdependent
parameters
"""
# This function will run often on CI and re-generate the .db-files
# That should ideally be a deterministic action
# (although this hopefully plays no role)
np.random.seed(0)
vNfixturepath = os.path.join(utils.fixturepath, f'version{version}')
os.makedirs(vNfixturepath, exist_ok=True)
path = os.path.join(vNfixturepath, 'some_runs.db')
if os.path.exists(path):
os.remove(path)
from qcodes.dataset.sqlite.database import connect
from qcodes.dataset.measurements import Measurement
from qcodes.dataset.experiment_container import Experiment
from qcodes import Parameter
connect(path)
exp = Experiment(path_to_db=path,
name='experiment_1',
sample_name='no_sample_1')
# Now make some parameters to use in measurements
params = []
for n in range(5):
params.append(
Parameter(f'p{n}',
label=f'Parameter {n}',
unit=f'unit {n}',
set_cmd=None,
get_cmd=None))
# Set up an experiment
meas = Measurement(exp)
meas.register_parameter(params[0])
meas.register_parameter(params[1])
meas.register_parameter(params[2], basis=(params[0], ))
meas.register_parameter(params[3], basis=(params[1], ))
meas.register_parameter(params[4], setpoints=(params[2], params[3]))
# Make a number of identical runs
for _ in range(10):
with meas.run() as datasaver:
for x in np.random.rand(10):
for y in np.random.rand(10):
z = np.random.rand()
datasaver.add_result((params[0], 0), (params[1], 1),
(params[2], x), (params[3], y),
(params[4], z))
def test_cannot_connect_to_newer_db():
conn = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"])
current_version = get_user_version(conn)
set_user_version(conn, current_version+1)
conn.close()
err_msg = f'is version {current_version + 1} but this version of QCoDeS ' \
f'supports up to version {current_version}'
with pytest.raises(RuntimeError, match=err_msg):
conn = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"])
def generate_empty_DB_file(version=VERSION):
"""
Generate an empty DB file with no runs
"""
from qcodes.dataset.sqlite.database import connect
vNfixturepath = os.path.join(utils.fixturepath, f'version{version}')
os.makedirs(vNfixturepath, exist_ok=True)
path = os.path.join(vNfixturepath, 'empty.db')
if os.path.exists(path):
os.remove(path)
connect(path)
def test_write_metadata_to_explicit_db(empty_temp_db):
default_db_location = qc.config["core"]["db_location"]
extra_db_location = str(Path(default_db_location).parent / "extra.db")
load_or_create_experiment(experiment_name="myexp", sample_name="mysample")
load_or_create_experiment(
conn=connect(extra_db_location), experiment_name="myexp", sample_name="mysample"
)
ds = DataSetInMem._create_new_run(name="foo")
assert ds._parameters is None
assert ds.path_to_db == default_db_location
ds.export("netcdf")
ds.write_metadata_to_db(path_to_db=extra_db_location)
loaded_ds = load_by_guid(ds.guid, conn=connect(extra_db_location))
ds.the_same_dataset_as(loaded_ds)
def two_empty_temp_db_connections():
"""
Yield the paths of two empty files. Meant for use with the
test_database_copy_paste
"""
with tempfile.TemporaryDirectory() as tmpdirname:
source_path = os.path.join(tmpdirname, 'source.db')
target_path = os.path.join(tmpdirname, 'target.db')
source_conn = connect(source_path)
target_conn = connect(target_path)
try:
yield (source_conn, target_conn)
finally:
source_conn.close()
target_conn.close()
def guids_from_dbs(
db_paths: Iterable[Path],
) -> Tuple[Dict[Path, List[str]], Dict[str, Path]]:
"""
Extract all guids from the supplied database paths.
Args:
db_paths: Path or str or directory where to search
Returns:
Tuple of Dictionary mapping paths to lists of guids as strings
and Dictionary mapping guids to db paths.
"""
dbdict = {}
for p in db_paths:
try:
conn = connect(str(p))
dbdict[p] = get_guids_from_run_spec(conn)
except (RuntimeError, DatabaseError) as e:
print(e)
finally:
conn.close()
gc.collect()
guiddict = {}
for dbpath, guids in dbdict.items():
guiddict.update({guid: dbpath for guid in guids})
return dbdict, guiddict
def load_by_counter(counter: int,
exp_id: int,
conn: Optional[ConnectionPlus] = None) -> DataSet:
"""
Load a dataset given its counter in a given experiment
Lookup is performed in the database file that is specified in the config.
Args:
counter: counter of the dataset within the given experiment
exp_id: id of the experiment where to look for the dataset
conn: connection to the database to load from. If not provided, a
connection to the DB file specified in the config is made
Returns:
dataset of the given counter in the given experiment
"""
conn = conn or connect(get_DB_location())
sql = """
SELECT run_id
FROM
runs
WHERE
result_counter= ? AND
exp_id = ?
"""
c = transaction(conn, sql, counter, exp_id)
run_id = one(c, 'run_id')
d = DataSet(conn=conn, run_id=run_id)
return d
def get_runs_from_db(path: str, start: int = 0,
stop: Union[None, int] = None,
get_structure: bool = False):
"""
Get a db 'overview' dictionary from the db located in `path`.
`start` and `stop` refer to indices of the runs in the db that we want
to have details on; if `stop` is None, we'll use runs until the end.
if `get_structure` is True, include info on the run data structure
in the return dict.
"""
conn = connect(path)
runs = get_runs(conn)
if stop is None:
stop = len(runs)
runs = runs[start:stop]
overview = {}
for run in runs:
run_id = run['run_id']
overview[run_id] = get_ds_info(conn, run_id,
get_structure=get_structure)
return overview
def load_experiment_by_name(
name: str,
sample: Optional[str] = None,
conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Try to load experiment with the specified name.
Nothing stops you from having many experiments with the same name and
sample_name. In that case this won't work. And warn you.
Args:
name: the name of the experiment
sample: the name of the sample
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
the requested experiment
Raises:
ValueError if the name is not unique and sample name is None.
"""
conn = conn or connect(get_DB_location())
if sample:
sql = """
SELECT
*
FROM
experiments
WHERE
sample_name = ? AND
name = ?
"""
c = transaction(conn, sql, sample, name)
else:
sql = """
SELECT
*
FROM
experiments
WHERE
name = ?
"""
c = transaction(conn, sql, name)
rows = c.fetchall()
if len(rows) == 0:
raise ValueError("Experiment not found")
elif len(rows) > 1:
_repr = []
for row in rows:
s = (f"exp_id:{row['exp_id']} ({row['name']}-{row['sample_name']})"
f" started at ({row['start_time']})")
_repr.append(s)
_repr_str = "\n".join(_repr)
raise ValueError(f"Many experiments matching your request"
f" found:\n{_repr_str}")
else:
e = Experiment(exp_id=rows[0]['exp_id'], conn=conn)
return e
def get_dataIDs(
db_name: str,
stage: str,
db_folder: Optional[str] = None,
quality: Optional[int] = None,
) -> List[int]:
""""""
if db_name[-2:] != "db":
db_name += ".db"
if db_folder is None:
db_folder = nt.config["db_folder"]
db_path = os.path.join(db_folder, db_name)
conn = connect(db_path)
if quality is None:
sql = f"""
SELECT run_id FROM runs WHERE {stage}={1} OR {stage} LIKE {str(1)}
"""
else:
sql = f"""
SELECT run_id FROM runs
WHERE ({stage}={1} OR {stage} LIKE {str(1)})
AND (good={quality} OR good LIKE {str(quality)})
"""
c = conn.execute(sql)
param_names_temp = many_many(c, "run_id")
return list(flatten_list(param_names_temp))
def load_by_counter(counter: int, exp_id: int,
conn: Optional[ConnectionPlus] = None) -> DataSet:
"""
Load a dataset given its counter in a given experiment
Lookup is performed in the database file that is specified in the config.
Note that the `counter` used in this function in not preserved when copying
data to another db file. We recommend using :func:`.load_by_run_spec` which
does not have this issue and is significantly more flexible.
Args:
counter: counter of the dataset within the given experiment
exp_id: id of the experiment where to look for the dataset
conn: connection to the database to load from. If not provided, a
connection to the DB file specified in the config is made
Returns:
:class:`.DataSet` of the given counter in the given experiment
"""
conn = conn or connect(get_DB_location())
sql = """
SELECT run_id
FROM
runs
WHERE
result_counter= ? AND
exp_id = ?
"""
c = transaction(conn, sql, counter, exp_id)
run_id = one(c, 'run_id')
d = DataSet(conn=conn, run_id=run_id)
return d
def load_or_create_experiment(
experiment_name: str,
sample_name: Optional[str] = None,
conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Find and return an experiment with the given name and sample name,
or create one if not found.
Args:
experiment_name: Name of the experiment to find or create
sample_name: Name of the sample
conn: Connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
The found or created experiment
"""
conn = conn or connect(get_DB_location())
try:
experiment = load_experiment_by_name(experiment_name,
sample_name,
conn=conn)
except ValueError as exception:
if "Experiment not found" in str(exception):
experiment = new_experiment(experiment_name,
sample_name,
conn=conn)
else:
raise exception
return experiment
def load_by_id(run_id: int, conn: Optional[ConnectionPlus] = None) -> DataSet:
"""
Load dataset by run id
If no connection is provided, lookup is performed in the database file that
is specified in the config.
Note that the `run_id` used in this function in not preserved when copying
data to another db file. We recommend using :func:`.load_by_run_spec` which
does not have this issue and is significantly more flexible.
Args:
run_id: run id of the dataset
conn: connection to the database to load from
Returns:
:class:`.DataSet` with the given run id
"""
if run_id is None:
raise ValueError('run_id has to be a positive integer, not None.')
conn = conn or connect(get_DB_location())
d = DataSet(conn=conn, run_id=run_id)
return d
def new_experiment(name: str,
sample_name: Optional[str],
format_string: str = "{}-{}-{}",
conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Create a new experiment (in the database file from config)
Args:
name: the name of the experiment
sample_name: the name of the current sample
format_string: basic format string for table-name
must contain 3 placeholders.
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
the new experiment
"""
sample_name = sample_name or "some_sample"
conn = conn or connect(get_DB_location())
exp_ids = get_matching_exp_ids(conn, name=name, sample_name=sample_name)
if len(exp_ids) >= 1:
log.warning(
f"There is (are) already experiment(s) with the name of {name} "
f"and sample name of {sample_name} in the database.")
experiment = Experiment(name=name,
sample_name=sample_name,
format_string=format_string,
conn=conn)
_set_default_experiment_id(path_to_dbfile(conn), experiment.exp_id)
return experiment
def load_by_guid(guid: str, conn: Optional[ConnectionPlus] = None) -> DataSet:
"""
Load a dataset by its GUID
If no connection is provided, lookup is performed in the database file that
is specified in the config.
Args:
guid: guid of the dataset
conn: connection to the database to load from
Returns:
dataset with the given guid
Raises:
NameError: if no run with the given GUID exists in the database
RuntimeError: if several runs with the given GUID are found
"""
conn = conn or connect(get_DB_location())
# this function raises a RuntimeError if more than one run matches the GUID
run_id = get_runid_from_guid(conn, guid)
if run_id == -1:
raise NameError(f'No run with GUID: {guid} found in database.')
return DataSet(run_id=run_id, conn=conn)
def shadow_conn(path_to_db: str):
"""
Simple context manager to create a connection for testing and
close it on exit
"""
conn = mut_db.connect(path_to_db)
yield conn
conn.close()
def test_path_to_dbfile():
with tempfile.TemporaryDirectory() as tempdir:
tempdb = os.path.join(tempdir, 'database.db')
conn = mut_db.connect(tempdb)
try:
assert path_to_dbfile(conn) == tempdb
finally:
conn.close()
def toggle_debug(self):
"""
Toggle debug mode, if debug mode is on all the queries made are
echoed back.
"""
self._debug = not self._debug
self.conn.close()
self.conn = connect(self.path_to_db, self._debug)
def test_connect():
conn = connect(':memory:')
assert isinstance(conn, sqlite3.Connection)
assert isinstance(conn, ConnectionPlus)
assert False is conn.atomic_in_progress
assert sqlite3.Row is conn.row_factory
def __init__(self,
path_to_db: Optional[str] = None,
exp_id: Optional[int] = None,
name: Optional[str] = None,
sample_name: Optional[str] = None,
format_string: str = "{}-{}-{}",
conn: Optional[ConnectionPlus] = None) -> None:
"""
Create or load an experiment. If exp_id is None, a new experiment is
created. If exp_id is not None, an experiment is loaded.
Args:
path_to_db: The path of the database file to create in/load from.
If a conn is passed together with path_to_db, an exception is
raised
exp_id: The id of the experiment to load
name: The name of the experiment to create. Ignored if exp_id is
not None
sample_name: The sample name for this experiment. Ignored if exp_id
is not None
format_string: The format string used to name result-tables.
Ignored if exp_id is not None.
conn: connection to the database. If not supplied, the constructor
first tries to use path_to_db to figure out where to connect to.
If path_to_db is not supplied either, a new connection
to the DB file specified in the config is made
"""
if path_to_db is not None and conn is not None:
raise ValueError('Received BOTH conn and path_to_db. Please '
'provide only one or the other.')
self._path_to_db = path_to_db or get_DB_location()
self.conn = conn or connect(self.path_to_db, get_DB_debug())
max_id = len(get_experiments(self.conn))
if exp_id is not None:
if exp_id not in range(1, max_id + 1):
raise ValueError('No such experiment in the database')
self._exp_id = exp_id
else:
# it is better to catch an invalid format string earlier than later
try:
# the corresponding function from sqlite module will try to
# format as `(name, exp_id, run_counter)`, hence we prepare
# for that here
format_string.format("name", 1, 1)
except Exception as e:
raise ValueError("Invalid format string. Can not format "
"(name, exp_id, run_counter)") from e
log.info("creating new experiment in {}".format(self.path_to_db))
name = name or f"experiment_{max_id+1}"
sample_name = sample_name or "some_sample"
self._exp_id = ne(self.conn, name, sample_name, format_string)
def test_path_to_dbfile(tmp_path):
tempdb = str(tmp_path / 'database.db')
conn = mut_db.connect(tempdb)
try:
assert path_to_dbfile(conn) == tempdb
assert conn.path_to_dbfile == tempdb
finally:
conn.close()
def load_experiment_by_name(
name: str,
sample: Optional[str] = None,
conn: Optional[ConnectionPlus] = None,
load_last_duplicate: bool = False,
) -> Experiment:
"""
Try to load experiment with the specified name.
Nothing stops you from having many experiments with the same name and
sample name. In that case this won't work unless load_last_duplicate
is set to True. Then, the last of duplicated experiments will be loaded.
Args:
name: the name of the experiment
sample: the name of the sample
load_last_duplicate: If True, prevent raising error for having
multiple experiments with the same name and sample name, and
load the last duplicated experiment, instead.
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
the requested experiment
Raises:
ValueError either if the name and sample name are not unique, unless
load_last_duplicate is True, or if no experiment found for the
supplied name and sample.
.
"""
conn = conn or connect(get_DB_location())
if sample is not None:
args_to_find = {"name": name, "sample_name": sample}
else:
args_to_find = {"name": name}
exp_ids = get_matching_exp_ids(conn, **args_to_find)
if len(exp_ids) == 0:
raise ValueError("Experiment not found")
elif len(exp_ids) > 1:
_repr = []
for exp_id in exp_ids:
exp = load_experiment(exp_id, conn=conn)
s = (f"exp_id:{exp.exp_id} ({exp.name}-{exp.sample_name})"
f" started at ({exp.started_at})")
_repr.append(s)
_repr_str = "\n".join(_repr)
if load_last_duplicate:
e = exp
else:
raise ValueError(f"Many experiments matching your request"
f" found:\n{_repr_str}")
else:
e = Experiment(exp_id=exp_ids[0], conn=conn)
_set_default_experiment_id(path_to_dbfile(conn), e.exp_id)
return e
def load_by_run_spec(*,
captured_run_id: Optional[int] = None,
captured_counter: Optional[int] = None,
experiment_name: Optional[str] = None,
sample_name: Optional[str] = None,
# guid parts
sample_id: Optional[int] = None,
location: Optional[int] = None,
work_station: Optional[int] = None,
conn: Optional[ConnectionPlus] = None) -> DataSet:
"""
Load a run from one or more pieces of runs specification. All
fields are optional but the function will raise an error if more than one
run matching the supplied specification is found. Along with the error
specs of the runs found will be printed.
Args:
captured_run_id: The run_id that was originally assigned to this
at the time of capture.
captured_counter: The counter that was originally assigned to this
at the time of capture.
experiment_name: name of the experiment that the run was captured
sample_name: The name of the sample given when creating the experiment.
sample_id: The sample_id assigned as part of the GUID.
location: The location code assigned as part of GUID.
work_station: The workstation assigned as part of the GUID.
conn: An optional connection to the database. If no connection is
supplied a connection to the default database will be opened.
Raises:
NameError: if no run or more than one run with the given specification
exists in the database
Returns:
:class:`.DataSet` matching the provided specification.
"""
conn = conn or connect(get_DB_location())
guids = get_guids_from_run_spec(conn,
captured_run_id=captured_run_id,
captured_counter=captured_counter,
experiment_name=experiment_name,
sample_name=sample_name)
matched_guids = filter_guids_by_parts(guids, location, sample_id,
work_station)
if len(matched_guids) == 1:
return load_by_guid(matched_guids[0], conn)
elif len(matched_guids) > 1:
print(generate_dataset_table(matched_guids, conn=conn))
raise NameError("More than one matching dataset found. "
"Please supply more information to uniquely"
"identify a dataset")
else:
raise NameError(f'No run matching the supplied information '
f'found.')
def load_last_experiment() -> Experiment:
"""
Load last experiment (from database file from config)
Returns:
last experiment
"""
last_exp_id = get_last_experiment(connect(get_DB_location()))
if last_exp_id is None:
raise ValueError('There are no experiments in the database file')
return Experiment(exp_id=last_exp_id)
def empty_temp_db_connection():
"""
Yield connection to an empty temporary DB file.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
path = os.path.join(tmpdirname, 'source.db')
conn = connect(path)
try:
yield conn
finally:
conn.close()
def two_empty_temp_db_connections(tmp_path):
"""
Yield connections to two empty files. Meant for use with the
test_database_extract_runs
"""
source_path = str(tmp_path / 'source.db')
target_path = str(tmp_path / 'target.db')
source_conn = connect(source_path)
target_conn = connect(target_path)
try:
yield (source_conn, target_conn)
finally:
source_conn.close()
target_conn.close()
# there is a very real chance that the tests will leave open
# connections to the database. These will have gone out of scope at
# this stage but a gc collection may not have run. The gc
# collection ensures that all connections belonging to now out of
# scope objects will be closed
gc.collect()
def load_last_experiment(conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Load last experiment (from database file from config)
Returns:
last experiment
"""
conn = conn or connect(get_DB_location())
last_exp_id = get_last_experiment(conn)
if last_exp_id is None:
raise ValueError('There are no experiments in the database file')
return Experiment(exp_id=last_exp_id, conn=conn)
def test_runs_table_columns(empty_temp_db):
"""
Ensure that the column names of a pristine runs table are what we expect
"""
colnames = mut_queries.RUNS_TABLE_COLUMNS.copy()
conn = mut_db.connect(get_DB_location())
query = "PRAGMA table_info(runs)"
cursor = conn.cursor()
for row in cursor.execute(query):
colnames.remove(row['name'])
assert colnames == []
def test_tables_exist(empty_temp_db, version):
conn = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"],
version=version)
cursor = conn.execute("select sql from sqlite_master"
" where type = 'table'")
expected_tables = ['experiments', 'runs', 'layouts', 'dependencies']
rows = [row for row in cursor]
assert len(rows) == len(expected_tables)
for row, expected_table in zip(rows, expected_tables):
assert expected_table in row['sql']
conn.close()
def experiments() -> List[Experiment]:
"""
List all the experiments in the container (database file from config)
Returns:
All the experiments in the container
"""
log.info("loading experiments from {}".format(get_DB_location()))
rows = get_experiments(connect(get_DB_location(), get_DB_debug()))
experiments = []
for row in rows:
experiments.append(load_experiment(row['exp_id']))
return experiments