def test_perform_actual_upgrade_2_to_3_empty():
v2fixpath = os.path.join(fixturepath, 'db_files', 'version2')
dbname_old = os.path.join(v2fixpath, 'empty.db')
if not os.path.exists(dbname_old):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn:
assert get_user_version(conn) == 2
desc_query = 'SELECT run_description FROM runs'
with pytest.raises(RuntimeError) as excinfo:
atomic_transaction(conn, desc_query)
assert error_caused_by(excinfo, 'no such column: run_description')
perform_db_upgrade_2_to_3(conn)
assert get_user_version(conn) == 3
c = atomic_transaction(conn, desc_query)
assert len(c.fetchall()) == 0
def test_perform_actual_upgrade_0_to_1():
# we cannot use the empty_temp_db, since that has already called connect
# and is therefore latest version already
v0fixpath = os.path.join(fixturepath, 'db_files', 'version0')
if not os.path.exists(v0fixpath):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
dbname_old = os.path.join(v0fixpath, 'empty.db')
with temporarily_copied_DB(dbname_old, debug=False, version=0) as conn:
assert get_user_version(conn) == 0
guid_table_query = "SELECT guid FROM runs"
with pytest.raises(RuntimeError):
atomic_transaction(conn, guid_table_query)
perform_db_upgrade_0_to_1(conn)
assert get_user_version(conn) == 1
c = atomic_transaction(conn, guid_table_query)
assert len(c.fetchall()) == 0
def test_perform_actual_upgrade_0_to_1():
# we cannot use the empty_temp_db, since that has already called connect
# and is therefore latest version already
v0fixpath = os.path.join(fixturepath, 'db_files', 'version0')
dbname_old = os.path.join(v0fixpath, 'empty.db')
if not os.path.exists(dbname_old):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the "
"https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
with temporarily_copied_DB(dbname_old, debug=False, version=0) as conn:
assert get_user_version(conn) == 0
guid_table_query = "SELECT guid FROM runs"
with pytest.raises(RuntimeError) as excinfo:
atomic_transaction(conn, guid_table_query)
assert error_caused_by(excinfo, 'no such column: guid')
perform_db_upgrade_0_to_1(conn)
assert get_user_version(conn) == 1
c = atomic_transaction(conn, guid_table_query)
assert len(c.fetchall()) == 0
def test_perform_actual_upgrade_1_to_2():
v1fixpath = os.path.join(fixturepath, 'db_files', 'version1')
if not os.path.exists(v1fixpath):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
dbname_old = os.path.join(v1fixpath, 'empty.db')
with temporarily_copied_DB(dbname_old, debug=False, version=1) as conn:
assert get_user_version(conn) == 1
guid_table_query = "SELECT guid FROM runs"
c = atomic_transaction(conn, guid_table_query)
assert len(c.fetchall()) == 0
index_query = "PRAGMA index_list(runs)"
c = atomic_transaction(conn, index_query)
assert len(c.fetchall()) == 0
perform_db_upgrade_1_to_2(conn)
c = atomic_transaction(conn, index_query)
assert len(c.fetchall()) == 2
def test_fix_wrong_run_descriptions():
v3fixpath = os.path.join(fixturepath, 'db_files', 'version3')
dbname_old = os.path.join(v3fixpath,
'some_runs_without_run_description.db')
if not os.path.exists(dbname_old):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn:
assert get_user_version(conn) == 3
ds1 = DataSet(conn=conn, run_id=1)
expected_description = ds1.description
empty_description = RunDescriber(InterDependencies_())
_fix_wrong_run_descriptions(conn, [1, 2, 3, 4])
ds2 = DataSet(conn=conn, run_id=2)
assert expected_description == ds2.description
ds3 = DataSet(conn=conn, run_id=3)
assert expected_description == ds3.description
ds4 = DataSet(conn=conn, run_id=4)
assert empty_description == ds4.description
def test_perform_actual_upgrade_0_to_1():
# we cannot use the empty_temp_db, since that has already called connect
# and is therefore latest version already
connection = connect(':memory:', debug=False, version=0)
assert get_user_version(connection) == 0
guid_table_query = "SELECT guid FROM runs"
with pytest.raises(RuntimeError):
atomic_transaction(connection, guid_table_query)
perform_db_upgrade_0_to_1(connection)
assert get_user_version(connection) == 1
c = atomic_transaction(connection, guid_table_query)
assert len(c.fetchall()) == 0
def test_cannot_connect_to_newer_db():
conn = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"])
current_version = get_user_version(conn)
set_user_version(conn, current_version + 1)
conn.close()
err_msg = f'is version {current_version + 1} but this version of QCoDeS ' \
f'supports up to version {current_version}'
with pytest.raises(RuntimeError, match=err_msg):
conn = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"])
def test_database_upgrade(empty_temp_db):
connection = connect(qc.config["core"]["db_location"],
qc.config["core"]["db_debug"])
userversion = get_user_version(connection)
if userversion != 0:
raise RuntimeError("trying to upgrade from version 0"
" but your database is version"
" {}".format(userversion))
sql = 'ALTER TABLE "runs" ADD COLUMN "quality"'
atomic_transaction(connection, sql)
set_user_version(connection, 1)
def test_perform_actual_upgrade_2_to_3_some_runs():
v2fixpath = os.path.join(fixturepath, 'db_files', 'version2')
dbname_old = os.path.join(v2fixpath, 'some_runs.db')
if not os.path.exists(dbname_old):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn:
assert get_user_version(conn) == 2
perform_db_upgrade_2_to_3(conn)
desc_query = 'SELECT run_description FROM runs'
c = atomic_transaction(conn, desc_query)
assert len(c.fetchall()) == 10
# retrieve the json string and recreate the object
sql = f"""
SELECT run_description
FROM runs
WHERE run_id == 1
"""
c = atomic_transaction(conn, sql)
json_str = one(c, 'run_description')
desc = RunDescriber.from_json(json_str)
idp = desc.interdeps
assert isinstance(idp, InterDependencies)
# here we verify that the dependencies encoded in
# tests/dataset/legacy_DB_generation/generate_version_2.py
# are recovered
p0 = [p for p in idp.paramspecs if p.name == 'p0'][0]
assert p0.depends_on == ''
assert p0.inferred_from == ''
assert p0.label == "Parameter 0"
assert p0.unit == "unit 0"
p4 = [p for p in idp.paramspecs if p.name == 'p4'][0]
assert p4.depends_on == 'p2, p3'
assert p4.inferred_from == ''
assert p4.label == "Parameter 4"
assert p4.unit == "unit 4"
def fix_version_4a_run_description_bug(conn: ConnectionPlus) -> Dict[str, int]:
"""
Fix function to fix a bug where the RunDescriber accidentally wrote itself
to string using the (new) InterDependencies_ object instead of the (old)
InterDependencies object. After the first run, this function should be
idempotent.
Args:
conn: the connection to the database
Returns:
A dict with the fix results ('runs_inspected', 'runs_fixed')
"""
user_version = get_user_version(conn)
if not user_version == 4:
raise RuntimeError('Database of wrong version. Will not apply fix. '
'Expected version 4, found version {user_version}')
no_of_runs_query = "SELECT max(run_id) FROM runs"
no_of_runs = one(atomic_transaction(conn, no_of_runs_query), 'max(run_id)')
no_of_runs = no_of_runs or 0
with atomic(conn) as conn:
pbar = tqdm(range(1, no_of_runs + 1))
pbar.set_description("Fixing database")
# collect some metrics
runs_inspected = 0
runs_fixed = 0
for run_id in pbar:
desc_str = get_run_description(conn, run_id)
desc_ser = json.loads(desc_str)
idps_ser = desc_ser['interdependencies']
if RunDescriber._is_description_old_style(idps_ser):
pass
else:
new_desc = RunDescriber.from_json(desc_str)
update_run_description(conn, run_id, new_desc.to_json())
runs_fixed += 1
runs_inspected += 1
return {'runs_inspected': runs_inspected, 'runs_fixed': runs_fixed}
def fix_wrong_run_descriptions(conn: ConnectionPlus,
run_ids: Sequence[int]) -> None:
"""
NB: This is a FIX function. Do not use it unless your database has been
diagnosed with the problem that this function fixes.
Overwrite faulty run_descriptions by using information from the layouts and
dependencies tables. If a correct description is found for a run, that
run is left untouched.
Args:
conn: The connection to the database
run_ids: The runs to (potentially) fix
"""
user_version = get_user_version(conn)
if not user_version == 3:
raise RuntimeError('Database of wrong version. Will not apply fix. '
'Expected version 3, found version {user_version}')
log.info('[*] Fixing run descriptions...')
for run_id in run_ids:
trusted_paramspecs = get_parameters(conn, run_id)
trusted_desc = RunDescriber(interdeps=InterDependencies(
*trusted_paramspecs))
actual_desc_str = select_one_where(conn, "runs", "run_description",
"run_id", run_id)
if actual_desc_str == trusted_desc.to_json():
log.info(f'[+] Run id: {run_id} had an OK description')
else:
log.info(f'[-] Run id: {run_id} had a broken description. '
f'Description found: {actual_desc_str}')
update_run_description(conn, run_id, trusted_desc.to_json())
log.info(f' Run id: {run_id} has been updated.')
def test_connect_upgrades_user_version(ver):
expected_version = ver if ver != LATEST_VERSION_ARG else LATEST_VERSION
conn = connect(':memory:', version=ver)
assert expected_version == get_user_version(conn)
def test_perform_upgrade_v3_to_v4():
"""
Test that a db upgrade from v2 to v4 works correctly.
"""
v3fixpath = os.path.join(fixturepath, 'db_files', 'version3')
dbname_old = os.path.join(v3fixpath, 'some_runs_upgraded_2.db')
if not os.path.exists(dbname_old):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the "
"https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn:
assert get_user_version(conn) == 3
sql = f"""
SELECT run_description
FROM runs
WHERE run_id == 1
"""
perform_db_upgrade_3_to_4(conn)
c = atomic_transaction(conn, sql)
json_str = one(c, 'run_description')
desc = RunDescriber.from_json(json_str)
idp = desc.interdeps
assert isinstance(idp, InterDependencies)
p0 = [p for p in idp.paramspecs if p.name == 'p0'][0]
assert p0.depends_on == ''
assert p0.depends_on_ == []
assert p0.inferred_from == ''
assert p0.inferred_from_ == []
assert p0.label == "Parameter 0"
assert p0.unit == "unit 0"
p1 = [p for p in idp.paramspecs if p.name == 'p1'][0]
assert p1.depends_on == ''
assert p1.depends_on_ == []
assert p1.inferred_from == ''
assert p1.inferred_from_ == []
assert p1.label == "Parameter 1"
assert p1.unit == "unit 1"
p2 = [p for p in idp.paramspecs if p.name == 'p2'][0]
assert p2.depends_on == ''
assert p2.depends_on_ == []
assert p2.inferred_from == 'p0'
assert p2.inferred_from_ == ['p0']
assert p2.label == "Parameter 2"
assert p2.unit == "unit 2"
p3 = [p for p in idp.paramspecs if p.name == 'p3'][0]
assert p3.depends_on == ''
assert p3.depends_on_ == []
assert p3.inferred_from == 'p1, p0'
assert p3.inferred_from_ == ['p1', 'p0']
assert p3.label == "Parameter 3"
assert p3.unit == "unit 3"
p4 = [p for p in idp.paramspecs if p.name == 'p4'][0]
assert p4.depends_on == 'p2, p3'
assert p4.depends_on_ == ['p2', 'p3']
assert p4.inferred_from == ''
assert p4.inferred_from_ == []
assert p4.label == "Parameter 4"
assert p4.unit == "unit 4"
p5 = [p for p in idp.paramspecs if p.name == 'p5'][0]
assert p5.depends_on == ''
assert p5.depends_on_ == []
assert p5.inferred_from == 'p0'
assert p5.inferred_from_ == ['p0']
assert p5.label == "Parameter 5"
assert p5.unit == "unit 5"
