def generate_some_links(N: int) -> List[Link]:
"""
Generate N links with the same head
"""
def _timestamp() -> int:
"""
return a random timestamp that is approximately
one day in the past.
"""
timestamp = datetime.now() - timedelta(days=1,
seconds=random.randint(1, 1000))
return int(round(timestamp.timestamp() * 1000))
known_types = ("fit", "analysis", "step")
known_descs = ("A second-order fit", "Manual analysis (see notebook)",
"Step 3 in the characterisation")
head_guid = generate_guid(_timestamp())
head_guids = [head_guid] * N
tail_guids = [generate_guid(_timestamp()) for _ in range(N)]
edge_types = [known_types[i % len(known_types)] for i in range(N)]
descriptions = [known_descs[i % len(known_descs)] for i in range(N)]
zipattrs = zip(head_guids, tail_guids, edge_types, descriptions)
links = [Link(hg, tg, n, d) for hg, tg, n, d in zipattrs]
return links
def test_atomic_creation(experiment):
""""
Test that dataset creation is atomic. Test for
https://github.com/QCoDeS/Qcodes/issues/1444
"""
def just_throw(*args):
raise RuntimeError("This breaks adding metadata")
# first we patch add_data_to_dynamic_columns to throw an exception
# if create_data is not atomic this would create a partial
# run in the db. Causing the next create_run to fail
with patch("qcodes.dataset.sqlite.queries.add_data_to_dynamic_columns",
new=just_throw):
x = ParamSpec("x", "numeric")
t = ParamSpec("t", "numeric")
y = ParamSpec("y", "numeric", depends_on=["x", "t"])
with pytest.raises(
RuntimeError,
match="Rolling back due to unhandled exception") as e:
mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
parameters=[x, t, y],
metadata={"a": 1},
)
assert error_caused_by(e, "This breaks adding metadata")
# since we are starting from an empty database and the above transaction
# should be rolled back there should be no runs in the run table
runs = mut_conn.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut_conn.transaction(new_conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 0
# if the above was not correctly rolled back we
# expect the next creation of a run to fail
mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y],
metadata={'a': 1})
runs = mut_conn.transaction(experiment.conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
with shadow_conn(experiment.path_to_db) as new_conn:
runs = mut_conn.transaction(new_conn,
'SELECT run_id FROM runs').fetchall()
assert len(runs) == 1
def test_link_to_string_and_back():
head_guid = generate_guid()
tail_guid = generate_guid()
edge_type = "analysis"
description = "hyper-spectral quantum blockchain ML"
link = Link(head_guid, tail_guid, edge_type, description)
lstr = link_to_str(link)
newlink = str_to_link(lstr)
assert newlink == link
def test_link_construction_raises(not_guid):
head_guid = generate_guid()
tail_guid = generate_guid()
edge_type = "fit"
match = re.escape(f'The guid given for head is not a valid guid. Received '
f'{not_guid}.')
with pytest.raises(ValueError, match=match):
Link(not_guid, tail_guid, edge_type)
match = re.escape(f'The guid given for tail is not a valid guid. Received '
f'{not_guid}')
with pytest.raises(ValueError, match=match):
Link(head_guid, not_guid, edge_type)
def test_get_dependents(experiment):
# more parameters, more complicated dependencies
x = ParamSpecBase("x", "numeric")
t = ParamSpecBase("t", "numeric")
y = ParamSpecBase("y", "numeric")
x_raw = ParamSpecBase("x_raw", "numeric")
x_cooked = ParamSpecBase("x_cooked", "numeric")
z = ParamSpecBase("z", "numeric")
deps_param_tree = {y: (x, t), z: (x_cooked, )}
inferred_param_tree = {x_cooked: (x_raw, )}
interdeps = InterDependencies_(dependencies=deps_param_tree,
inferences=inferred_param_tree)
description = RunDescriber(interdeps=interdeps)
(_, run_id, _) = mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
description=description,
)
deps = mut_queries._get_dependents(experiment.conn, run_id)
expected_deps = [
mut_queries._get_layout_id(experiment.conn, 'y', run_id),
mut_queries._get_layout_id(experiment.conn, 'z', run_id)
]
assert deps == expected_deps
def test_str_to_link():
head_guid = generate_guid()
tail_guid = generate_guid()
edge_type = "test"
description = "used in test_str_to_link"
lstr = json.dumps({
"head": head_guid,
"tail": tail_guid,
"edge_type": edge_type,
"description": description
})
expected_link = Link(head_guid, tail_guid, edge_type, description)
assert str_to_link(lstr) == expected_link
def test_link_construction_passes():
head_guid = generate_guid()
tail_guid = generate_guid()
edge_type = "fit"
description = "We did a second order fit with math"
link = Link(head_guid, tail_guid, edge_type)
assert link.head == head_guid
assert link.tail == tail_guid
assert link.edge_type == edge_type
assert link.description == ""
link = Link(head_guid, tail_guid, edge_type, description)
assert link.description == description
def generate_some_links(N: int) -> List[Link]:
"""
Generate N links with the same head
"""
known_types = ("fit", "analysis", "step")
known_descs = ("A second-order fit", "Manual analysis (see notebook)",
"Step 3 in the characterisation")
head_guid = generate_guid()
head_guids = [head_guid] * N
tail_guids = [generate_guid() for _ in range(N)]
edge_types = [known_types[i % len(known_types)] for i in range(N)]
descriptions = [known_descs[i % len(known_descs)] for i in range(N)]
zipattrs = zip(head_guids, tail_guids, edge_types, descriptions)
links = [Link(hg, tg, n, d) for hg, tg, n, d in zipattrs]
return links
def test_get_dependents(experiment):
x = ParamSpec('x', 'numeric')
t = ParamSpec('t', 'numeric')
y = ParamSpec('y', 'numeric', depends_on=['x', 't'])
# Make a dataset
(_, run_id, _) = mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, y])
deps = mut_queries._get_dependents(experiment.conn, run_id)
layout_id = mut_queries._get_layout_id(experiment.conn, 'y', run_id)
assert deps == [layout_id]
# more parameters, more complicated dependencies
x_raw = ParamSpec('x_raw', 'numeric')
x_cooked = ParamSpec('x_cooked', 'numeric', inferred_from=['x_raw'])
z = ParamSpec('z', 'numeric', depends_on=['x_cooked'])
(_, run_id,
_) = mut_queries.create_run(experiment.conn,
experiment.exp_id,
name='testrun',
guid=generate_guid(),
parameters=[x, t, x_raw, x_cooked, y, z])
deps = mut_queries._get_dependents(experiment.conn, run_id)
expected_deps = [
mut_queries._get_layout_id(experiment.conn, 'y', run_id),
mut_queries._get_layout_id(experiment.conn, 'z', run_id)
]
assert deps == expected_deps
def _new(self, name, exp_id, specs: SPECS = None, values=None,
metadata=None) -> None:
"""
Actually perform all the side effects needed for
the creation of a new dataset.
"""
_, run_id, __ = create_run(self.conn, exp_id, name,
generate_guid(),
specs, values, metadata)
# this is really the UUID (an ever increasing count in the db)
self.run_id = run_id
self._completed = False
def _both_zero(run_id: int, conn, guid_comps) -> None:
guid_str = generate_guid(timeint=guid_comps['time'],
sampleint=guid_comps['sample'])
with atomic(conn) as conn:
sql = f"""
UPDATE runs
SET guid = ?
where run_id == {run_id}
"""
cur = conn.cursor()
cur.execute(sql, (guid_str, ))
log.info(f'Succesfully updated run number {run_id}.')
def perform_db_upgrade_0_to_1(conn: SomeConnection) -> None:
"""
Perform the upgrade from version 0 to version 1
Returns:
A bool indicating whether everything went well. The next upgrade in
the upgrade chain should run conditioned in this bool
"""
log.info('Starting database upgrade version 0 -> 1')
start_version = get_user_version(conn)
if start_version != 0:
log.warn('Can not upgrade, current database version is '
f'{start_version}, aborting.')
return
sql = "SELECT name FROM sqlite_master WHERE type='table' AND name='runs'"
cur = atomic_transaction(conn, sql)
n_run_tables = len(cur.fetchall())
if n_run_tables == 1:
with atomic(conn) as conn:
sql = "ALTER TABLE runs ADD COLUMN guid TEXT"
transaction(conn, sql)
# now assign GUIDs to existing runs
cur = transaction(conn, 'SELECT run_id FROM runs')
run_ids = [r[0] for r in many_many(cur, 'run_id')]
for run_id in run_ids:
query = f"""
SELECT run_timestamp
FROM runs
WHERE run_id == {run_id}
"""
cur = transaction(conn, query)
timestamp = one(cur, 'run_timestamp')
timeint = int(np.round(timestamp * 1000))
sql = f"""
UPDATE runs
SET guid = ?
where run_id == {run_id}
"""
sampleint = 3736062718 # 'deafcafe'
cur.execute(
sql,
(generate_guid(timeint=timeint, sampleint=sampleint), ))
else:
raise RuntimeError(f"found {n_run_tables} runs tables expected 1")
log.info('Succesfully upgraded database version 0 -> 1.')
set_user_version(conn, 1)
def test_get_dependents_simple(experiment, simple_run_describer):
(_, run_id, _) = mut_queries.create_run(
experiment.conn,
experiment.exp_id,
name="testrun",
guid=generate_guid(),
description=simple_run_describer,
)
deps = mut_queries._get_dependents(experiment.conn, run_id)
layout_id = mut_queries._get_layout_id(experiment.conn, "y", run_id)
assert deps == [layout_id]
def make_test_guid(cfg, loc: int, smpl: int, stat: int):
cfg['GUID_components']['location'] = loc
cfg['GUID_components']['work_station'] = stat
cfg['GUID_components']['sample'] = smpl
guid = generate_guid()
gen_time = int(np.round(time.time() * 1000))
comps = parse_guid(guid)
assert comps['location'] == loc
assert comps['work_station'] == stat
assert comps['sample'] == smpl
assert comps['time'] - gen_time < 2
return guid
def _create_new_run(
cls,
name: str,
path_to_db: Optional[Union[Path, str]] = None,
exp_id: Optional[int] = None,
) -> DataSetInMem:
if path_to_db is not None:
path_to_db = str(path_to_db)
with contextlib.closing(
conn_from_dbpath_or_conn(conn=None,
path_to_db=path_to_db)) as conn:
if exp_id is None:
exp_id = get_default_experiment_id(conn)
name = name or "dataset"
sample_name = get_sample_name_from_experiment_id(conn, exp_id)
exp_name = get_experiment_name_from_experiment_id(conn, exp_id)
guid = generate_guid()
run_counter, run_id, _ = create_run(conn,
exp_id,
name,
guid=guid,
parameters=None,
create_run_table=False)
ds = cls(
run_id=run_id,
captured_run_id=run_id,
counter=run_counter,
captured_counter=run_counter,
name=name,
exp_id=exp_id,
exp_name=exp_name,
sample_name=sample_name,
guid=guid,
path_to_db=conn.path_to_dbfile,
run_timestamp_raw=None,
completed_timestamp_raw=None,
metadata=None,
)
return ds
def perform_db_upgrade_0_to_1(conn: ConnectionPlus) -> None:
"""
Perform the upgrade from version 0 to version 1
Add a GUID column to the runs table and assign guids for all existing runs
"""
sql = "SELECT name FROM sqlite_master WHERE type='table' AND name='runs'"
cur = atomic_transaction(conn, sql)
n_run_tables = len(cur.fetchall())
if n_run_tables == 1:
with atomic(conn) as conn:
sql = "ALTER TABLE runs ADD COLUMN guid TEXT"
transaction(conn, sql)
# now assign GUIDs to existing runs
cur = transaction(conn, 'SELECT run_id FROM runs')
run_ids = [r[0] for r in many_many(cur, 'run_id')]
pbar = tqdm(range(1, len(run_ids) + 1), file=sys.stdout)
pbar.set_description("Upgrading database; v0 -> v1")
for run_id in pbar:
query = f"""
SELECT run_timestamp
FROM runs
WHERE run_id == {run_id}
"""
cur = transaction(conn, query)
timestamp = one(cur, 'run_timestamp')
timeint = int(np.round(timestamp * 1000))
sql = f"""
UPDATE runs
SET guid = ?
where run_id == {run_id}
"""
sampleint = 3736062718 # 'deafcafe'
cur.execute(
sql,
(generate_guid(timeint=timeint, sampleint=sampleint), ))
else:
raise RuntimeError(f"found {n_run_tables} runs tables expected 1")
def test_generate_guid(loc, stat, smpl):
# update config to generate a particular guid. Read it back to verify
with default_config():
cfg = qc.config
cfg['GUID_components']['location'] = loc
cfg['GUID_components']['work_station'] = stat
cfg['GUID_components']['sample'] = smpl
guid = generate_guid()
gen_time = int(np.round(time.time() * 1000))
comps = parse_guid(guid)
if smpl == 0:
smpl = int('a' * 8, base=16)
assert comps['location'] == loc
assert comps['work_station'] == stat
assert comps['sample'] == smpl
assert comps['time'] - gen_time < 2
def __init__(self,
path_to_db: str = None,
run_id: Optional[int] = None,
conn: Optional[ConnectionPlus] = None,
exp_id=None,
name: str = None,
specs: Optional[SpecsOrInterDeps] = None,
values=None,
metadata=None) -> None:
"""
Create a new DataSet object. The object can either hold a new run or
an already existing run. If a run_id is provided, then an old run is
looked up, else a new run is created.
Args:
path_to_db: path to the sqlite file on disk. If not provided, the
path will be read from the config.
run_id: provide this when loading an existing run, leave it
as None when creating a new run
conn: connection to the DB; if provided and `path_to_db` is
provided as well, then a ValueError is raised (this is to
prevent the possibility of providing a connection to a DB
file that is different from `path_to_db`)
exp_id: the id of the experiment in which to create a new run.
Ignored if run_id is provided.
name: the name of the dataset. Ignored if run_id is provided.
specs: paramspecs belonging to the dataset. Ignored if run_id is
provided.
values: values to insert into the dataset. Ignored if run_id is
provided.
metadata: metadata to insert into the dataset. Ignored if run_id
is provided.
"""
if path_to_db is not None and conn is not None:
raise ValueError("Both `path_to_db` and `conn` arguments have "
"been passed together with non-None values. "
"This is not allowed.")
self._path_to_db = path_to_db or get_DB_location()
self.conn = make_connection_plus_from(conn) if conn is not None else \
connect(self.path_to_db)
self._run_id = run_id
self._debug = False
self.subscribers: Dict[str, _Subscriber] = {}
self._interdeps: InterDependencies_
if run_id is not None:
if not run_exists(self.conn, run_id):
raise ValueError(f"Run with run_id {run_id} does not exist in "
f"the database")
self._completed = completed(self.conn, self.run_id)
run_desc = self._get_run_description_from_db()
if run_desc._old_style_deps:
# TODO: what if the old run had invalid interdep.s?
old_idps: InterDependencies = cast(InterDependencies,
run_desc.interdeps)
self._interdeps = old_to_new(old_idps)
else:
new_idps: InterDependencies_ = cast(InterDependencies_,
run_desc.interdeps)
self._interdeps = new_idps
self._metadata = get_metadata_from_run_id(self.conn, run_id)
self._started = self.run_timestamp_raw is not None
else:
# Actually perform all the side effects needed for the creation
# of a new dataset. Note that a dataset is created (in the DB)
# with no parameters; they are written to disk when the dataset
# is marked as started
if exp_id is None:
if len(get_experiments(self.conn)) > 0:
exp_id = get_last_experiment(self.conn)
else:
raise ValueError("No experiments found."
"You can start a new one with:"
" new_experiment(name, sample_name)")
name = name or "dataset"
_, run_id, __ = create_run(self.conn,
exp_id,
name,
generate_guid(),
parameters=None,
values=values,
metadata=metadata)
# this is really the UUID (an ever increasing count in the db)
self._run_id = run_id
self._completed = False
self._started = False
if isinstance(specs, InterDependencies_):
self._interdeps = specs
elif specs is not None:
self._interdeps = old_to_new(InterDependencies(*specs))
else:
self._interdeps = InterDependencies_()
self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
def __init__(self, path_to_db: str=None,
run_id: Optional[int]=None,
conn=None,
exp_id=None,
name: str=None,
specs: SPECS=None,
values=None,
metadata=None) -> None:
"""
Create a new DataSet object. The object can either hold a new run or
an already existing run. If a run_id is provided, then an old run is
looked up, else a new run is created.
Args:
path_to_db: path to the sqlite file on disk. If not provided, the
path will be read from the config.
run_id: provide this when loading an existing run, leave it
as None when creating a new run
conn: connection to the DB
exp_id: the id of the experiment in which to create a new run.
Ignored if run_id is provided.
name: the name of the dataset. Ignored if run_id is provided.
specs: paramspecs belonging to the dataset. Ignored if run_id is
provided.
values: values to insert into the dataset. Ignored if run_id is
provided.
metadata: metadata to insert into the dataset. Ignored if run_id
is provided.
"""
# TODO: handle fail here by defaulting to
# a standard db
self.path_to_db = path_to_db or get_DB_location()
if conn is None:
self.conn = connect(self.path_to_db)
else:
self.conn = conn
self.run_id = run_id
self._debug = False
self.subscribers: Dict[str, _Subscriber] = {}
if run_id:
if not run_exists(self.conn, run_id):
raise ValueError(f"Run with run_id {run_id} does not exist in "
f"the database")
self._completed = completed(self.conn, self.run_id)
else:
if exp_id is None:
if len(get_experiments(self.conn)) > 0:
exp_id = get_last_experiment(self.conn)
else:
raise ValueError("No experiments found."
"You can start a new one with:"
" new_experiment(name, sample_name)")
# Actually perform all the side effects needed for
# the creation of a new dataset.
name = name or "dataset"
_, run_id, __ = create_run(self.conn, exp_id, name,
generate_guid(),
specs, values, metadata)
# this is really the UUID (an ever increasing count in the db)
self.run_id = run_id
self._completed = False