def generate_some_links(N: int) -> List[Link]: """ Generate N links with the same head """ def _timestamp() -> int: """ return a random timestamp that is approximately one day in the past. """ timestamp = datetime.now() - timedelta(days=1, seconds=random.randint(1, 1000)) return int(round(timestamp.timestamp() * 1000)) known_types = ("fit", "analysis", "step") known_descs = ("A second-order fit", "Manual analysis (see notebook)", "Step 3 in the characterisation") head_guid = generate_guid(_timestamp()) head_guids = [head_guid] * N tail_guids = [generate_guid(_timestamp()) for _ in range(N)] edge_types = [known_types[i % len(known_types)] for i in range(N)] descriptions = [known_descs[i % len(known_descs)] for i in range(N)] zipattrs = zip(head_guids, tail_guids, edge_types, descriptions) links = [Link(hg, tg, n, d) for hg, tg, n, d in zipattrs] return links
def test_atomic_creation(experiment): """" Test that dataset creation is atomic. Test for https://github.com/QCoDeS/Qcodes/issues/1444 """ def just_throw(*args): raise RuntimeError("This breaks adding metadata") # first we patch add_data_to_dynamic_columns to throw an exception # if create_data is not atomic this would create a partial # run in the db. Causing the next create_run to fail with patch("qcodes.dataset.sqlite.queries.add_data_to_dynamic_columns", new=just_throw): x = ParamSpec("x", "numeric") t = ParamSpec("t", "numeric") y = ParamSpec("y", "numeric", depends_on=["x", "t"]) with pytest.raises( RuntimeError, match="Rolling back due to unhandled exception") as e: mut_queries.create_run( experiment.conn, experiment.exp_id, name="testrun", guid=generate_guid(), parameters=[x, t, y], metadata={"a": 1}, ) assert error_caused_by(e, "This breaks adding metadata") # since we are starting from an empty database and the above transaction # should be rolled back there should be no runs in the run table runs = mut_conn.transaction(experiment.conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 0 with shadow_conn(experiment.path_to_db) as new_conn: runs = mut_conn.transaction(new_conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 0 # if the above was not correctly rolled back we # expect the next creation of a run to fail mut_queries.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, y], metadata={'a': 1}) runs = mut_conn.transaction(experiment.conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 1 with shadow_conn(experiment.path_to_db) as new_conn: runs = mut_conn.transaction(new_conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 1
def test_link_to_string_and_back(): head_guid = generate_guid() tail_guid = generate_guid() edge_type = "analysis" description = "hyper-spectral quantum blockchain ML" link = Link(head_guid, tail_guid, edge_type, description) lstr = link_to_str(link) newlink = str_to_link(lstr) assert newlink == link
def test_link_construction_raises(not_guid): head_guid = generate_guid() tail_guid = generate_guid() edge_type = "fit" match = re.escape(f'The guid given for head is not a valid guid. Received ' f'{not_guid}.') with pytest.raises(ValueError, match=match): Link(not_guid, tail_guid, edge_type) match = re.escape(f'The guid given for tail is not a valid guid. Received ' f'{not_guid}') with pytest.raises(ValueError, match=match): Link(head_guid, not_guid, edge_type)
def test_get_dependents(experiment): # more parameters, more complicated dependencies x = ParamSpecBase("x", "numeric") t = ParamSpecBase("t", "numeric") y = ParamSpecBase("y", "numeric") x_raw = ParamSpecBase("x_raw", "numeric") x_cooked = ParamSpecBase("x_cooked", "numeric") z = ParamSpecBase("z", "numeric") deps_param_tree = {y: (x, t), z: (x_cooked, )} inferred_param_tree = {x_cooked: (x_raw, )} interdeps = InterDependencies_(dependencies=deps_param_tree, inferences=inferred_param_tree) description = RunDescriber(interdeps=interdeps) (_, run_id, _) = mut_queries.create_run( experiment.conn, experiment.exp_id, name="testrun", guid=generate_guid(), description=description, ) deps = mut_queries._get_dependents(experiment.conn, run_id) expected_deps = [ mut_queries._get_layout_id(experiment.conn, 'y', run_id), mut_queries._get_layout_id(experiment.conn, 'z', run_id) ] assert deps == expected_deps
def test_str_to_link(): head_guid = generate_guid() tail_guid = generate_guid() edge_type = "test" description = "used in test_str_to_link" lstr = json.dumps({ "head": head_guid, "tail": tail_guid, "edge_type": edge_type, "description": description }) expected_link = Link(head_guid, tail_guid, edge_type, description) assert str_to_link(lstr) == expected_link
def test_link_construction_passes(): head_guid = generate_guid() tail_guid = generate_guid() edge_type = "fit" description = "We did a second order fit with math" link = Link(head_guid, tail_guid, edge_type) assert link.head == head_guid assert link.tail == tail_guid assert link.edge_type == edge_type assert link.description == "" link = Link(head_guid, tail_guid, edge_type, description) assert link.description == description
def generate_some_links(N: int) -> List[Link]: """ Generate N links with the same head """ known_types = ("fit", "analysis", "step") known_descs = ("A second-order fit", "Manual analysis (see notebook)", "Step 3 in the characterisation") head_guid = generate_guid() head_guids = [head_guid] * N tail_guids = [generate_guid() for _ in range(N)] edge_types = [known_types[i % len(known_types)] for i in range(N)] descriptions = [known_descs[i % len(known_descs)] for i in range(N)] zipattrs = zip(head_guids, tail_guids, edge_types, descriptions) links = [Link(hg, tg, n, d) for hg, tg, n, d in zipattrs] return links
def test_get_dependents(experiment): x = ParamSpec('x', 'numeric') t = ParamSpec('t', 'numeric') y = ParamSpec('y', 'numeric', depends_on=['x', 't']) # Make a dataset (_, run_id, _) = mut_queries.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, y]) deps = mut_queries._get_dependents(experiment.conn, run_id) layout_id = mut_queries._get_layout_id(experiment.conn, 'y', run_id) assert deps == [layout_id] # more parameters, more complicated dependencies x_raw = ParamSpec('x_raw', 'numeric') x_cooked = ParamSpec('x_cooked', 'numeric', inferred_from=['x_raw']) z = ParamSpec('z', 'numeric', depends_on=['x_cooked']) (_, run_id, _) = mut_queries.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, x_raw, x_cooked, y, z]) deps = mut_queries._get_dependents(experiment.conn, run_id) expected_deps = [ mut_queries._get_layout_id(experiment.conn, 'y', run_id), mut_queries._get_layout_id(experiment.conn, 'z', run_id) ] assert deps == expected_deps
def _new(self, name, exp_id, specs: SPECS = None, values=None, metadata=None) -> None: """ Actually perform all the side effects needed for the creation of a new dataset. """ _, run_id, __ = create_run(self.conn, exp_id, name, generate_guid(), specs, values, metadata) # this is really the UUID (an ever increasing count in the db) self.run_id = run_id self._completed = False
def _both_zero(run_id: int, conn, guid_comps) -> None: guid_str = generate_guid(timeint=guid_comps['time'], sampleint=guid_comps['sample']) with atomic(conn) as conn: sql = f""" UPDATE runs SET guid = ? where run_id == {run_id} """ cur = conn.cursor() cur.execute(sql, (guid_str, )) log.info(f'Succesfully updated run number {run_id}.')
def perform_db_upgrade_0_to_1(conn: SomeConnection) -> None: """ Perform the upgrade from version 0 to version 1 Returns: A bool indicating whether everything went well. The next upgrade in the upgrade chain should run conditioned in this bool """ log.info('Starting database upgrade version 0 -> 1') start_version = get_user_version(conn) if start_version != 0: log.warn('Can not upgrade, current database version is ' f'{start_version}, aborting.') return sql = "SELECT name FROM sqlite_master WHERE type='table' AND name='runs'" cur = atomic_transaction(conn, sql) n_run_tables = len(cur.fetchall()) if n_run_tables == 1: with atomic(conn) as conn: sql = "ALTER TABLE runs ADD COLUMN guid TEXT" transaction(conn, sql) # now assign GUIDs to existing runs cur = transaction(conn, 'SELECT run_id FROM runs') run_ids = [r[0] for r in many_many(cur, 'run_id')] for run_id in run_ids: query = f""" SELECT run_timestamp FROM runs WHERE run_id == {run_id} """ cur = transaction(conn, query) timestamp = one(cur, 'run_timestamp') timeint = int(np.round(timestamp * 1000)) sql = f""" UPDATE runs SET guid = ? where run_id == {run_id} """ sampleint = 3736062718 # 'deafcafe' cur.execute( sql, (generate_guid(timeint=timeint, sampleint=sampleint), )) else: raise RuntimeError(f"found {n_run_tables} runs tables expected 1") log.info('Succesfully upgraded database version 0 -> 1.') set_user_version(conn, 1)
def test_get_dependents_simple(experiment, simple_run_describer): (_, run_id, _) = mut_queries.create_run( experiment.conn, experiment.exp_id, name="testrun", guid=generate_guid(), description=simple_run_describer, ) deps = mut_queries._get_dependents(experiment.conn, run_id) layout_id = mut_queries._get_layout_id(experiment.conn, "y", run_id) assert deps == [layout_id]
def make_test_guid(cfg, loc: int, smpl: int, stat: int): cfg['GUID_components']['location'] = loc cfg['GUID_components']['work_station'] = stat cfg['GUID_components']['sample'] = smpl guid = generate_guid() gen_time = int(np.round(time.time() * 1000)) comps = parse_guid(guid) assert comps['location'] == loc assert comps['work_station'] == stat assert comps['sample'] == smpl assert comps['time'] - gen_time < 2 return guid
def _create_new_run( cls, name: str, path_to_db: Optional[Union[Path, str]] = None, exp_id: Optional[int] = None, ) -> DataSetInMem: if path_to_db is not None: path_to_db = str(path_to_db) with contextlib.closing( conn_from_dbpath_or_conn(conn=None, path_to_db=path_to_db)) as conn: if exp_id is None: exp_id = get_default_experiment_id(conn) name = name or "dataset" sample_name = get_sample_name_from_experiment_id(conn, exp_id) exp_name = get_experiment_name_from_experiment_id(conn, exp_id) guid = generate_guid() run_counter, run_id, _ = create_run(conn, exp_id, name, guid=guid, parameters=None, create_run_table=False) ds = cls( run_id=run_id, captured_run_id=run_id, counter=run_counter, captured_counter=run_counter, name=name, exp_id=exp_id, exp_name=exp_name, sample_name=sample_name, guid=guid, path_to_db=conn.path_to_dbfile, run_timestamp_raw=None, completed_timestamp_raw=None, metadata=None, ) return ds
def perform_db_upgrade_0_to_1(conn: ConnectionPlus) -> None: """ Perform the upgrade from version 0 to version 1 Add a GUID column to the runs table and assign guids for all existing runs """ sql = "SELECT name FROM sqlite_master WHERE type='table' AND name='runs'" cur = atomic_transaction(conn, sql) n_run_tables = len(cur.fetchall()) if n_run_tables == 1: with atomic(conn) as conn: sql = "ALTER TABLE runs ADD COLUMN guid TEXT" transaction(conn, sql) # now assign GUIDs to existing runs cur = transaction(conn, 'SELECT run_id FROM runs') run_ids = [r[0] for r in many_many(cur, 'run_id')] pbar = tqdm(range(1, len(run_ids) + 1), file=sys.stdout) pbar.set_description("Upgrading database; v0 -> v1") for run_id in pbar: query = f""" SELECT run_timestamp FROM runs WHERE run_id == {run_id} """ cur = transaction(conn, query) timestamp = one(cur, 'run_timestamp') timeint = int(np.round(timestamp * 1000)) sql = f""" UPDATE runs SET guid = ? where run_id == {run_id} """ sampleint = 3736062718 # 'deafcafe' cur.execute( sql, (generate_guid(timeint=timeint, sampleint=sampleint), )) else: raise RuntimeError(f"found {n_run_tables} runs tables expected 1")
def test_generate_guid(loc, stat, smpl): # update config to generate a particular guid. Read it back to verify with default_config(): cfg = qc.config cfg['GUID_components']['location'] = loc cfg['GUID_components']['work_station'] = stat cfg['GUID_components']['sample'] = smpl guid = generate_guid() gen_time = int(np.round(time.time() * 1000)) comps = parse_guid(guid) if smpl == 0: smpl = int('a' * 8, base=16) assert comps['location'] == loc assert comps['work_station'] == stat assert comps['sample'] == smpl assert comps['time'] - gen_time < 2
def __init__(self, path_to_db: str = None, run_id: Optional[int] = None, conn: Optional[ConnectionPlus] = None, exp_id=None, name: str = None, specs: Optional[SpecsOrInterDeps] = None, values=None, metadata=None) -> None: """ Create a new DataSet object. The object can either hold a new run or an already existing run. If a run_id is provided, then an old run is looked up, else a new run is created. Args: path_to_db: path to the sqlite file on disk. If not provided, the path will be read from the config. run_id: provide this when loading an existing run, leave it as None when creating a new run conn: connection to the DB; if provided and `path_to_db` is provided as well, then a ValueError is raised (this is to prevent the possibility of providing a connection to a DB file that is different from `path_to_db`) exp_id: the id of the experiment in which to create a new run. Ignored if run_id is provided. name: the name of the dataset. Ignored if run_id is provided. specs: paramspecs belonging to the dataset. Ignored if run_id is provided. values: values to insert into the dataset. Ignored if run_id is provided. metadata: metadata to insert into the dataset. Ignored if run_id is provided. """ if path_to_db is not None and conn is not None: raise ValueError("Both `path_to_db` and `conn` arguments have " "been passed together with non-None values. " "This is not allowed.") self._path_to_db = path_to_db or get_DB_location() self.conn = make_connection_plus_from(conn) if conn is not None else \ connect(self.path_to_db) self._run_id = run_id self._debug = False self.subscribers: Dict[str, _Subscriber] = {} self._interdeps: InterDependencies_ if run_id is not None: if not run_exists(self.conn, run_id): raise ValueError(f"Run with run_id {run_id} does not exist in " f"the database") self._completed = completed(self.conn, self.run_id) run_desc = self._get_run_description_from_db() if run_desc._old_style_deps: # TODO: what if the old run had invalid interdep.s? old_idps: InterDependencies = cast(InterDependencies, run_desc.interdeps) self._interdeps = old_to_new(old_idps) else: new_idps: InterDependencies_ = cast(InterDependencies_, run_desc.interdeps) self._interdeps = new_idps self._metadata = get_metadata_from_run_id(self.conn, run_id) self._started = self.run_timestamp_raw is not None else: # Actually perform all the side effects needed for the creation # of a new dataset. Note that a dataset is created (in the DB) # with no parameters; they are written to disk when the dataset # is marked as started if exp_id is None: if len(get_experiments(self.conn)) > 0: exp_id = get_last_experiment(self.conn) else: raise ValueError("No experiments found." "You can start a new one with:" " new_experiment(name, sample_name)") name = name or "dataset" _, run_id, __ = create_run(self.conn, exp_id, name, generate_guid(), parameters=None, values=values, metadata=metadata) # this is really the UUID (an ever increasing count in the db) self._run_id = run_id self._completed = False self._started = False if isinstance(specs, InterDependencies_): self._interdeps = specs elif specs is not None: self._interdeps = old_to_new(InterDependencies(*specs)) else: self._interdeps = InterDependencies_() self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
def __init__(self, path_to_db: str=None, run_id: Optional[int]=None, conn=None, exp_id=None, name: str=None, specs: SPECS=None, values=None, metadata=None) -> None: """ Create a new DataSet object. The object can either hold a new run or an already existing run. If a run_id is provided, then an old run is looked up, else a new run is created. Args: path_to_db: path to the sqlite file on disk. If not provided, the path will be read from the config. run_id: provide this when loading an existing run, leave it as None when creating a new run conn: connection to the DB exp_id: the id of the experiment in which to create a new run. Ignored if run_id is provided. name: the name of the dataset. Ignored if run_id is provided. specs: paramspecs belonging to the dataset. Ignored if run_id is provided. values: values to insert into the dataset. Ignored if run_id is provided. metadata: metadata to insert into the dataset. Ignored if run_id is provided. """ # TODO: handle fail here by defaulting to # a standard db self.path_to_db = path_to_db or get_DB_location() if conn is None: self.conn = connect(self.path_to_db) else: self.conn = conn self.run_id = run_id self._debug = False self.subscribers: Dict[str, _Subscriber] = {} if run_id: if not run_exists(self.conn, run_id): raise ValueError(f"Run with run_id {run_id} does not exist in " f"the database") self._completed = completed(self.conn, self.run_id) else: if exp_id is None: if len(get_experiments(self.conn)) > 0: exp_id = get_last_experiment(self.conn) else: raise ValueError("No experiments found." "You can start a new one with:" " new_experiment(name, sample_name)") # Actually perform all the side effects needed for # the creation of a new dataset. name = name or "dataset" _, run_id, __ = create_run(self.conn, exp_id, name, generate_guid(), specs, values, metadata) # this is really the UUID (an ever increasing count in the db) self.run_id = run_id self._completed = False