def test_atomic_creation(experiment): """" Test that dataset creation is atomic. Test for https://github.com/QCoDeS/Qcodes/issues/1444 """ def just_throw(*args): raise RuntimeError("This breaks adding metadata") # first we patch add_meta_data to throw an exception # if create_data is not atomic this would create a partial # run in the db. Causing the next create_run to fail with patch('qcodes.dataset.sqlite_base.add_meta_data', new=just_throw): x = ParamSpec('x', 'numeric') t = ParamSpec('t', 'numeric') y = ParamSpec('y', 'numeric', depends_on=['x', 't']) with pytest.raises( RuntimeError, match="Rolling back due to unhandled exception") as e: mut.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, y], metadata={'a': 1}) assert error_caused_by(e, "This breaks adding metadata") # since we are starting from an empty database and the above transaction # should be rolled back there should be no runs in the run table runs = mut.transaction(experiment.conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 0 with shadow_conn(experiment.path_to_db) as new_conn: runs = mut.transaction(new_conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 0 # if the above was not correctly rolled back we # expect the next creation of a run to fail mut.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, y], metadata={'a': 1}) runs = mut.transaction(experiment.conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 1 with shadow_conn(experiment.path_to_db) as new_conn: runs = mut.transaction(new_conn, 'SELECT run_id FROM runs').fetchall() assert len(runs) == 1
def _extract_single_dataset_into_db(dataset: DataSet, target_conn: ConnectionPlus, target_exp_id: int) -> None: """ NB: This function should only be called from within :meth:extract_runs_into_db Insert the given dataset into the specified database file as the latest run. Trying to insert a run already in the DB is a NOOP. Args: dataset: A dataset representing the run to be copied target_conn: connection to the DB. Must be atomically guarded target_exp_id: The exp_id of the (target DB) experiment in which to insert the run """ if not dataset.completed: raise ValueError('Dataset not completed. An incomplete dataset ' 'can not be copied. The incomplete dataset has ' f'GUID: {dataset.guid} and run_id: {dataset.run_id}') source_conn = dataset.conn run_id = get_runid_from_guid(target_conn, dataset.guid) if run_id != -1: return if dataset.parameters is not None: param_names = dataset.parameters.split(',') else: param_names = [] parspecs_dict = { p.name: p for p in new_to_old(dataset._interdeps).paramspecs } parspecs = [parspecs_dict[p] for p in param_names] metadata = dataset.metadata snapshot_raw = dataset.snapshot_raw _, target_run_id, target_table_name = create_run(target_conn, target_exp_id, name=dataset.name, guid=dataset.guid, parameters=parspecs, metadata=metadata) _populate_results_table(source_conn, target_conn, dataset.table_name, target_table_name) mark_run_complete(target_conn, target_run_id) _rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw, dataset.completed_timestamp_raw) if snapshot_raw is not None: add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def _new(self, name, exp_id, specs: SPECS = None, values=None, metadata=None) -> None: """ Actually perform all the side effects needed for the creation of a new dataset. """ _, run_id, __ = create_run(self.conn, exp_id, name, specs, values, metadata) # this is really the UUID (an ever increasing count in the db) self.run_id = run_id self._completed = False
def test_get_dependents(experiment): x = ParamSpec('x', 'numeric') t = ParamSpec('t', 'numeric') y = ParamSpec('y', 'numeric', depends_on=['x', 't']) # Make a dataset (_, run_id, _) = mut.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, y]) deps = mut.get_dependents(experiment.conn, run_id) layout_id = mut.get_layout_id(experiment.conn, 'y', run_id) assert deps == [layout_id] # more parameters, more complicated dependencies x_raw = ParamSpec('x_raw', 'numeric') x_cooked = ParamSpec('x_cooked', 'numeric', inferred_from=['x_raw']) z = ParamSpec('z', 'numeric', depends_on=['x_cooked']) (_, run_id, _) = mut.create_run(experiment.conn, experiment.exp_id, name='testrun', guid=generate_guid(), parameters=[x, t, x_raw, x_cooked, y, z]) deps = mut.get_dependents(experiment.conn, run_id) expected_deps = [ mut.get_layout_id(experiment.conn, 'y', run_id), mut.get_layout_id(experiment.conn, 'z', run_id) ] assert deps == expected_deps
def __init__(self, path_to_db: str = None, run_id: Optional[int] = None, conn: Optional[ConnectionPlus] = None, exp_id=None, name: str = None, specs: Optional[SpecsOrInterDeps] = None, values=None, metadata=None) -> None: """ Create a new DataSet object. The object can either hold a new run or an already existing run. If a run_id is provided, then an old run is looked up, else a new run is created. Args: path_to_db: path to the sqlite file on disk. If not provided, the path will be read from the config. run_id: provide this when loading an existing run, leave it as None when creating a new run conn: connection to the DB; if provided and `path_to_db` is provided as well, then a ValueError is raised (this is to prevent the possibility of providing a connection to a DB file that is different from `path_to_db`) exp_id: the id of the experiment in which to create a new run. Ignored if run_id is provided. name: the name of the dataset. Ignored if run_id is provided. specs: paramspecs belonging to the dataset. Ignored if run_id is provided. values: values to insert into the dataset. Ignored if run_id is provided. metadata: metadata to insert into the dataset. Ignored if run_id is provided. """ if path_to_db is not None and conn is not None: raise ValueError("Both `path_to_db` and `conn` arguments have " "been passed together with non-None values. " "This is not allowed.") self._path_to_db = path_to_db or get_DB_location() self.conn = make_connection_plus_from(conn) if conn is not None else \ connect(self.path_to_db) self._run_id = run_id self._debug = False self.subscribers: Dict[str, _Subscriber] = {} self._interdeps: InterDependencies_ if run_id is not None: if not run_exists(self.conn, run_id): raise ValueError(f"Run with run_id {run_id} does not exist in " f"the database") self._completed = completed(self.conn, self.run_id) run_desc = self._get_run_description_from_db() if run_desc._old_style_deps: # TODO: what if the old run had invalid interdep.s? old_idps: InterDependencies = cast(InterDependencies, run_desc.interdeps) self._interdeps = old_to_new(old_idps) else: new_idps: InterDependencies_ = cast(InterDependencies_, run_desc.interdeps) self._interdeps = new_idps self._metadata = get_metadata_from_run_id(self.conn, run_id) self._started = self.run_timestamp_raw is not None else: # Actually perform all the side effects needed for the creation # of a new dataset. Note that a dataset is created (in the DB) # with no parameters; they are written to disk when the dataset # is marked as started if exp_id is None: if len(get_experiments(self.conn)) > 0: exp_id = get_last_experiment(self.conn) else: raise ValueError("No experiments found." "You can start a new one with:" " new_experiment(name, sample_name)") name = name or "dataset" _, run_id, __ = create_run(self.conn, exp_id, name, generate_guid(), parameters=None, values=values, metadata=metadata) # this is really the UUID (an ever increasing count in the db) self._run_id = run_id self._completed = False self._started = False if isinstance(specs, InterDependencies_): self._interdeps = specs elif specs is not None: self._interdeps = old_to_new(InterDependencies(*specs)) else: self._interdeps = InterDependencies_() self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
def __init__(self, path_to_db: str=None, run_id: Optional[int]=None, conn=None, exp_id=None, name: str=None, specs: SPECS=None, values=None, metadata=None) -> None: """ Create a new DataSet object. The object can either hold a new run or an already existing run. If a run_id is provided, then an old run is looked up, else a new run is created. Args: path_to_db: path to the sqlite file on disk. If not provided, the path will be read from the config. run_id: provide this when loading an existing run, leave it as None when creating a new run conn: connection to the DB exp_id: the id of the experiment in which to create a new run. Ignored if run_id is provided. name: the name of the dataset. Ignored if run_id is provided. specs: paramspecs belonging to the dataset. Ignored if run_id is provided. values: values to insert into the dataset. Ignored if run_id is provided. metadata: metadata to insert into the dataset. Ignored if run_id is provided. """ # TODO: handle fail here by defaulting to # a standard db self.path_to_db = path_to_db or get_DB_location() if conn is None: self.conn = connect(self.path_to_db) else: self.conn = conn self.run_id = run_id self._debug = False self.subscribers: Dict[str, _Subscriber] = {} if run_id: if not run_exists(self.conn, run_id): raise ValueError(f"Run with run_id {run_id} does not exist in " f"the database") self._completed = completed(self.conn, self.run_id) else: if exp_id is None: if len(get_experiments(self.conn)) > 0: exp_id = get_last_experiment(self.conn) else: raise ValueError("No experiments found." "You can start a new one with:" " new_experiment(name, sample_name)") # Actually perform all the side effects needed for # the creation of a new dataset. name = name or "dataset" _, run_id, __ = create_run(self.conn, exp_id, name, generate_guid(), specs, values, metadata) # this is really the UUID (an ever increasing count in the db) self.run_id = run_id self._completed = False