def test_integration_station_and_measurement(two_empty_temp_db_connections,
inst):
"""
An integration test where the runs in the source DB file are produced
with the Measurement object and there is a Station as well
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp = Experiment(conn=source_conn)
# Set up measurement scenario
station = Station(inst)
meas = Measurement(exp=source_exp, station=station)
meas.register_parameter(inst.back)
meas.register_parameter(inst.plunger)
meas.register_parameter(inst.cutter, setpoints=(inst.back, inst.plunger))
with meas.run() as datasaver:
for back_v in [1, 2, 3]:
for plung_v in [-3, -2.5, 0]:
datasaver.add_result((inst.back, back_v),
(inst.plunger, plung_v),
(inst.cutter, back_v + plung_v))
extract_runs_into_db(source_path, target_path, 1)
target_ds = DataSet(conn=target_conn, run_id=1)
assert datasaver.dataset.the_same_dataset_as(target_ds)
def test_atomicity(two_empty_temp_db_connections, some_interdeps):
"""
Test the atomicity of the transaction by extracting and inserting two
runs where the second one is not completed. The not completed error must
roll back any changes to the target
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
# The target file must exist for us to be able to see whether it has
# changed
Path(target_path).touch()
source_exp = Experiment(conn=source_conn)
source_ds_1 = DataSet(conn=source_conn, exp_id=source_exp.exp_id)
source_ds_2 = DataSet(conn=source_conn, exp_id=source_exp.exp_id)
for ds in (source_ds_1, source_ds_2):
ds.set_interdependencies(some_interdeps[1])
ds.mark_started()
ds.add_result({name: 2.1 for name in some_interdeps[1].names})
# importantly, source_ds_2 is NOT marked as completed
source_ds_1.mark_completed()
# now check that the target file is untouched
with raise_if_file_changed(target_path):
# although the not completed error is a ValueError, we get the
# RuntimeError from SQLite
with pytest.raises(RuntimeError):
extract_runs_into_db(source_path, target_path, 1, 2)
def test_load_by_X_functions(two_empty_temp_db_connections, some_interdeps):
"""
Test some different loading functions
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp1 = Experiment(conn=source_conn)
source_ds_1_1 = DataSet(conn=source_conn, exp_id=source_exp1.exp_id)
source_exp2 = Experiment(conn=source_conn)
source_ds_2_1 = DataSet(conn=source_conn, exp_id=source_exp2.exp_id)
source_ds_2_2 = DataSet(conn=source_conn,
exp_id=source_exp2.exp_id,
name="customname")
for ds in (source_ds_1_1, source_ds_2_1, source_ds_2_2):
ds.set_interdependencies(some_interdeps[1])
ds.mark_started()
ds.add_result({name: 0.0 for name in some_interdeps[1].names})
ds.mark_completed()
extract_runs_into_db(source_path, target_path, source_ds_2_2.run_id)
test_ds = load_by_guid(source_ds_2_2.guid, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
test_ds = load_by_id(1, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
test_ds = load_by_counter(1, 1, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
def test_runs_from_different_experiments_raises(two_empty_temp_db_connections,
some_interdeps):
"""
Test that inserting runs from multiple experiments raises
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp_1 = Experiment(conn=source_conn)
source_exp_2 = Experiment(conn=source_conn)
# make 5 runs in first experiment
exp_1_run_ids = []
for _ in range(5):
source_dataset = DataSet(conn=source_conn, exp_id=source_exp_1.exp_id)
exp_1_run_ids.append(source_dataset.run_id)
source_dataset.set_interdependencies(some_interdeps[1])
source_dataset.mark_started()
for val in range(10):
source_dataset.add_result(
{name: val
for name in some_interdeps[1].names})
source_dataset.mark_completed()
# make 5 runs in second experiment
exp_2_run_ids = []
for _ in range(5):
source_dataset = DataSet(conn=source_conn, exp_id=source_exp_2.exp_id)
exp_2_run_ids.append(source_dataset.run_id)
source_dataset.set_interdependencies(some_interdeps[1])
source_dataset.mark_started()
for val in range(10):
source_dataset.add_result(
{name: val
for name in some_interdeps[1].names})
source_dataset.mark_completed()
run_ids = exp_1_run_ids + exp_2_run_ids
source_exp_ids = np.unique([1, 2])
matchstring = ('Did not receive runs from a single experiment\\. '
f'Got runs from experiments {source_exp_ids}')
# make the matchstring safe to use as a regexp
matchstring = matchstring.replace('[', '\\[').replace(']', '\\]')
with pytest.raises(ValueError, match=matchstring):
extract_runs_into_db(source_path, target_path, *run_ids)
def test_old_versions_not_touched(two_empty_temp_db_connections,
some_interdeps):
source_conn, target_conn = two_empty_temp_db_connections
target_path = path_to_dbfile(target_conn)
source_path = path_to_dbfile(source_conn)
_, new_v = get_db_version_and_newest_available_version(source_path)
fixturepath = os.sep.join(qcodes.tests.dataset.__file__.split(os.sep)[:-1])
fixturepath = os.path.join(fixturepath,
'fixtures', 'db_files', 'version2',
'some_runs.db')
if not os.path.exists(fixturepath):
pytest.skip("No db-file fixtures found. You can generate test db-files"
" using the scripts in the legacy_DB_generation folder")
# First test that we cannot use an old version as source
with raise_if_file_changed(fixturepath):
with pytest.warns(UserWarning) as warning:
extract_runs_into_db(fixturepath, target_path, 1)
expected_mssg = ('Source DB version is 2, but this '
f'function needs it to be in version {new_v}. '
'Run this function again with '
'upgrade_source_db=True to auto-upgrade '
'the source DB file.')
assert warning[0].message.args[0] == expected_mssg
# Then test that we cannot use an old version as target
# first create a run in the new version source
source_exp = Experiment(conn=source_conn)
source_ds = DataSet(conn=source_conn, exp_id=source_exp.exp_id)
source_ds.set_interdependencies(some_interdeps[1])
source_ds.mark_started()
source_ds.add_results([{name: 0.0
for name in some_interdeps[1].names}])
source_ds.mark_completed()
with raise_if_file_changed(fixturepath):
with pytest.warns(UserWarning) as warning:
extract_runs_into_db(source_path, fixturepath, 1)
expected_mssg = ('Target DB version is 2, but this '
f'function needs it to be in version {new_v}. '
'Run this function again with '
'upgrade_target_db=True to auto-upgrade '
'the target DB file.')
assert warning[0].message.args[0] == expected_mssg
def test_experiments_with_NULL_sample_name(two_empty_temp_db_connections,
some_interdeps):
"""
In older API versions (corresponding to DB version 3),
users could get away with setting the sample name to None
This test checks that such an experiment gets correctly recognised and
is thus not ever re-inserted into the target DB
"""
source_conn, target_conn = two_empty_temp_db_connections
source_exp_1 = Experiment(conn=source_conn, name='null_sample_name')
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
# make 5 runs in experiment
exp_1_run_ids = []
for _ in range(5):
source_dataset = DataSet(conn=source_conn, exp_id=source_exp_1.exp_id)
exp_1_run_ids.append(source_dataset.run_id)
source_dataset.set_interdependencies(some_interdeps[1])
source_dataset.mark_started()
for val in range(10):
source_dataset.add_result(
{name: val
for name in some_interdeps[1].names})
source_dataset.mark_completed()
sql = """
UPDATE experiments
SET sample_name = NULL
WHERE exp_id = 1
"""
source_conn.execute(sql)
source_conn.commit()
assert source_exp_1.sample_name is None
extract_runs_into_db(source_path, target_path, 1, 2, 3, 4, 5)
assert len(get_experiments(target_conn)) == 1
extract_runs_into_db(source_path, target_path, 1, 2, 3, 4, 5)
assert len(get_experiments(target_conn)) == 1
assert len(Experiment(exp_id=1, conn=target_conn)) == 5
def test_load_by_X_functions(two_empty_temp_db_connections,
some_interdeps):
"""
Test some different loading functions
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp1 = Experiment(conn=source_conn)
source_ds_1_1 = DataSet(conn=source_conn, exp_id=source_exp1.exp_id)
source_exp2 = Experiment(conn=source_conn)
source_ds_2_1 = DataSet(conn=source_conn, exp_id=source_exp2.exp_id)
source_ds_2_2 = DataSet(conn=source_conn,
exp_id=source_exp2.exp_id,
name="customname")
for ds in (source_ds_1_1, source_ds_2_1, source_ds_2_2):
ds.set_interdependencies(some_interdeps[1])
ds.mark_started()
ds.add_results([{name: 0.0 for name in some_interdeps[1].names}])
ds.mark_completed()
extract_runs_into_db(source_path, target_path, source_ds_2_2.run_id)
extract_runs_into_db(source_path, target_path, source_ds_2_1.run_id)
extract_runs_into_db(source_path, target_path, source_ds_1_1.run_id)
test_ds = load_by_guid(source_ds_2_2.guid, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
test_ds = load_by_id(1, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
test_ds = load_by_run_spec(captured_run_id=source_ds_2_2.captured_run_id,
conn=target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
assert source_exp2.exp_id == 2
# this is now the first run in the db so run_id is 1
target_run_id = 1
# and the experiment ids will be interchanged.
target_exp_id = 1
test_ds = load_by_counter(target_run_id, target_exp_id, target_conn)
assert source_ds_2_2.the_same_dataset_as(test_ds)
def new_experiment(name: str,
sample_name: Optional[str],
format_string: str = "{}-{}-{}",
conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Create a new experiment (in the database file from config)
Args:
name: the name of the experiment
sample_name: the name of the current sample
format_string: basic format string for table-name
must contain 3 placeholders.
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
the new experiment
"""
sample_name = sample_name or "some_sample"
conn = conn or connect(get_DB_location())
exp_ids = get_matching_exp_ids(conn, name=name, sample_name=sample_name)
if len(exp_ids) >= 1:
log.warning(
f"There is (are) already experiment(s) with the name of {name} "
f"and sample name of {sample_name} in the database.")
experiment = Experiment(name=name,
sample_name=sample_name,
format_string=format_string,
conn=conn)
_set_default_experiment_id(path_to_dbfile(conn), experiment.exp_id)
return experiment
def test_result_table_naming_and_run_id(two_empty_temp_db_connections,
some_interdeps):
"""
Check that a correct result table name is given and that a correct run_id
is assigned
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp1 = Experiment(conn=source_conn)
source_ds_1_1 = DataSet(conn=source_conn, exp_id=source_exp1.exp_id)
source_ds_1_1.set_interdependencies(some_interdeps[1])
source_ds_1_1.mark_started()
source_ds_1_1.add_results([{name: 0.0
for name in some_interdeps[1].names}])
source_ds_1_1.mark_completed()
source_exp2 = Experiment(conn=source_conn)
source_ds_2_1 = DataSet(conn=source_conn, exp_id=source_exp2.exp_id)
source_ds_2_1.set_interdependencies(some_interdeps[1])
source_ds_2_1.mark_started()
source_ds_2_1.add_results([{name: 0.0
for name in some_interdeps[1].names}])
source_ds_2_1.mark_completed()
source_ds_2_2 = DataSet(conn=source_conn,
exp_id=source_exp2.exp_id,
name="customname")
source_ds_2_2.set_interdependencies(some_interdeps[1])
source_ds_2_2.mark_started()
source_ds_2_2.add_results([{name: 0.0
for name in some_interdeps[1].names}])
source_ds_2_2.mark_completed()
extract_runs_into_db(source_path, target_path, source_ds_2_2.run_id)
# The target ds ought to have a runs table "results-1-1"
# and ought to be the same dataset as its "ancestor"
target_ds = DataSet(conn=target_conn, run_id=1)
assert target_ds.name == "customname"
assert target_ds.table_name == "results-1-1"
assert target_ds.the_same_dataset_as(source_ds_2_2)
def test_path_to_dbfile():
with tempfile.TemporaryDirectory() as tempdir:
tempdb = os.path.join(tempdir, 'database.db')
conn = mut_db.connect(tempdb)
try:
assert path_to_dbfile(conn) == tempdb
assert conn.path_to_dbfile == tempdb
finally:
conn.close()
def test_path_to_dbfile(tmp_path):
tempdb = str(tmp_path / 'database.db')
conn = mut_db.connect(tempdb)
try:
assert path_to_dbfile(conn) == tempdb
assert conn.path_to_dbfile == tempdb
finally:
conn.close()
def load_experiment_by_name(
name: str,
sample: Optional[str] = None,
conn: Optional[ConnectionPlus] = None,
load_last_duplicate: bool = False,
) -> Experiment:
"""
Try to load experiment with the specified name.
Nothing stops you from having many experiments with the same name and
sample name. In that case this won't work unless load_last_duplicate
is set to True. Then, the last of duplicated experiments will be loaded.
Args:
name: the name of the experiment
sample: the name of the sample
load_last_duplicate: If True, prevent raising error for having
multiple experiments with the same name and sample name, and
load the last duplicated experiment, instead.
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
the requested experiment
Raises:
ValueError either if the name and sample name are not unique, unless
load_last_duplicate is True, or if no experiment found for the
supplied name and sample.
.
"""
conn = conn or connect(get_DB_location())
if sample is not None:
args_to_find = {"name": name, "sample_name": sample}
else:
args_to_find = {"name": name}
exp_ids = get_matching_exp_ids(conn, **args_to_find)
if len(exp_ids) == 0:
raise ValueError("Experiment not found")
elif len(exp_ids) > 1:
_repr = []
for exp_id in exp_ids:
exp = load_experiment(exp_id, conn=conn)
s = (f"exp_id:{exp.exp_id} ({exp.name}-{exp.sample_name})"
f" started at ({exp.started_at})")
_repr.append(s)
_repr_str = "\n".join(_repr)
if load_last_duplicate:
e = exp
else:
raise ValueError(f"Many experiments matching your request"
f" found:\n{_repr_str}")
else:
e = Experiment(exp_id=exp_ids[0], conn=conn)
_set_default_experiment_id(path_to_dbfile(conn), e.exp_id)
return e
def reset_default_experiment_id(conn: Optional[ConnectionPlus] = None) -> None:
"""
Resets the default experiment id to to the last experiment in the db.
"""
global _default_experiment
if conn is None:
_default_experiment = {}
else:
db_path = path_to_dbfile(conn)
_default_experiment[db_path] = None
def test_column_mismatch(two_empty_temp_db_connections, some_interdeps, inst):
"""
Test insertion of runs with no metadata and no snapshot into a DB already
containing a run that has both
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
target_exp = Experiment(conn=target_conn)
# Set up measurement scenario
station = Station(inst)
meas = Measurement(exp=target_exp, station=station)
meas.register_parameter(inst.back)
meas.register_parameter(inst.plunger)
meas.register_parameter(inst.cutter, setpoints=(inst.back, inst.plunger))
with meas.run() as datasaver:
for back_v in [1, 2, 3]:
for plung_v in [-3, -2.5, 0]:
datasaver.add_result((inst.back, back_v),
(inst.plunger, plung_v),
(inst.cutter, back_v+plung_v))
datasaver.dataset.add_metadata('meta_tag', 'meta_value')
Experiment(conn=source_conn)
source_ds = DataSet(conn=source_conn)
source_ds.set_interdependencies(some_interdeps[1])
source_ds.mark_started()
source_ds.add_results([{name: 2.1
for name in some_interdeps[1].names}])
source_ds.mark_completed()
extract_runs_into_db(source_path, target_path, 1)
# compare
target_copied_ds = DataSet(conn=target_conn, run_id=2)
assert target_copied_ds.the_same_dataset_as(source_ds)
def test_dataset_location(empty_temp_db_connection):
"""
Test that an dataset and experiment points to the correct db file when
a connection is supplied.
"""
exp = new_experiment("test", "test1", conn=empty_temp_db_connection)
ds = DataSet(conn=empty_temp_db_connection)
assert path_to_dbfile(empty_temp_db_connection) == \
empty_temp_db_connection.path_to_dbfile
assert exp.path_to_db == empty_temp_db_connection.path_to_dbfile
assert ds.path_to_db == empty_temp_db_connection.path_to_dbfile
def test_extracting_dataless_run(two_empty_temp_db_connections):
"""
Although contrived, it could happen that a run with no data is extracted
"""
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
Experiment(conn=source_conn)
source_ds = DataSet(conn=source_conn)
source_ds.mark_started()
source_ds.mark_completed()
extract_runs_into_db(source_path, target_path, source_ds.run_id)
loaded_ds = DataSet(conn=target_conn, run_id=1)
assert loaded_ds.the_same_dataset_as(source_ds)
def test_real_dataset_2d(two_empty_temp_db_connections, inst):
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
source_exp = load_or_create_experiment(experiment_name="myexp",
conn=source_conn)
source_dataset, _, _ = do2d(inst.back,
0,
1,
10,
0,
inst.plunger,
0,
0.1,
15,
0,
inst.cutter,
exp=source_exp)
extract_runs_into_db(source_path, target_path, source_dataset.run_id)
target_dataset = load_by_guid(source_dataset.guid, conn=target_conn)
assert source_dataset.the_same_dataset_as(target_dataset)
# explicit regression test for https://github.com/QCoDeS/Qcodes/issues/3953
assert source_dataset.description.shapes == {
"extract_run_inst_cutter": (10, 15)
}
assert source_dataset.description.shapes == target_dataset.description.shapes
source_data = source_dataset.get_parameter_data(
)["extract_run_inst_cutter"]
target_data = target_dataset.get_parameter_data(
)["extract_run_inst_cutter"]
for source_data, target_data in zip(source_data.values(),
target_data.values()):
assert_array_equal(source_data, target_data)
def test_missing_runs_raises(two_empty_temp_db_connections, some_interdeps):
"""
Test that an error is raised if we attempt to extract a run not present in
the source DB
"""
source_conn, target_conn = two_empty_temp_db_connections
source_exp_1 = Experiment(conn=source_conn)
# make 5 runs in first experiment
exp_1_run_ids = []
for _ in range(5):
source_dataset = DataSet(conn=source_conn, exp_id=source_exp_1.exp_id)
exp_1_run_ids.append(source_dataset.run_id)
source_dataset.set_interdependencies(some_interdeps[1])
source_dataset.mark_started()
for val in range(10):
source_dataset.add_result(
{name: val
for name in some_interdeps[1].names})
source_dataset.mark_completed()
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
run_ids = [1, 8, 5, 3, 2, 4, 4, 4, 7, 8]
wrong_ids = [8, 7, 8]
expected_err = ("Error: not all run_ids exist in the source database. "
"The following run(s) is/are not present: "
f"{wrong_ids}")
with pytest.raises(ValueError, match=re.escape(expected_err)):
extract_runs_into_db(source_path, target_path, *run_ids)
def load_experiment(exp_id: int,
conn: Optional[ConnectionPlus] = None) -> Experiment:
"""
Load experiment with the specified id (from database file from config)
Args:
exp_id: experiment id
conn: connection to the database. If not supplied, a new connection
to the DB file specified in the config is made
Returns:
experiment with the specified id
"""
conn = conn_from_dbpath_or_conn(conn=conn, path_to_db=None)
if not isinstance(exp_id, int):
raise ValueError('Experiment ID must be an integer')
experiment = Experiment(exp_id=exp_id, conn=conn)
_set_default_experiment_id(path_to_dbfile(conn), experiment.exp_id)
return experiment
def get_default_experiment_id(conn: ConnectionPlus) -> int:
"""
Returns the latest created/ loaded experiment's exp_id as the default
experiment. If it is not set the maximum exp_id returned as the default.
If no experiment is found in the database, a ValueError is raised.
Args:
conn: Open connection to the db in question.
Returns:
exp_id of the default experiment.
Raises:
ValueError: If no experiment exists in the given db.
"""
db_path = path_to_dbfile(conn)
exp_id = _get_latest_default_experiment_id(db_path)
if exp_id is None:
exp_id = get_last_experiment(conn)
if exp_id is None:
raise ValueError("No experiments found."
" You can create one with:"
" new_experiment(name, sample_name)")
return exp_id
def test_basic_extraction(two_empty_temp_db_connections, some_interdeps):
source_conn, target_conn = two_empty_temp_db_connections
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
type_casters = {
'numeric':
float,
'array': (lambda x: np.array(x)
if hasattr(x, '__iter__') else np.array([x])),
'text':
str
}
source_exp = Experiment(conn=source_conn)
source_dataset = DataSet(conn=source_conn, name="basic_copy_paste_name")
with pytest.raises(RuntimeError) as excinfo:
extract_runs_into_db(source_path, target_path, source_dataset.run_id)
assert error_caused_by(excinfo, ('Dataset not completed. An incomplete '
'dataset can not be copied. The '
'incomplete dataset has GUID: '
f'{source_dataset.guid} and run_id: '
f'{source_dataset.run_id}'))
source_dataset.set_interdependencies(some_interdeps[0])
source_dataset.mark_started()
for value in range(10):
result = {
ps.name: type_casters[ps.type](value)
for ps in some_interdeps[0].paramspecs
}
source_dataset.add_result(result)
source_dataset.add_metadata('goodness', 'fair')
source_dataset.add_metadata('test', True)
source_dataset.mark_completed()
extract_runs_into_db(source_path, target_path, source_dataset.run_id)
target_exp = Experiment(conn=target_conn, exp_id=1)
length1 = len(target_exp)
assert length1 == 1
# trying to insert the same run again should be a NOOP
with raise_if_file_changed(target_path):
extract_runs_into_db(source_path, target_path, source_dataset.run_id)
assert len(target_exp) == length1
target_dataset = DataSet(conn=target_conn, run_id=1)
# Now make the interesting comparisons: are the target objects the same as
# the source objects?
assert source_dataset.the_same_dataset_as(target_dataset)
source_data = source_dataset.get_data(
*source_dataset.parameters.split(','))
target_data = target_dataset.get_data(
*target_dataset.parameters.split(','))
assert source_data == target_data
exp_attrs = [
'name', 'sample_name', 'format_string', 'started_at', 'finished_at'
]
for exp_attr in exp_attrs:
assert getattr(source_exp, exp_attr) == getattr(target_exp, exp_attr)
# trying to insert the same run again should be a NOOP
with raise_if_file_changed(target_path):
extract_runs_into_db(source_path, target_path, source_dataset.run_id)
def test_correct_experiment_routing(two_empty_temp_db_connections,
some_interdeps):
"""
Test that existing experiments are correctly identified AND that multiple
insertions of the same runs don't matter (run insertion is idempotent)
"""
source_conn, target_conn = two_empty_temp_db_connections
source_exp_1 = Experiment(conn=source_conn)
# make 5 runs in first experiment
exp_1_run_ids = []
for _ in range(5):
source_dataset = DataSet(conn=source_conn, exp_id=source_exp_1.exp_id)
exp_1_run_ids.append(source_dataset.run_id)
source_dataset.set_interdependencies(some_interdeps[1])
source_dataset.mark_started()
for val in range(10):
source_dataset.add_result(
{name: val
for name in some_interdeps[1].names})
source_dataset.mark_completed()
# make a new experiment with 1 run
source_exp_2 = Experiment(conn=source_conn)
ds = DataSet(conn=source_conn, exp_id=source_exp_2.exp_id, name="lala")
exp_2_run_ids = [ds.run_id]
ds.set_interdependencies(some_interdeps[1])
ds.mark_started()
for val in range(10):
ds.add_result({name: val for name in some_interdeps[1].names})
ds.mark_completed()
source_path = path_to_dbfile(source_conn)
target_path = path_to_dbfile(target_conn)
# now copy 2 runs
extract_runs_into_db(source_path, target_path, *exp_1_run_ids[:2])
target_exp1 = Experiment(conn=target_conn, exp_id=1)
assert len(target_exp1) == 2
# copy two other runs, one of them already in
extract_runs_into_db(source_path, target_path, *exp_1_run_ids[1:3])
assert len(target_exp1) == 3
# insert run from different experiment
extract_runs_into_db(source_path, target_path, ds.run_id)
assert len(target_exp1) == 3
target_exp2 = Experiment(conn=target_conn, exp_id=2)
assert len(target_exp2) == 1
# finally insert every single run from experiment 1
extract_runs_into_db(source_path, target_path, *exp_1_run_ids)
# check for idempotency once more by inserting all the runs but in another
# order
with raise_if_file_changed(target_path):
extract_runs_into_db(source_path, target_path, *exp_1_run_ids[::-1])
target_exps = get_experiments(target_conn)
assert len(target_exps) == 2
assert len(target_exp1) == 5
assert len(target_exp2) == 1
# check that all the datasets match up
for run_id in exp_1_run_ids + exp_2_run_ids:
source_ds = DataSet(conn=source_conn, run_id=run_id)
target_ds = load_by_guid(guid=source_ds.guid, conn=target_conn)
assert source_ds.the_same_dataset_as(target_ds)
source_data = source_ds.get_data(*source_ds.parameters.split(','))
target_data = target_ds.get_data(*target_ds.parameters.split(','))
assert source_data == target_data
