def test_new_to_old(some_paramspecbases):
(ps1, ps2, ps3, ps4) = some_paramspecbases
idps_new = InterDependencies_(dependencies={ps1: (ps2, ps3)},
standalones=(ps4, ))
paramspec1 = ParamSpec(name=ps1.name,
paramtype=ps1.type,
label=ps1.label,
unit=ps1.unit,
depends_on=[ps2.name, ps3.name])
paramspec2 = ParamSpec(name=ps2.name,
paramtype=ps2.type,
label=ps2.label,
unit=ps2.unit)
paramspec3 = ParamSpec(name=ps3.name,
paramtype=ps3.type,
label=ps3.label,
unit=ps3.unit)
paramspec4 = ParamSpec(name=ps4.name,
paramtype=ps4.type,
label=ps4.label,
unit=ps4.unit)
idps_old_expected = InterDependencies(paramspec2, paramspec3, paramspec1,
paramspec4)
assert new_to_old(idps_new) == idps_old_expected
#
idps_new = InterDependencies_(inferences={ps1: (ps2, ps3)},
standalones=(ps4, ))
paramspec1 = ParamSpec(name=ps1.name,
paramtype=ps1.type,
label=ps1.label,
unit=ps1.unit,
inferred_from=[ps2.name, ps3.name])
paramspec2 = ParamSpec(name=ps2.name,
paramtype=ps2.type,
label=ps2.label,
unit=ps2.unit)
paramspec3 = ParamSpec(name=ps3.name,
paramtype=ps3.type,
label=ps3.label,
unit=ps3.unit)
paramspec4 = ParamSpec(name=ps4.name,
paramtype=ps4.type,
label=ps4.label,
unit=ps4.unit)
idps_old_expected = InterDependencies(paramspec2, paramspec3, paramspec1,
paramspec4)
assert new_to_old(idps_new) == idps_old_expected
def _extract_single_dataset_into_db(dataset: DataSet,
target_conn: ConnectionPlus,
target_exp_id: int) -> None:
"""
NB: This function should only be called from within
:meth:extract_runs_into_db
Insert the given dataset into the specified database file as the latest
run.
Trying to insert a run already in the DB is a NOOP.
Args:
dataset: A dataset representing the run to be copied
target_conn: connection to the DB. Must be atomically guarded
target_exp_id: The exp_id of the (target DB) experiment in which to
insert the run
"""
if not dataset.completed:
raise ValueError('Dataset not completed. An incomplete dataset '
'can not be copied. The incomplete dataset has '
f'GUID: {dataset.guid} and run_id: {dataset.run_id}')
source_conn = dataset.conn
run_id = get_runid_from_guid(target_conn, dataset.guid)
if run_id != -1:
return
if dataset.parameters is not None:
param_names = dataset.parameters.split(',')
else:
param_names = []
parspecs_dict = {
p.name: p
for p in new_to_old(dataset._interdeps).paramspecs
}
parspecs = [parspecs_dict[p] for p in param_names]
metadata = dataset.metadata
snapshot_raw = dataset.snapshot_raw
_, target_run_id, target_table_name = create_run(target_conn,
target_exp_id,
name=dataset.name,
guid=dataset.guid,
parameters=parspecs,
metadata=metadata)
_populate_results_table(source_conn, target_conn, dataset.table_name,
target_table_name)
mark_run_complete(target_conn, target_run_id)
_rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw,
dataset.completed_timestamp_raw)
if snapshot_raw is not None:
add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def test_convert_v2(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v2 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=2)
v1 = v2_to_v1(v2)
v0 = v2_to_v0(v2)
_assert_dicts_are_related_as_expected(v0, v1, v2)
def test_construct_currect_rundesciber_from_v2(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v2 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=2)
rds1 = RunDescriber._from_dict(v2)
rds2 = from_dict_to_current(v2)
assert rds1._to_dict() == v2
assert rds2._to_dict() == v2
def test_jsonization_as_v0_for_storage(some_interdeps):
"""
Test that a RunDescriber can be json-dumped as version 0
"""
idps_new = some_interdeps[0]
idps_old = new_to_old(idps_new)
new_desc = RunDescriber(idps_new)
old_json = json.dumps({'version': 0,
'interdependencies': idps_old._to_dict()})
assert serial.to_json_as_version(new_desc, 0) == old_json
def test_construct_current_rundescriber_from_v3(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v3 = RunDescriberV3Dict(interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=3,
shapes=None)
rds1 = RunDescriber._from_dict(v3)
rds_upgraded = from_dict_to_current(v3)
assert rds1._to_dict() == v3
assert rds_upgraded._to_dict() == v3
def test_default_dictization_as_v0_for_storage(some_interdeps):
"""
Test that a RunDescriber always gets converted to dict that represents
an old style RunDescriber, even when given new style interdeps
"""
idps_new = some_interdeps[0]
idps_old = new_to_old(idps_new)
new_desc = RunDescriber(idps_new)
old_desc = {'version': 0, 'interdependencies': idps_old._to_dict()}
assert serial.to_dict_for_storage(new_desc) == old_desc
def test_dictization_of_version_2(some_interdeps):
"""
Test conversion to dictionary of a RunDescriber version 2 object
"""
for idps in some_interdeps:
desc = RunDescriber(idps)
idps_old = new_to_old(desc.interdeps)
ser = desc._to_dict()
assert ser['version'] == 2
assert ser['interdependencies'] == idps_old._to_dict()
assert ser['interdependencies_'] == idps._to_dict()
assert len(ser.keys()) == 3
def test_default_jsonization_for_storage(some_interdeps):
"""
Test that a RunDescriber is json-dumped as version 2
"""
idps_new = some_interdeps[0]
idps_old = new_to_old(idps_new)
new_desc = RunDescriber(idps_new)
expected_json = json.dumps({'version': 2,
'interdependencies': idps_old._to_dict(),
'interdependencies_': idps_new._to_dict()})
assert serial.to_json_for_storage(new_desc) == expected_json
def test_construct_currect_rundesciber_from_v1(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v1 = RunDescriberV1Dict(interdependencies=interdeps_._to_dict(), version=1)
rds1 = RunDescriber._from_dict(v1)
rds2 = from_dict_to_current(v1)
expected_v2_dict = RunDescriberV2Dict(
interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=2)
assert rds1._to_dict() == expected_v2_dict
assert rds2._to_dict() == expected_v2_dict
def test_dictization_of_current_version(some_interdeps):
"""
Test conversion to dictionary of a RunDescriber
"""
for idps in some_interdeps:
desc = RunDescriber(idps)
idps_old = new_to_old(desc.interdeps)
ser = desc._to_dict()
assert ser['version'] == 3
assert ser['interdependencies'] == idps_old._to_dict()
assert ser['interdependencies_'] == idps._to_dict()
assert ser['shapes'] is None
assert len(ser.keys()) == 4
def _perform_start_actions(self) -> None:
"""
Perform the actions that must take place once the run has been started
"""
paramspecs = new_to_old(self._interdeps).paramspecs
for spec in paramspecs:
add_parameter(self.conn, self.table_name, spec)
desc_str = serial.to_json_for_storage(self.description)
update_run_description(self.conn, self.run_id, desc_str)
set_run_timestamp(self.conn, self.run_id)
def test_construct_currect_rundesciber_from_fake_v3(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v3 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=3)
v3['foobar'] = {"foo": ["bar"]}
rds1 = RunDescriber._from_dict(v3)
rds2 = from_dict_to_current(v3)
v2 = v3.copy()
v2.pop('foobar')
v2['version'] = 2
assert rds1._to_dict() == v2
assert rds2._to_dict() == v2
def test_construct_current_rundescriber_from_v1(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v1 = RunDescriberV1Dict(interdependencies=interdeps_._to_dict(), version=1)
rds1 = RunDescriber._from_dict(v1)
rds_upgraded = from_dict_to_current(v1)
expected_v3_dict = RunDescriberV3Dict(
interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=3,
shapes=None,
)
assert rds1._to_dict() == expected_v3_dict
assert rds_upgraded._to_dict() == expected_v3_dict
def test_construct_current_rundescriber_from_fake_v4(some_interdeps):
interdeps_ = some_interdeps[0]
interdeps = new_to_old(interdeps_)
v4 = RunDescriberV3Dict(interdependencies=interdeps._to_dict(),
interdependencies_=interdeps_._to_dict(),
version=4,
shapes=None)
v4['foobar'] = {"foo": ["bar"]}
rds1 = RunDescriber._from_dict(v4)
rds_upgraded = from_dict_to_current(v4)
v3 = v4.copy()
v3.pop('foobar')
v3['version'] = 3
assert rds1._to_dict() == v3
assert rds_upgraded._to_dict() == v3
def _perform_start_actions(self) -> None:
"""
Perform the actions that must take place once the run has been started
"""
with contextlib.closing(
conn_from_dbpath_or_conn(conn=None,
path_to_db=self._path_to_db)) as conn:
paramspecs = new_to_old(self.description.interdeps).paramspecs
for spec in paramspecs:
add_parameter(spec,
conn=conn,
run_id=self.run_id,
insert_into_results_table=False)
desc_str = serial.to_json_for_storage(self.description)
update_run_description(conn, self.run_id, desc_str)
self._run_timestamp_raw = time.time()
set_run_timestamp(conn, self.run_id, self._run_timestamp_raw)
pdl_str = links_to_str(self._parent_dataset_links)
update_parent_datasets(conn, self.run_id, pdl_str)
def _get_paramspecs(self) -> SPECS:
old_interdeps = new_to_old(self.description.interdeps)
return list(old_interdeps.paramspecs)
def test_old_to_new_and_back(some_paramspecs):
idps_old = InterDependencies(*some_paramspecs[1].values())
idps_new = old_to_new(idps_old)
assert new_to_old(idps_new) == idps_old
