def test_new_to_old(some_paramspecbases): (ps1, ps2, ps3, ps4) = some_paramspecbases idps_new = InterDependencies_(dependencies={ps1: (ps2, ps3)}, standalones=(ps4, )) paramspec1 = ParamSpec(name=ps1.name, paramtype=ps1.type, label=ps1.label, unit=ps1.unit, depends_on=[ps2.name, ps3.name]) paramspec2 = ParamSpec(name=ps2.name, paramtype=ps2.type, label=ps2.label, unit=ps2.unit) paramspec3 = ParamSpec(name=ps3.name, paramtype=ps3.type, label=ps3.label, unit=ps3.unit) paramspec4 = ParamSpec(name=ps4.name, paramtype=ps4.type, label=ps4.label, unit=ps4.unit) idps_old_expected = InterDependencies(paramspec2, paramspec3, paramspec1, paramspec4) assert new_to_old(idps_new) == idps_old_expected # idps_new = InterDependencies_(inferences={ps1: (ps2, ps3)}, standalones=(ps4, )) paramspec1 = ParamSpec(name=ps1.name, paramtype=ps1.type, label=ps1.label, unit=ps1.unit, inferred_from=[ps2.name, ps3.name]) paramspec2 = ParamSpec(name=ps2.name, paramtype=ps2.type, label=ps2.label, unit=ps2.unit) paramspec3 = ParamSpec(name=ps3.name, paramtype=ps3.type, label=ps3.label, unit=ps3.unit) paramspec4 = ParamSpec(name=ps4.name, paramtype=ps4.type, label=ps4.label, unit=ps4.unit) idps_old_expected = InterDependencies(paramspec2, paramspec3, paramspec1, paramspec4) assert new_to_old(idps_new) == idps_old_expected
def _extract_single_dataset_into_db(dataset: DataSet, target_conn: ConnectionPlus, target_exp_id: int) -> None: """ NB: This function should only be called from within :meth:extract_runs_into_db Insert the given dataset into the specified database file as the latest run. Trying to insert a run already in the DB is a NOOP. Args: dataset: A dataset representing the run to be copied target_conn: connection to the DB. Must be atomically guarded target_exp_id: The exp_id of the (target DB) experiment in which to insert the run """ if not dataset.completed: raise ValueError('Dataset not completed. An incomplete dataset ' 'can not be copied. The incomplete dataset has ' f'GUID: {dataset.guid} and run_id: {dataset.run_id}') source_conn = dataset.conn run_id = get_runid_from_guid(target_conn, dataset.guid) if run_id != -1: return if dataset.parameters is not None: param_names = dataset.parameters.split(',') else: param_names = [] parspecs_dict = { p.name: p for p in new_to_old(dataset._interdeps).paramspecs } parspecs = [parspecs_dict[p] for p in param_names] metadata = dataset.metadata snapshot_raw = dataset.snapshot_raw _, target_run_id, target_table_name = create_run(target_conn, target_exp_id, name=dataset.name, guid=dataset.guid, parameters=parspecs, metadata=metadata) _populate_results_table(source_conn, target_conn, dataset.table_name, target_table_name) mark_run_complete(target_conn, target_run_id) _rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw, dataset.completed_timestamp_raw) if snapshot_raw is not None: add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def test_convert_v2(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v2 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=2) v1 = v2_to_v1(v2) v0 = v2_to_v0(v2) _assert_dicts_are_related_as_expected(v0, v1, v2)
def test_construct_currect_rundesciber_from_v2(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v2 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=2) rds1 = RunDescriber._from_dict(v2) rds2 = from_dict_to_current(v2) assert rds1._to_dict() == v2 assert rds2._to_dict() == v2
def test_jsonization_as_v0_for_storage(some_interdeps): """ Test that a RunDescriber can be json-dumped as version 0 """ idps_new = some_interdeps[0] idps_old = new_to_old(idps_new) new_desc = RunDescriber(idps_new) old_json = json.dumps({'version': 0, 'interdependencies': idps_old._to_dict()}) assert serial.to_json_as_version(new_desc, 0) == old_json
def test_construct_current_rundescriber_from_v3(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v3 = RunDescriberV3Dict(interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=3, shapes=None) rds1 = RunDescriber._from_dict(v3) rds_upgraded = from_dict_to_current(v3) assert rds1._to_dict() == v3 assert rds_upgraded._to_dict() == v3
def test_default_dictization_as_v0_for_storage(some_interdeps): """ Test that a RunDescriber always gets converted to dict that represents an old style RunDescriber, even when given new style interdeps """ idps_new = some_interdeps[0] idps_old = new_to_old(idps_new) new_desc = RunDescriber(idps_new) old_desc = {'version': 0, 'interdependencies': idps_old._to_dict()} assert serial.to_dict_for_storage(new_desc) == old_desc
def test_dictization_of_version_2(some_interdeps): """ Test conversion to dictionary of a RunDescriber version 2 object """ for idps in some_interdeps: desc = RunDescriber(idps) idps_old = new_to_old(desc.interdeps) ser = desc._to_dict() assert ser['version'] == 2 assert ser['interdependencies'] == idps_old._to_dict() assert ser['interdependencies_'] == idps._to_dict() assert len(ser.keys()) == 3
def test_default_jsonization_for_storage(some_interdeps): """ Test that a RunDescriber is json-dumped as version 2 """ idps_new = some_interdeps[0] idps_old = new_to_old(idps_new) new_desc = RunDescriber(idps_new) expected_json = json.dumps({'version': 2, 'interdependencies': idps_old._to_dict(), 'interdependencies_': idps_new._to_dict()}) assert serial.to_json_for_storage(new_desc) == expected_json
def test_construct_currect_rundesciber_from_v1(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v1 = RunDescriberV1Dict(interdependencies=interdeps_._to_dict(), version=1) rds1 = RunDescriber._from_dict(v1) rds2 = from_dict_to_current(v1) expected_v2_dict = RunDescriberV2Dict( interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=2) assert rds1._to_dict() == expected_v2_dict assert rds2._to_dict() == expected_v2_dict
def test_dictization_of_current_version(some_interdeps): """ Test conversion to dictionary of a RunDescriber """ for idps in some_interdeps: desc = RunDescriber(idps) idps_old = new_to_old(desc.interdeps) ser = desc._to_dict() assert ser['version'] == 3 assert ser['interdependencies'] == idps_old._to_dict() assert ser['interdependencies_'] == idps._to_dict() assert ser['shapes'] is None assert len(ser.keys()) == 4
def _perform_start_actions(self) -> None: """ Perform the actions that must take place once the run has been started """ paramspecs = new_to_old(self._interdeps).paramspecs for spec in paramspecs: add_parameter(self.conn, self.table_name, spec) desc_str = serial.to_json_for_storage(self.description) update_run_description(self.conn, self.run_id, desc_str) set_run_timestamp(self.conn, self.run_id)
def test_construct_currect_rundesciber_from_fake_v3(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v3 = RunDescriberV2Dict(interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=3) v3['foobar'] = {"foo": ["bar"]} rds1 = RunDescriber._from_dict(v3) rds2 = from_dict_to_current(v3) v2 = v3.copy() v2.pop('foobar') v2['version'] = 2 assert rds1._to_dict() == v2 assert rds2._to_dict() == v2
def test_construct_current_rundescriber_from_v1(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v1 = RunDescriberV1Dict(interdependencies=interdeps_._to_dict(), version=1) rds1 = RunDescriber._from_dict(v1) rds_upgraded = from_dict_to_current(v1) expected_v3_dict = RunDescriberV3Dict( interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=3, shapes=None, ) assert rds1._to_dict() == expected_v3_dict assert rds_upgraded._to_dict() == expected_v3_dict
def test_construct_current_rundescriber_from_fake_v4(some_interdeps): interdeps_ = some_interdeps[0] interdeps = new_to_old(interdeps_) v4 = RunDescriberV3Dict(interdependencies=interdeps._to_dict(), interdependencies_=interdeps_._to_dict(), version=4, shapes=None) v4['foobar'] = {"foo": ["bar"]} rds1 = RunDescriber._from_dict(v4) rds_upgraded = from_dict_to_current(v4) v3 = v4.copy() v3.pop('foobar') v3['version'] = 3 assert rds1._to_dict() == v3 assert rds_upgraded._to_dict() == v3
def _perform_start_actions(self) -> None: """ Perform the actions that must take place once the run has been started """ with contextlib.closing( conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db)) as conn: paramspecs = new_to_old(self.description.interdeps).paramspecs for spec in paramspecs: add_parameter(spec, conn=conn, run_id=self.run_id, insert_into_results_table=False) desc_str = serial.to_json_for_storage(self.description) update_run_description(conn, self.run_id, desc_str) self._run_timestamp_raw = time.time() set_run_timestamp(conn, self.run_id, self._run_timestamp_raw) pdl_str = links_to_str(self._parent_dataset_links) update_parent_datasets(conn, self.run_id, pdl_str)
def _get_paramspecs(self) -> SPECS: old_interdeps = new_to_old(self.description.interdeps) return list(old_interdeps.paramspecs)
def test_old_to_new_and_back(some_paramspecs): idps_old = InterDependencies(*some_paramspecs[1].values()) idps_new = old_to_new(idps_old) assert new_to_old(idps_new) == idps_old