def upgrade_5_to_6(conn: ConnectionPlus) -> None:
"""
Perform the upgrade from version 5 to version 6.
The upgrade ensures that the runs_description has a top-level entry
called 'version'. Note that version changes of the runs_description will
not be tracked as schema upgrades.
"""
no_of_runs_query = "SELECT max(run_id) FROM runs"
no_of_runs = one(atomic_transaction(conn, no_of_runs_query), 'max(run_id)')
no_of_runs = no_of_runs or 0
# If one run fails, we want the whole upgrade to roll back, hence the
# entire upgrade is one atomic transaction
with atomic(conn) as conn:
pbar = tqdm(range(1, no_of_runs + 1))
pbar.set_description("Upgrading database, version 5 -> 6")
empty_idps_ser = InterDependencies()._to_dict()
for run_id in pbar:
json_str = get_run_description(conn, run_id)
if json_str is None:
new_json = json.dumps({
'version': 0,
'interdependencies': empty_idps_ser
})
else:
ser = json.loads(json_str)
new_ser = {'version': 0} # let 'version' be the first entry
new_ser['interdependencies'] = ser['interdependencies']
new_json = json.dumps(new_ser)
update_run_description(conn, run_id, new_json)
def test_update_runs_description(dataset):
invalid_descs = ['{}', 'description']
for idesc in invalid_descs:
with pytest.raises(ValueError):
mut_queries.update_run_description(dataset.conn, dataset.run_id,
idesc)
desc = serial.to_json_for_storage(RunDescriber((InterDependencies_())))
mut_queries.update_run_description(dataset.conn, dataset.run_id, desc)
def _perform_start_actions(self) -> None:
"""
Perform the actions that must take place once the run has been started
"""
paramspecs = new_to_old(self._interdeps).paramspecs
for spec in paramspecs:
add_parameter(self.conn, self.table_name, spec)
update_run_description(self.conn, self.run_id,
self.description.to_json())
set_run_timestamp(self.conn, self.run_id)
def fix_version_4a_run_description_bug(conn: ConnectionPlus) -> Dict[str, int]:
"""
Fix function to fix a bug where the RunDescriber accidentally wrote itself
to string using the (new) InterDependencies_ object instead of the (old)
InterDependencies object. After the first run, this function should be
idempotent.
Args:
conn: the connection to the database
Returns:
A dict with the fix results ('runs_inspected', 'runs_fixed')
"""
user_version = get_user_version(conn)
if not user_version == 4:
raise RuntimeError('Database of wrong version. Will not apply fix. '
'Expected version 4, found version {user_version}')
no_of_runs_query = "SELECT max(run_id) FROM runs"
no_of_runs = one(atomic_transaction(conn, no_of_runs_query), 'max(run_id)')
no_of_runs = no_of_runs or 0
with atomic(conn) as conn:
pbar = tqdm(range(1, no_of_runs + 1))
pbar.set_description("Fixing database")
# collect some metrics
runs_inspected = 0
runs_fixed = 0
for run_id in pbar:
desc_str = get_run_description(conn, run_id)
desc_ser = json.loads(desc_str)
idps_ser = desc_ser['interdependencies']
if RunDescriber._is_description_old_style(idps_ser):
pass
else:
new_desc = RunDescriber.from_json(desc_str)
update_run_description(conn, run_id, new_desc.to_json())
runs_fixed += 1
runs_inspected += 1
return {'runs_inspected': runs_inspected, 'runs_fixed': runs_fixed}
def _perform_start_actions(self) -> None:
"""
Perform the actions that must take place once the run has been started
"""
paramspecs = new_to_old(self._interdeps).paramspecs
for spec in paramspecs:
add_parameter(self.conn, self.table_name, spec)
desc_str = serial.to_json_for_storage(self.description)
update_run_description(self.conn, self.run_id, desc_str)
set_run_timestamp(self.conn, self.run_id)
pdl_str = links_to_str(self._parent_dataset_links)
update_parent_datasets(self.conn, self.run_id, pdl_str)
def fix_wrong_run_descriptions(conn: ConnectionPlus,
run_ids: Sequence[int]) -> None:
"""
NB: This is a FIX function. Do not use it unless your database has been
diagnosed with the problem that this function fixes.
Overwrite faulty run_descriptions by using information from the layouts and
dependencies tables. If a correct description is found for a run, that
run is left untouched.
Args:
conn: The connection to the database
run_ids: The runs to (potentially) fix
"""
user_version = get_user_version(conn)
if not user_version == 3:
raise RuntimeError('Database of wrong version. Will not apply fix. '
'Expected version 3, found version {user_version}')
log.info('[*] Fixing run descriptions...')
for run_id in run_ids:
trusted_paramspecs = _get_parameters(conn, run_id)
trusted_desc = v0.RunDescriber(
v0.InterDependencies(*trusted_paramspecs))
actual_desc_str = select_one_where(conn, "runs",
"run_description",
"run_id", run_id)
trusted_json = serial.to_json_as_version(trusted_desc, 0)
if actual_desc_str == trusted_json:
log.info(f'[+] Run id: {run_id} had an OK description')
else:
log.info(f'[-] Run id: {run_id} had a broken description. '
f'Description found: {actual_desc_str}')
update_run_description(conn, run_id, trusted_json)
log.info(f' Run id: {run_id} has been updated.')
def _perform_start_actions(self) -> None:
"""
Perform the actions that must take place once the run has been started
"""
with contextlib.closing(
conn_from_dbpath_or_conn(conn=None,
path_to_db=self._path_to_db)) as conn:
paramspecs = new_to_old(self.description.interdeps).paramspecs
for spec in paramspecs:
add_parameter(spec,
conn=conn,
run_id=self.run_id,
insert_into_results_table=False)
desc_str = serial.to_json_for_storage(self.description)
update_run_description(conn, self.run_id, desc_str)
self._run_timestamp_raw = time.time()
set_run_timestamp(conn, self.run_id, self._run_timestamp_raw)
pdl_str = links_to_str(self._parent_dataset_links)
update_parent_datasets(conn, self.run_id, pdl_str)