def _extract_single_dataset_into_db(dataset: DataSet,
target_conn: ConnectionPlus,
target_exp_id: int) -> None:
"""
NB: This function should only be called from within
:meth:extract_runs_into_db
Insert the given dataset into the specified database file as the latest
run.
Trying to insert a run already in the DB is a NOOP.
Args:
dataset: A dataset representing the run to be copied
target_conn: connection to the DB. Must be atomically guarded
target_exp_id: The exp_id of the (target DB) experiment in which to
insert the run
"""
if not dataset.completed:
raise ValueError('Dataset not completed. An incomplete dataset '
'can not be copied. The incomplete dataset has '
f'GUID: {dataset.guid} and run_id: {dataset.run_id}')
source_conn = dataset.conn
run_id = get_runid_from_guid(target_conn, dataset.guid)
if run_id != -1:
return
if dataset.parameters is not None:
param_names = dataset.parameters.split(',')
else:
param_names = []
parspecs_dict = {
p.name: p
for p in new_to_old(dataset._interdeps).paramspecs
}
parspecs = [parspecs_dict[p] for p in param_names]
metadata = dataset.metadata
snapshot_raw = dataset.snapshot_raw
_, target_run_id, target_table_name = create_run(target_conn,
target_exp_id,
name=dataset.name,
guid=dataset.guid,
parameters=parspecs,
metadata=metadata)
_populate_results_table(source_conn, target_conn, dataset.table_name,
target_table_name)
mark_run_complete(target_conn, target_run_id)
_rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw,
dataset.completed_timestamp_raw)
if snapshot_raw is not None:
add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def add_snapshot(self, snapshot: str, overwrite: bool = False) -> None:
"""
Adds a snapshot to this run
Args:
snapshot: the raw JSON dump of the snapshot
overwrite: force overwrite an existing snapshot
"""
if self.snapshot is None or overwrite:
add_meta_data(self.conn, self.run_id, {'snapshot': snapshot})
elif self.snapshot is not None and not overwrite:
log.warning('This dataset already has a snapshot. Use overwrite'
'=True to overwrite that')
def add_metadata(self, tag: str, metadata: Any):
"""
Adds metadata to the DataSet. The metadata is stored under the
provided tag. Note that None is not allowed as a metadata value.
Args:
tag: represents the key in the metadata dictionary
metadata: actual metadata
"""
self._metadata[tag] = metadata
# `add_meta_data` is not atomic by itself, hence using `atomic`
with atomic(self.conn) as conn:
add_meta_data(conn, self.run_id, {tag: metadata})
def set_database(
db_name: str,
db_folder: Optional[str] = None,
) -> None:
""""""
if db_folder is None:
db_folder = nt.config["db_folder"]
if db_name[-2:] != "db":
db_name += ".db"
db_path = os.path.join(db_folder, db_name)
qc.config["core"]["db_location"] = db_path
# check if label columns exist, create if not
db_conn = connect(db_path)
with atomic(db_conn) as conn:
try:
get_metadata(db_conn, '0', nt.config["core"]["labels"][0])
except (RuntimeError, KeyError):
for label in nt.config["core"]["labels"]:
add_meta_data(conn, 0, {label: 0})
def new_database(
db_name: str,
db_folder: Optional[str] = None,
) -> str:
"""
Ceate new database and initialise it
"""
if db_folder is None:
db_folder = nt.config["db_folder"]
if db_name[-2:] != "db":
db_name += ".db"
path = os.path.join(db_folder, db_name)
qc.initialise_or_create_database_at(path)
# add label columns
db_conn = connect(path)
with atomic(db_conn) as conn:
add_meta_data(conn, 0, {"original_guid": 0})
for label in nt.config["core"]["labels"]:
add_meta_data(conn, 0, {label: 0})
return path
