def _extract_single_dataset_into_db(dataset: DataSet, target_conn: ConnectionPlus, target_exp_id: int) -> None: """ NB: This function should only be called from within :meth:extract_runs_into_db Insert the given dataset into the specified database file as the latest run. Trying to insert a run already in the DB is a NOOP. Args: dataset: A dataset representing the run to be copied target_conn: connection to the DB. Must be atomically guarded target_exp_id: The exp_id of the (target DB) experiment in which to insert the run """ if not dataset.completed: raise ValueError('Dataset not completed. An incomplete dataset ' 'can not be copied. The incomplete dataset has ' f'GUID: {dataset.guid} and run_id: {dataset.run_id}') source_conn = dataset.conn run_id = get_runid_from_guid(target_conn, dataset.guid) if run_id != -1: return if dataset.parameters is not None: param_names = dataset.parameters.split(',') else: param_names = [] parspecs_dict = { p.name: p for p in new_to_old(dataset._interdeps).paramspecs } parspecs = [parspecs_dict[p] for p in param_names] metadata = dataset.metadata snapshot_raw = dataset.snapshot_raw _, target_run_id, target_table_name = create_run(target_conn, target_exp_id, name=dataset.name, guid=dataset.guid, parameters=parspecs, metadata=metadata) _populate_results_table(source_conn, target_conn, dataset.table_name, target_table_name) mark_run_complete(target_conn, target_run_id) _rewrite_timestamps(target_conn, target_run_id, dataset.run_timestamp_raw, dataset.completed_timestamp_raw) if snapshot_raw is not None: add_meta_data(target_conn, target_run_id, {'snapshot': snapshot_raw})
def add_snapshot(self, snapshot: str, overwrite: bool = False) -> None: """ Adds a snapshot to this run Args: snapshot: the raw JSON dump of the snapshot overwrite: force overwrite an existing snapshot """ if self.snapshot is None or overwrite: add_meta_data(self.conn, self.run_id, {'snapshot': snapshot}) elif self.snapshot is not None and not overwrite: log.warning('This dataset already has a snapshot. Use overwrite' '=True to overwrite that')
def add_metadata(self, tag: str, metadata: Any): """ Adds metadata to the DataSet. The metadata is stored under the provided tag. Note that None is not allowed as a metadata value. Args: tag: represents the key in the metadata dictionary metadata: actual metadata """ self._metadata[tag] = metadata # `add_meta_data` is not atomic by itself, hence using `atomic` with atomic(self.conn) as conn: add_meta_data(conn, self.run_id, {tag: metadata})
def set_database( db_name: str, db_folder: Optional[str] = None, ) -> None: """""" if db_folder is None: db_folder = nt.config["db_folder"] if db_name[-2:] != "db": db_name += ".db" db_path = os.path.join(db_folder, db_name) qc.config["core"]["db_location"] = db_path # check if label columns exist, create if not db_conn = connect(db_path) with atomic(db_conn) as conn: try: get_metadata(db_conn, '0', nt.config["core"]["labels"][0]) except (RuntimeError, KeyError): for label in nt.config["core"]["labels"]: add_meta_data(conn, 0, {label: 0})
def new_database( db_name: str, db_folder: Optional[str] = None, ) -> str: """ Ceate new database and initialise it """ if db_folder is None: db_folder = nt.config["db_folder"] if db_name[-2:] != "db": db_name += ".db" path = os.path.join(db_folder, db_name) qc.initialise_or_create_database_at(path) # add label columns db_conn = connect(path) with atomic(db_conn) as conn: add_meta_data(conn, 0, {"original_guid": 0}) for label in nt.config["core"]["labels"]: add_meta_data(conn, 0, {label: 0}) return path