def available( *, only_latest: bool = False, ) -> pd.DataFrame: r"""List all databases that are available to the user. Args: only_latest: keep only latest version Returns: table with name, version and private flag """ databases = [] for repository in config.REPOSITORIES: pattern = f'*/{define.DB}/*/{define.DB}-*.yaml' backend = audbackend.create( repository.backend, repository.host, repository.name, ) for p in backend.glob(pattern): name, _, version, _ = p.split('/')[-4:] databases.append([ name, repository.backend, repository.host, repository.name, version, ]) df = pd.DataFrame.from_records( databases, columns=['name', 'backend', 'host', 'repository', 'version'], ) if only_latest: # Pick latest version for every database, see # https://stackoverflow.com/a/53842408 df = df[df['version'] == df.groupby('name')['version'].transform( lambda x: audeer.sort_versions(x)[-1])] else: # Sort by version df = df.sort_values(by=['version'], key=audeer.sort_versions) df = df.sort_values(by=['name']) return df.set_index('name')
def versions( self, path: str, *, ext: str = None, ) -> typing.List[str]: r"""Versions of a file. Args: path: path to file on backend ext: file extension, if ``None`` uses characters after last dot Returns: list of versions in ascending order """ folder, file = self.split(path) name = audeer.basename_wo_ext(file, ext=ext) vs = self._versions(folder, name) return audeer.sort_versions(vs)
def versions(name: str, ) -> typing.List[str]: r"""Available versions of database. Args: name: name of database Returns: list of versions """ vs = [] for repository in config.REPOSITORIES: backend = audbackend.create( repository.backend, repository.host, repository.name, ) header = backend.join(name, 'db.yaml') vs.extend(backend.versions(header)) return audeer.sort_versions(vs)
def versions( server: str, repository: str, group_id: str, name: str, ) -> typing.List: r"""Versions of an artifact on Artifactory. It lists all folders under the given path and considers all as versions that are conform with :func:`audeer.is_semantic_version`. Args: server: URL of Artifactory server, e.g. ``'https://audeering.jfrog.io/artifactory'`` repository: repository of artifact group_id: group ID of artifact name: name of artifact Returns: versions of artifact on Artifactory """ artifact_url = url( server, repository=repository, group_id=group_id, name=name, ) path = _path(artifact_url) try: versions = [os.path.basename(str(p)) for p in path if p.is_dir] versions = [v for v in versions if audeer.is_semantic_version(v)] except (FileNotFoundError, RuntimeError): versions = [] return audeer.sort_versions(versions)
def test_update_database(): version = '2.1.0' start_version = '2.0.0' db = audb.load_to( DB_ROOT_VERSION[version], DB_NAME, version=start_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) # == Fail with missing dependency file previous_version = start_version dep_file = os.path.join( DB_ROOT_VERSION[version], audb.core.define.DEPENDENCIES_FILE, ) os.remove(dep_file) print(audeer.list_file_names(DB_ROOT_VERSION[version])) error_msg = ( f"You want to depend on '{previous_version}' " f"of {DB_NAME}, " f"but you don't have a '{audb.core.define.DEPENDENCIES_FILE}' " f"file present " f"in {DB_ROOT_VERSION[version]}. " f"Did you forgot to call " f"'audb.load_to({DB_ROOT_VERSION[version]}, {DB_NAME}, " f"version={previous_version}?") with pytest.raises(RuntimeError, match=re.escape(error_msg)): audb.publish( DB_ROOT_VERSION[version], version, pytest.PUBLISH_REPOSITORY, previous_version=previous_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) # Reload data to restore dependency file shutil.rmtree(DB_ROOT_VERSION[version]) db = audb.load_to( DB_ROOT_VERSION[version], DB_NAME, version=start_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) # Remove one file as in version 3.0.0 remove_file = os.path.join('audio', '001.wav') remove_path = os.path.join(DB_ROOT_VERSION[version], remove_file) os.remove(remove_path) db.drop_files(remove_file) db.save(DB_ROOT_VERSION[version]) # == Fail as 2.0.0 is not the latest version previous_version = 'latest' error_msg = (f"You want to depend on '{audb.latest_version(DB_NAME)}' " f"of {DB_NAME}, " f"but the MD5 sum of your " f"'{audb.core.define.DEPENDENCIES_FILE}' file " f"in {DB_ROOT_VERSION[version]} " f"does not match the MD5 sum of the corresponding file " f"for the requested version in the repository. " f"Did you forgot to call " f"'audb.load_to({DB_ROOT_VERSION[version]}, {DB_NAME}, " f"version='{audb.latest_version(DB_NAME)}') " f"or modified the file manually?") with pytest.raises(RuntimeError, match=re.escape(error_msg)): audb.publish( DB_ROOT_VERSION[version], version, pytest.PUBLISH_REPOSITORY, previous_version=previous_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) # == Fail as we require a previous version previous_version = None error_msg = ( f"You did not set a dependency to a previous version, " f"but you have a '{audb.core.define.DEPENDENCIES_FILE}' file present " f"in {DB_ROOT_VERSION[version]}.") with pytest.raises(RuntimeError, match=re.escape(error_msg)): audb.publish( DB_ROOT_VERSION[version], version, pytest.PUBLISH_REPOSITORY, previous_version=previous_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) previous_version = start_version deps = audb.publish( DB_ROOT_VERSION[version], version, pytest.PUBLISH_REPOSITORY, previous_version=previous_version, num_workers=pytest.NUM_WORKERS, verbose=False, ) # Check that depencies include previous and actual version only versions = audeer.sort_versions([deps.version(f) for f in deps.files]) assert versions[-1] == version assert versions[0] == previous_version # Check that there is no difference in the database # if published from scratch or from previous version db1 = audb.load( DB_NAME, version=version, full_path=False, num_workers=pytest.NUM_WORKERS, verbose=False, ) db2 = audb.load( DB_NAME, version='3.0.0', full_path=False, num_workers=pytest.NUM_WORKERS, verbose=False, ) db1.meta['audb'] = {} db2.meta['audb'] = {} assert db1 == db2
def test_sort_versions_errors(versions, error_message): with pytest.raises(ValueError, match=error_message): audeer.sort_versions(versions)
def test_sort_versions(versions, expected_versions): sorted_versions = audeer.sort_versions(versions) assert sorted_versions == expected_versions