def available(
*,
only_latest: bool = False,
) -> pd.DataFrame:
r"""List all databases that are available to the user.
Args:
only_latest: keep only latest version
Returns:
table with name, version and private flag
"""
databases = []
for repository in config.REPOSITORIES:
pattern = f'*/{define.DB}/*/{define.DB}-*.yaml'
backend = audbackend.create(
repository.backend,
repository.host,
repository.name,
)
for p in backend.glob(pattern):
name, _, version, _ = p.split('/')[-4:]
databases.append([
name,
repository.backend,
repository.host,
repository.name,
version,
])
df = pd.DataFrame.from_records(
databases,
columns=['name', 'backend', 'host', 'repository', 'version'],
)
if only_latest:
# Pick latest version for every database, see
# https://stackoverflow.com/a/53842408
df = df[df['version'] == df.groupby('name')['version'].transform(
lambda x: audeer.sort_versions(x)[-1])]
else:
# Sort by version
df = df.sort_values(by=['version'], key=audeer.sort_versions)
df = df.sort_values(by=['name'])
return df.set_index('name')
def versions(
self,
path: str,
*,
ext: str = None,
) -> typing.List[str]:
r"""Versions of a file.
Args:
path: path to file on backend
ext: file extension, if ``None`` uses characters after last dot
Returns:
list of versions in ascending order
"""
folder, file = self.split(path)
name = audeer.basename_wo_ext(file, ext=ext)
vs = self._versions(folder, name)
return audeer.sort_versions(vs)
def versions(name: str, ) -> typing.List[str]:
r"""Available versions of database.
Args:
name: name of database
Returns:
list of versions
"""
vs = []
for repository in config.REPOSITORIES:
backend = audbackend.create(
repository.backend,
repository.host,
repository.name,
)
header = backend.join(name, 'db.yaml')
vs.extend(backend.versions(header))
return audeer.sort_versions(vs)
def versions(
server: str,
repository: str,
group_id: str,
name: str,
) -> typing.List:
r"""Versions of an artifact on Artifactory.
It lists all folders under the given path
and considers all as versions that are conform with
:func:`audeer.is_semantic_version`.
Args:
server: URL of Artifactory server,
e.g. ``'https://audeering.jfrog.io/artifactory'``
repository: repository of artifact
group_id: group ID of artifact
name: name of artifact
Returns:
versions of artifact on Artifactory
"""
artifact_url = url(
server,
repository=repository,
group_id=group_id,
name=name,
)
path = _path(artifact_url)
try:
versions = [os.path.basename(str(p)) for p in path if p.is_dir]
versions = [v for v in versions if audeer.is_semantic_version(v)]
except (FileNotFoundError, RuntimeError):
versions = []
return audeer.sort_versions(versions)
def test_update_database():
version = '2.1.0'
start_version = '2.0.0'
db = audb.load_to(
DB_ROOT_VERSION[version],
DB_NAME,
version=start_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
# == Fail with missing dependency file
previous_version = start_version
dep_file = os.path.join(
DB_ROOT_VERSION[version],
audb.core.define.DEPENDENCIES_FILE,
)
os.remove(dep_file)
print(audeer.list_file_names(DB_ROOT_VERSION[version]))
error_msg = (
f"You want to depend on '{previous_version}' "
f"of {DB_NAME}, "
f"but you don't have a '{audb.core.define.DEPENDENCIES_FILE}' "
f"file present "
f"in {DB_ROOT_VERSION[version]}. "
f"Did you forgot to call "
f"'audb.load_to({DB_ROOT_VERSION[version]}, {DB_NAME}, "
f"version={previous_version}?")
with pytest.raises(RuntimeError, match=re.escape(error_msg)):
audb.publish(
DB_ROOT_VERSION[version],
version,
pytest.PUBLISH_REPOSITORY,
previous_version=previous_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
# Reload data to restore dependency file
shutil.rmtree(DB_ROOT_VERSION[version])
db = audb.load_to(
DB_ROOT_VERSION[version],
DB_NAME,
version=start_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
# Remove one file as in version 3.0.0
remove_file = os.path.join('audio', '001.wav')
remove_path = os.path.join(DB_ROOT_VERSION[version], remove_file)
os.remove(remove_path)
db.drop_files(remove_file)
db.save(DB_ROOT_VERSION[version])
# == Fail as 2.0.0 is not the latest version
previous_version = 'latest'
error_msg = (f"You want to depend on '{audb.latest_version(DB_NAME)}' "
f"of {DB_NAME}, "
f"but the MD5 sum of your "
f"'{audb.core.define.DEPENDENCIES_FILE}' file "
f"in {DB_ROOT_VERSION[version]} "
f"does not match the MD5 sum of the corresponding file "
f"for the requested version in the repository. "
f"Did you forgot to call "
f"'audb.load_to({DB_ROOT_VERSION[version]}, {DB_NAME}, "
f"version='{audb.latest_version(DB_NAME)}') "
f"or modified the file manually?")
with pytest.raises(RuntimeError, match=re.escape(error_msg)):
audb.publish(
DB_ROOT_VERSION[version],
version,
pytest.PUBLISH_REPOSITORY,
previous_version=previous_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
# == Fail as we require a previous version
previous_version = None
error_msg = (
f"You did not set a dependency to a previous version, "
f"but you have a '{audb.core.define.DEPENDENCIES_FILE}' file present "
f"in {DB_ROOT_VERSION[version]}.")
with pytest.raises(RuntimeError, match=re.escape(error_msg)):
audb.publish(
DB_ROOT_VERSION[version],
version,
pytest.PUBLISH_REPOSITORY,
previous_version=previous_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
previous_version = start_version
deps = audb.publish(
DB_ROOT_VERSION[version],
version,
pytest.PUBLISH_REPOSITORY,
previous_version=previous_version,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
# Check that depencies include previous and actual version only
versions = audeer.sort_versions([deps.version(f) for f in deps.files])
assert versions[-1] == version
assert versions[0] == previous_version
# Check that there is no difference in the database
# if published from scratch or from previous version
db1 = audb.load(
DB_NAME,
version=version,
full_path=False,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
db2 = audb.load(
DB_NAME,
version='3.0.0',
full_path=False,
num_workers=pytest.NUM_WORKERS,
verbose=False,
)
db1.meta['audb'] = {}
db2.meta['audb'] = {}
assert db1 == db2
def test_sort_versions_errors(versions, error_message):
with pytest.raises(ValueError, match=error_message):
audeer.sort_versions(versions)
def test_sort_versions(versions, expected_versions):
sorted_versions = audeer.sort_versions(versions)
assert sorted_versions == expected_versions