def _put_media(
media: typing.Set[str],
db_root: str,
db_name: str,
version: str,
deps: Dependencies,
backend: audbackend.Backend,
num_workers: typing.Optional[int],
verbose: bool,
):
# create a mapping from archives to media and
# select archives with new or altered files for upload
map_media_to_files = collections.defaultdict(list)
for file in deps.media:
if not deps.removed(file):
map_media_to_files[deps.archive(file)].append(file)
if deps.version(file) == version:
media.add(deps.archive(file))
lock = threading.Lock()
def job(archive):
if archive in map_media_to_files:
for file in map_media_to_files[archive]:
with lock:
deps._add_media(db_root, file, version)
archive_file = backend.join(
db_name,
define.DEPEND_TYPE_NAMES[define.DependType.MEDIA],
archive,
)
backend.put_archive(
db_root,
map_media_to_files[archive],
archive_file,
version,
)
# upload new and altered archives if it contains at least one file
audeer.run_tasks(
job,
params=[([archive], {}) for archive in media],
num_workers=num_workers,
progress_bar=verbose,
task_description='Put media',
)
def _find_media(
db: audformat.Database,
db_root: str,
version: str,
deps: Dependencies,
archives: typing.Mapping[str, str],
verbose: bool,
) -> typing.Set[str]:
# release dependencies to removed media
# and select according archives for upload
media = set()
db_media = db.files
for file in set(deps.media) - set(db_media):
media.add(deps.archive(file))
deps._drop(file)
# update version of altered media and insert new ones
for file in audeer.progress_bar(
db_media,
desc='Find media',
disable=not verbose,
):
path = os.path.join(db_root, file)
if file not in deps:
checksum = audbackend.md5(path)
if file in archives:
archive = archives[file]
else:
archive = audeer.uid(from_string=file.replace('\\', '/'))
deps._add_media(db_root, file, version, archive, checksum)
elif not deps.removed(file):
checksum = audbackend.md5(path)
if checksum != deps.checksum(file):
archive = deps.archive(file)
deps._add_media(db_root, file, version, archive, checksum)
return media