def _put_media( media: typing.Set[str], db_root: str, db_name: str, version: str, deps: Dependencies, backend: audbackend.Backend, num_workers: typing.Optional[int], verbose: bool, ): # create a mapping from archives to media and # select archives with new or altered files for upload map_media_to_files = collections.defaultdict(list) for file in deps.media: if not deps.removed(file): map_media_to_files[deps.archive(file)].append(file) if deps.version(file) == version: media.add(deps.archive(file)) lock = threading.Lock() def job(archive): if archive in map_media_to_files: for file in map_media_to_files[archive]: with lock: deps._add_media(db_root, file, version) archive_file = backend.join( db_name, define.DEPEND_TYPE_NAMES[define.DependType.MEDIA], archive, ) backend.put_archive( db_root, map_media_to_files[archive], archive_file, version, ) # upload new and altered archives if it contains at least one file audeer.run_tasks( job, params=[([archive], {}) for archive in media], num_workers=num_workers, progress_bar=verbose, task_description='Put media', )
def _find_media( db: audformat.Database, db_root: str, version: str, deps: Dependencies, archives: typing.Mapping[str, str], verbose: bool, ) -> typing.Set[str]: # release dependencies to removed media # and select according archives for upload media = set() db_media = db.files for file in set(deps.media) - set(db_media): media.add(deps.archive(file)) deps._drop(file) # update version of altered media and insert new ones for file in audeer.progress_bar( db_media, desc='Find media', disable=not verbose, ): path = os.path.join(db_root, file) if file not in deps: checksum = audbackend.md5(path) if file in archives: archive = archives[file] else: archive = audeer.uid(from_string=file.replace('\\', '/')) deps._add_media(db_root, file, version, archive, checksum) elif not deps.removed(file): checksum = audbackend.md5(path) if checksum != deps.checksum(file): archive = deps.archive(file) deps._add_media(db_root, file, version, archive, checksum) return media