def analyze_lib(lib_dir,
cover_pattern,
*,
ignore_existing=False,
full_scan=False,
all_formats=False):
""" Recursively analyze library, and return a list of work. """
work = []
stats = collections.OrderedDict(
((k, 0) for k in ("files", "albums", "missing covers", "errors")))
with tqdm.tqdm(desc="Analyzing library",
unit="dir",
postfix=stats) as progress, \
tqdm_logging.redirect_logging(progress):
for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
new_work = analyze_dir(stats,
rootpath,
rel_filepaths,
cover_pattern,
ignore_existing=ignore_existing,
full_scan=full_scan,
all_formats=all_formats)
progress.set_postfix(stats, refresh=False)
progress.update(1)
work.extend(new_work)
return work
def analyze_lib(lib_dir, cover_filename, *, ignore_existing=False):
""" Recursively analyze library, and return a dict of path -> (artist, album). """
work = {}
stats = collections.OrderedDict(((k, 0) for k in("files", "albums", "missing covers", "errors")))
with tqdm.tqdm(desc="Analyzing library",
unit="dir",
postfix=stats) as progress, \
tqdm_logging.redirect_logging(progress):
for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
metadata = analyze_dir(stats,
rootpath,
rel_filepaths,
cover_filename,
ignore_existing=ignore_existing)
progress.set_postfix(stats, refresh=False)
progress.update(1)
if all(metadata[:-1]):
work[rootpath] = metadata[:-1]
return work
def get_covers(work, args):
""" Get missing covers. """
with contextlib.ExitStack() as cm:
if args.cover_pattern == EMBEDDED_ALBUM_ART_SYMBOL:
tmp_prefix = "%s_" % (os.path.splitext(
os.path.basename(inspect.getfile(inspect.currentframe())))[0])
tmp_dir = cm.enter_context(
tempfile.TemporaryDirectory(prefix=tmp_prefix))
# setup progress report
stats = collections.OrderedDict(
((k, 0) for k in ("ok", "errors", "no result found")))
progress = cm.enter_context(
tqdm.tqdm(total=len(work),
miniters=1,
desc="Searching covers",
unit="cover",
postfix=stats))
cm.enter_context(tqdm_logging.redirect_logging(progress))
def post_download(future):
work = futures[future]
try:
status = future.result()
except Exception as exception:
stats["errors"] += 1
logging.getLogger("sacad_r").error(
"Error occured while searching %s: "
"%s %s" %
(work, exception.__class__.__qualname__, exception))
else:
if status:
if work.cover_filepath == EMBEDDED_ALBUM_ART_SYMBOL:
try:
embed_album_art(work.tmp_cover_filepath,
work.audio_filepaths)
except Exception as exception:
stats["errors"] += 1
logging.getLogger("sacad_r").error(
"Error occured while embedding %s: "
"%s %s" %
(work, exception.__class__.__qualname__,
exception))
else:
stats["ok"] += 1
finally:
os.remove(work.tmp_cover_filepath)
else:
stats["ok"] += 1
else:
stats["no result found"] += 1
logging.getLogger("sacad_r").warning("Unable to find %s" %
(work))
progress.set_postfix(stats, refresh=False)
progress.update(1)
# post work
i = 0
# default event loop on Windows has a 512 fd limit, see https://docs.python.org/3/library/asyncio-eventloops.html#windows
# also on Linux default max open fd limit is 1024 (ulimit -n)
# so work in smaller chunks to avoid hitting fd limit
# this also updates the progress faster (instead of working on all searches, work on finishing the chunk before
# getting to the next one)
work_chunk_length = 4 if sys.platform.startswith("win") else 12
for work_chunk in ichunk(work, work_chunk_length):
futures = {}
for i, cur_work in enumerate(work_chunk, i):
if cur_work.cover_filepath == EMBEDDED_ALBUM_ART_SYMBOL:
cover_filepath = os.path.join(
tmp_dir, "%00u.%s" % (i, args.format.name.lower()))
cur_work.tmp_cover_filepath = cover_filepath
else:
cover_filepath = cur_work.cover_filepath
os.makedirs(os.path.dirname(cover_filepath), exist_ok=True)
coroutine = sacad.search_and_download(
cur_work.metadata.album,
cur_work.metadata.artist,
args.format,
args.size,
cover_filepath,
size_tolerance_prct=args.size_tolerance_prct,
amazon_tlds=args.amazon_tlds,
no_lq_sources=args.no_lq_sources,
preserve_format=args.preserve_format)
future = asyncio.ensure_future(coroutine)
futures[future] = cur_work
for future in futures:
future.add_done_callback(post_download)
# wait for end of work
root_future = asyncio.gather(*futures.keys())
asyncio.get_event_loop().run_until_complete(root_future)
def get_covers(work, args):
""" Get missing covers. """
with contextlib.ExitStack() as cm:
if args.filename == EMBEDDED_ALBUM_ART_SYMBOL:
tmp_prefix = "%s_" % (os.path.splitext(os.path.basename(inspect.getfile(inspect.currentframe())))[0])
tmp_dir = cm.enter_context(tempfile.TemporaryDirectory(prefix=tmp_prefix))
# setup progress report
stats = collections.OrderedDict(((k, 0) for k in("ok", "errors", "no result found")))
progress = cm.enter_context(tqdm.tqdm(total=len(work),
miniters=1,
desc="Searching covers",
unit="cover",
postfix=stats))
cm.enter_context(tqdm_logging.redirect_logging(progress))
def update_progress(future):
path, cover_filepath, artist, album = futures[future]
try:
status = future.result()
except Exception as exception:
stats["errors"] += 1
logging.getLogger("sacad_r").error("Error occured while searching cover for "
"'%s' by '%s' from '%s': %s %s" % (album,
artist,
path,
exception.__class__.__qualname__,
exception))
else:
if status:
if args.filename == EMBEDDED_ALBUM_ART_SYMBOL:
try:
embed_album_art(cover_filepath, path)
except Exception as exception:
stats["errors"] += 1
logging.getLogger("sacad_r").error("Error occured while embedding cover for "
"'%s' by '%s' from '%s': %s %s" % (album,
artist,
path,
exception.__class__.__qualname__,
exception))
else:
stats["ok"] += 1
finally:
os.remove(cover_filepath)
else:
stats["ok"] += 1
else:
stats["no result found"] += 1
logging.getLogger("sacad_r").warning("Unable to find cover for '%s' by '%s' from '%s'" % (album, artist, path))
progress.set_postfix(stats, refresh=False)
progress.update(1)
# post work
async_loop = asyncio.get_event_loop()
i = 0
# default event loop on Windows has a 512 fd limit, see https://docs.python.org/3/library/asyncio-eventloops.html#windows
# also on Linux default max open fd limit is 1024 (ulimit -n)
# so work in smaller chunks to avoid hitting fd limit
# this also updates the progress faster (instead of working on all searches, work on finishing the chunk before
# getting to the next one)
work_chunk_length = 16
for work_chunk in ichunk(work.items(), work_chunk_length):
futures = {}
for i, (path, (artist, album)) in enumerate(work_chunk, i):
if args.filename == EMBEDDED_ALBUM_ART_SYMBOL:
cover_filepath = os.path.join(tmp_dir, "%00u.%s" % (i, args.format.name.lower()))
else:
cover_filepath = os.path.join(path, args.filename)
coroutine = sacad.search_and_download(album,
artist,
args.format,
args.size,
cover_filepath,
size_tolerance_prct=args.size_tolerance_prct,
amazon_tlds=args.amazon_tlds,
no_lq_sources=args.no_lq_sources,
async_loop=async_loop)
future = asyncio.ensure_future(coroutine, loop=async_loop)
futures[future] = (path, cover_filepath, artist, album)
for future in futures:
future.add_done_callback(update_progress)
# wait for end of work
root_future = asyncio.gather(*futures.keys(), loop=async_loop)
async_loop.run_until_complete(root_future)