def parallel_process(self, interests_articles: List[Generator]):
"""Synopsis: Extract articles of interest from archives in parallel"""
typer.secho('Working with archives (it could take a while): ',
bold=True)
with click_spinner.spinner():
parallel(self._get_art_list,
interests_articles,
threadpool=True,
n_workers=4)
def export_meta(stem: Union[str, Path],
parsed_bin: dict,
saida: Union[str, Path],
ext: str = ".csv") -> None:
file_version, string, blocks = parsed_bin.values()
func = partialler(_export_meta, stem=stem, saida=saida, ext=ext)
func.__module__ = _export_meta.__module__
parallel(func, list(blocks.items()), n_workers=os.cpu_count(), pause=0.5)
def export_level(
stem: Union[str, Path],
parsed_bin: dict,
saida: Union[str, Path],
ext: str = ".fth",
index: pd.DatetimeIndex = None,
dtype: Union[str, np.dtype] = np.float16,
) -> None:
_, _, blocks = parsed_bin.values()
blocks = [((t, i), b) for (t, i), b in blocks.items()
if t in SPECTRAL_BLOCKS]
if not index:
index = [None] * len(blocks)
items = list(zip(blocks, index))
func = partialler(_export_level,
stem=stem,
saida=saida,
ext=ext,
dtype=dtype)
func.__module__ = _export_meta.__module__
parallel(func, items, n_workers=os.cpu_count(), pause=0.5)
def remove_broken_images(image_paths: List[Path]) -> List:
"""removes images that cannot be opened by PIL.
Args:
image_paths (List[Path]): List of `Path`s to images that are to be checked.
Returns:
List: List of `Path`s where all paths are to valid images.
"""
return parallel(verify_image,
image_paths,
progress=progress_bar,
threadpool=True)