def __init__(
self,
topics,
debug,
name,
video_format,
low_quality,
output_dir,
no_zim,
fname,
languages,
locale_name,
title,
description,
creator,
publisher,
tags,
keep_build_dir,
autoplay,
use_any_optimized_version,
s3_url_with_credentials,
playlist,
subtitles_enough,
subtitles_setting,
tmp_dir,
threads,
):
# video-encoding info
self.video_format = video_format
self.low_quality = low_quality
# zim params
self.fname = fname
self.languages = (
[] if languages is None else [l.strip() for l in languages.split(",")]
)
self.tags = [] if tags is None else [t.strip() for t in tags.split(",")]
self.title = title
self.description = description
self.creator = creator
self.publisher = publisher
self.name = name
# directory setup
self.output_dir = pathlib.Path(output_dir).expanduser().resolve()
if tmp_dir:
pathlib.Path(tmp_dir).mkdir(parents=True, exist_ok=True)
self.build_dir = pathlib.Path(tempfile.mkdtemp(dir=tmp_dir))
# scraper options
self.topics = (
[]
if not topics
else [c.strip().replace(" ", "+") for c in topics.split(",")]
)
self.autoplay = autoplay
self.playlist = playlist
self.subtitles_enough = subtitles_enough
self.subtitles_setting = (
subtitles_setting
if subtitles_setting == ALL
or subtitles_setting == MATCHING
or subtitles_setting == NONE
else self.to_ted_langcodes(
[lang.strip() for lang in subtitles_setting.split(",")]
)
)
self.threads = threads
self.yt_downloader = None
# optimization cache
self.s3_url_with_credentials = s3_url_with_credentials
self.use_any_optimized_version = use_any_optimized_version
self.s3_storage = None
self.video_quality = "low" if self.low_quality else "high"
# debug/developer options
self.no_zim = no_zim
self.keep_build_dir = keep_build_dir
self.debug = debug
# class members
self.videos = []
self.playlist_title = None
self.playlist_description = None
self.source_languages = (
[] if not self.languages else self.to_ted_langcodes(self.languages)
)
self.zim_lang = None
self.already_visited = []
# set and record locale for translations
locale_details = get_language_details(locale_name)
if locale_details["querytype"] != "locale":
locale_name = locale_details["iso-639-1"]
try:
self.locale = setlocale(ROOT_DIR, locale_name)
except locale.Error:
logger.error(
f"No locale for {locale_name}. Use --locale to specify it. "
"defaulting to en_US"
)
self.locale = setlocale(ROOT_DIR, "en")
# locale's language code
self.locale_name = self.to_ted_langcodes(locale_name)
def test_selocale_unsupported(tmp_path):
with pytest.raises(locale.Error):
setlocale(tmp_path, "bam")
def __init__(
self,
archive,
collection,
nb_items_per_page,
show_description,
output_dir,
no_zim,
fname,
debug,
keep_build_dir,
skip_download,
language,
locale_name,
tags,
name=None,
title=None,
description=None,
creator=None,
publisher=None,
favicon=None,
main_logo=None,
secondary_logo=None,
main_color=None,
secondary_color=None,
):
# options & zim params
self.archive = archive
self.collection = handle_user_provided_file(source=collection,
nocopy=True)
self.nb_items_per_page = nb_items_per_page
self.show_author = True
self.show_description = show_description
self.fname = fname
self.language = language
self.tags = [t.strip() for t in tags.split(",")]
self.title = title
self.description = description
self.creator = creator
self.publisher = publisher
self.name = name
self.favicon = favicon
self.main_logo = main_logo
self.secondary_logo = secondary_logo
self.main_color = main_color
self.secondary_color = secondary_color
# process-related
self.output_dir = Path(output_dir).expanduser().resolve()
self.period = datetime.datetime.now().strftime("%Y-%m")
# debug/devel options
self.no_zim = no_zim
self.debug = debug
self.keep_build_dir = keep_build_dir
self.skip_download = skip_download
self.build_dir = self.output_dir.joinpath("build")
# store ZIM-related info
self.zim_info = ZimInfo(
language=language,
tags=tags,
title=title,
description=description,
creator=creator,
publisher=publisher,
name=name,
scraper=SCRAPER,
)
# set and record locale for translations
locale_name = locale_name or get_language_details(
self.language)["iso-639-1"]
try:
self.locale = setlocale(ROOT_DIR, locale_name)
except locale.Error:
logger.error(
f"No locale for {locale_name}. Use --locale to specify it. defaulting to en_US"
)
self.locale = setlocale(ROOT_DIR, "en")
def test_translation(lang, expected):
setlocale(pathlib.Path(__file__).parent, lang)
assert _("Hello World!") == expected
def test_setlocale(tmp_path, code, expected):
assert setlocale(tmp_path, code) == expected
def __init__(
self,
collection_type,
youtube_id,
api_key,
video_format,
low_quality,
nb_videos_per_page,
all_subtitles,
autoplay,
output_dir,
no_zim,
fname,
debug,
keep_build_dir,
skip_download,
max_concurrency,
youtube_store,
language,
locale_name,
tags,
title=None,
description=None,
creator=None,
publisher=None,
name=None,
profile_image=None,
banner_image=None,
main_color=None,
secondary_color=None,
only_test_branding=None,
):
# data-retrieval info
self.collection_type = collection_type
self.youtube_id = youtube_id
self.api_key = api_key
# video-encoding info
self.video_format = video_format
self.low_quality = low_quality
# options & zim params
self.nb_videos_per_page = nb_videos_per_page
self.all_subtitles = all_subtitles
self.autoplay = autoplay
self.fname = fname
self.language = language
self.tags = [t.strip() for t in tags.split(",")]
self.title = title
self.description = description
self.creator = creator
self.publisher = publisher
self.name = name
self.profile_image = profile_image
self.banner_image = banner_image
self.main_color = main_color
self.secondary_color = secondary_color
# process-related
self.output_dir = Path(output_dir).expanduser().resolve()
self.playlists = []
self.uploads_playlist_id = None
self.videos_ids = []
self.main_channel_id = None # use for branding
self.only_test_branding = only_test_branding
# debug/devel options
self.no_zim = no_zim
self.debug = debug
self.keep_build_dir = keep_build_dir
self.skip_download = skip_download
self.max_concurrency = max_concurrency
self.build_dir = self.output_dir.joinpath("build")
# store ZIM-related info
self.zim_info = ZimInfo(
language=language,
tags=tags,
title=title,
description=description,
creator=creator,
publisher=publisher,
name=name,
scraper=SCRAPER,
favicon="favicon.jpg",
)
# update youtube credentials store
youtube_store.update(build_dir=self.build_dir,
api_key=self.api_key,
cache_dir=self.cache_dir)
# set and record locale for translations
locale_name = locale_name or get_language_details(
self.language)["iso-639-1"]
try:
self.locale = setlocale(ROOT_DIR, locale_name)
except locale.Error:
logger.error(
f"No locale for {locale_name}. Use --locale to specify it. defaulting to en_US"
)
self.locale = setlocale(ROOT_DIR, "en")
def run(self):
if not self.inputs:
logger.info(
"Arguments valid, no inputs to process. Exiting with error code 100"
)
return 100
self.find_main_page_metadata()
# make sure Language metadata is ISO-639-3 and setup translations
try:
lang_data = get_language_details(self.language)
self.language = lang_data["iso-639-3"]
setlocale(
pathlib.Path(__file__).parent, lang_data.get("iso-639-1"))
except Exception:
logger.error(
f"Invalid language setting `{self.language}`. Using `eng`.")
self.env = self.init_env()
# init head insert
template = self.env.get_template(HEAD_INSERT_FILE)
self.head_insert = ("<head>" + template.render()).encode("utf-8")
if self.custom_css:
self.css_insert = (
f'\n<link type="text/css" href="{CUSTOM_CSS_URL}" '
'rel="Stylesheet" />\n</head>').encode("utf-8")
else:
self.css_insert = None
self.creator = Creator(
self.full_filename,
main_path="A/index.html",
language=self.language or "eng",
title=self.title,
date=datetime.date.today(),
**self.metadata,
).start()
self.add_replayer()
for filename in pkg_resources.resource_listdir("warc2zim",
"templates"):
if filename == HEAD_INSERT_FILE or filename == SW_JS:
continue
self.creator.add_item(
StaticArticle(self.env, filename, self.main_url))
for record in self.iter_all_warc_records():
self.add_items_for_warc_record(record)
# process revisits, headers only
for url, record in self.revisits.items():
if url not in self.indexed_urls:
logger.debug("Adding revisit {0} -> {1}".format(
url, record.rec_headers["WARC-Refers-To-Target-URI"]))
self.creator.add_item(WARCHeadersItem(record))
self.indexed_urls.add(url)
if self.favicon_url:
self.add_illustration()
logger.debug(f"Found {self.total_records} records in WARCs")
self.creator.finish()
def __init__(
self,
collection_type,
youtube_id,
api_key,
video_format,
low_quality,
nb_videos_per_page,
all_subtitles,
autoplay,
output_dir,
no_zim,
fname,
debug,
tmp_dir,
keep_build_dir,
max_concurrency,
youtube_store,
language,
locale_name,
tags,
dateafter,
use_any_optimized_version,
s3_url_with_credentials,
title=None,
description=None,
creator=None,
publisher=None,
name=None,
profile_image=None,
banner_image=None,
main_color=None,
secondary_color=None,
):
# data-retrieval info
self.collection_type = collection_type
self.youtube_id = youtube_id
self.api_key = api_key
self.dateafter = dateafter
# video-encoding info
self.video_format = video_format
self.low_quality = low_quality
# options & zim params
self.nb_videos_per_page = nb_videos_per_page
self.all_subtitles = all_subtitles
self.autoplay = autoplay
self.fname = fname
self.language = language
self.tags = [t.strip() for t in tags.split(",")]
self.title = title
self.description = description
self.creator = creator
self.publisher = publisher
self.name = name
self.profile_image = profile_image
self.banner_image = banner_image
self.main_color = main_color
self.secondary_color = secondary_color
# directory setup
self.output_dir = Path(output_dir).expanduser().resolve()
if tmp_dir:
tmp_dir = Path(tmp_dir).expanduser().resolve()
tmp_dir.mkdir(parents=True, exist_ok=True)
self.build_dir = Path(tempfile.mkdtemp(dir=tmp_dir))
# process-related
self.playlists = []
self.uploads_playlist_id = None
self.videos_ids = []
self.main_channel_id = None # use for branding
# debug/devel options
self.no_zim = no_zim
self.debug = debug
self.keep_build_dir = keep_build_dir
self.max_concurrency = max_concurrency
# update youtube credentials store
youtube_store.update(build_dir=self.build_dir,
api_key=self.api_key,
cache_dir=self.cache_dir)
# Optimization-cache
self.s3_url_with_credentials = s3_url_with_credentials
self.use_any_optimized_version = use_any_optimized_version
self.video_quality = "low" if self.low_quality else "high"
self.s3_storage = None
# set and record locale for translations
locale_name = locale_name or get_language_details(
self.language)["iso-639-1"]
try:
self.locale = setlocale(ROOT_DIR, locale_name)
except locale.Error:
logger.error(
f"No locale for {locale_name}. Use --locale to specify it. defaulting to en_US"
)
self.locale = setlocale(ROOT_DIR, "en")