def test_install_local_with_delete(self):
fp = NamedTemporaryFile(suffix=".zip", delete=False)
url = get_content_pack_url(TEST_LANGUAGE)
download_file(url, fp=fp, callback=None)
call_command("retrievecontentpack", "local", TEST_LANGUAGE, fp.name)
self.assertTrue(os.path.exists(TEST_CONTENT_DB))
delete_language(TEST_LANGUAGE)
self.assertFalse(os.path.exists(TEST_CONTENT_DB))
def test_install_local_with_delete(self):
fp = NamedTemporaryFile(suffix=".zip", delete=False)
url = get_content_pack_url(TEST_LANGUAGE)
download_file(url, fp=fp, callback=None)
call_command("retrievecontentpack", "local", TEST_LANGUAGE, fp.name)
self.assertTrue(os.path.exists(
TEST_CONTENT_DB
))
delete_language(TEST_LANGUAGE)
self.assertFalse(os.path.exists(
TEST_CONTENT_DB
))
def get_language_pack(lang_code, software_version, callback):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
logging.debug("Downloading zip from %s" % request_url)
path, response = download_file(request_url, callback=callback_percent_proxy(callback))
return path
def download_content_pack(fobj, lang, callback=None):
"""Given a file object where the content pack lang will be stored, return a
zipfile object pointing to the content pack.
"""
url = get_content_pack_url(lang)
logger.info("Downloading content pack from {}".format(url))
def _callback(fraction):
sys.stdout.write("\rProgress: [{}{}] ({}%)".format(
"=" * (int(fraction * 100) / 2),
" " * (50 - int(fraction * 100) / 2),
int(fraction * 100),
))
if callback:
callback(fraction)
download_file(url, fp=fobj, callback=_callback)
zf = zipfile.ZipFile(fobj)
return zf
def get_language_pack(lang_code, software_version, callback):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
logging.debug("Downloading zip from %s" % request_url)
# aron: hack, download_file uses urllib.urlretrieve, which doesn't
# return a status code. So before we make the full request, we
# check first if the said lang pack url exists. If not, error out.
if requests.head(request_url).status_code == 404:
raise requests.exceptions.HTTPError("Language pack %s not found. Please double check that it exists." % lang_code)
path, response = download_file(request_url, callback=callback_percent_proxy(callback))
return path
def get_language_pack(lang_code, software_version, callback):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
logging.debug("Downloading zip from %s" % request_url)
# aron: hack, download_file uses urllib.urlretrieve, which doesn't
# return a status code. So before we make the full request, we
# check first if the said lang pack url exists. If not, error out.
if requests.head(request_url).status_code == 404:
raise requests.exceptions.HTTPError(
"Language pack %s not found. Please double check that it exists." %
lang_code)
path, response = download_file(request_url,
callback=callback_percent_proxy(callback))
return path
def download_video(youtube_id, extension="mp4", callback=None):
"""
Downloads video file to disk
"""
ensure_dir(django_settings.CONTENT_ROOT)
url = get_video_url(youtube_id, extension)
thumb_url = get_thumbnail_url(youtube_id, extension)
filepath = get_video_local_path(youtube_id, extension)
thumb_filepath = get_thumbnail_local_path(youtube_id)
logger.info(
"Downloading {id} (video: {video}, thumbnail: {thumbnail})".format(
id=youtube_id,
video=url,
thumbnail=thumb_url,
))
# Download video
try:
download_file(url,
dst=filepath,
callback=callback_percent_proxy(callback,
end_percent=95),
max_retries=settings.DOWNLOAD_MAX_RETRIES)
except HTTPError as e:
logger.error("HTTP status {status} for URL: {url}".format(
status=e.response.status_code,
url=e.response.url,
))
raise
except (socket.timeout, IOError) as e:
logger.error("Network error for URL: {url}, exception: {exc}".format(
url=url, exc=str(e)))
delete_downloaded_files(youtube_id)
raise
except Exception as e:
logger.exception(e)
delete_downloaded_files(youtube_id)
raise
# Download thumbnail - don't fail if it doesn't succeed, because at this
# stage, we know that the video has been downloaded.
try:
download_file(thumb_url,
dst=thumb_filepath,
callback=callback_percent_proxy(callback,
start_percent=95,
end_percent=100),
max_retries=settings.DOWNLOAD_MAX_RETRIES)
except HTTPError as e:
logger.error("HTTP status {status} for URL: {url}".format(
status=e.response.status_code,
url=e.response.url,
))
except (socket.timeout, IOError) as e:
logger.error("Network error for URL: {url}, exception: {exc}".format(
url=url, exc=str(e)))
def download_video(youtube_id, extension="mp4", callback=None):
"""
Downloads video file to disk
"""
ensure_dir(django_settings.CONTENT_ROOT)
url = get_video_url(youtube_id, extension)
thumb_url = get_thumbnail_url(youtube_id, extension)
filepath = get_video_local_path(youtube_id, extension)
thumb_filepath = get_thumbnail_local_path(youtube_id)
logger.info(
"Downloading {id} (video: {video}, thumbnail: {thumbnail})".format(
id=youtube_id,
video=url,
thumbnail=thumb_url,
)
)
# Download video
try:
download_file(
url,
dst=filepath,
callback=callback_percent_proxy(callback, end_percent=95),
max_retries=settings.DOWNLOAD_MAX_RETRIES
)
except HTTPError as e:
logger.error(
"HTTP status {status} for URL: {url}".format(
status=e.response.status_code,
url=e.response.url,
)
)
raise
except (socket.timeout, IOError) as e:
logger.error("Network error for URL: {url}, exception: {exc}".format(
url=url,
exc=str(e)
))
delete_downloaded_files(youtube_id)
raise
except Exception as e:
logger.exception(e)
delete_downloaded_files(youtube_id)
raise
# Download thumbnail - don't fail if it doesn't succeed, because at this
# stage, we know that the video has been downloaded.
try:
download_file(
thumb_url,
dst=thumb_filepath,
callback=callback_percent_proxy(callback, start_percent=95, end_percent=100),
max_retries=settings.DOWNLOAD_MAX_RETRIES
)
except HTTPError as e:
logger.error(
"HTTP status {status} for URL: {url}".format(
status=e.response.status_code,
url=e.response.url,
)
)
except (socket.timeout, IOError) as e:
logger.error("Network error for URL: {url}, exception: {exc}".format(
url=url,
exc=str(e)
))
