def display_summary():
"""
Displays the current tracked download's progress.
"""
global start_time
if start_time:
elapsed = time.time() - start_time
fasts = ProgressTracker.download_size_current / elapsed
remaining = (ProgressTracker.download_size_total -
ProgressTracker.download_size_current) / fasts
else:
start_time = time.time()
remaining = None
fasts = 1024**2
progress_message = "%d/%d DL: %s/%s (%s, %s)" % (
ProgressTracker.item_count_current,
ProgressTracker.item_count_total,
ProgressTracker.format_filesize(
ProgressTracker.download_size_current),
ProgressTracker.format_filesize(
ProgressTracker.download_size_total),
ProgressTracker.format_percentage(
ProgressTracker.download_size_current,
ProgressTracker.download_size_total),
ProgressTracker.format_seconds(
remaining, ProgressTracker.download_size_total, fasts))
logger.display_message(False, "Progress", progress_message)
logger.display_message(
True, "Progress",
"%s: %s: %s" % (ProgressTracker.current_product,
ProgressTracker.current_subproduct,
ProgressTracker.current_download))
def downloads_from_key(hapi, key):
"""Returns a list of HumbleDownload objetcts from a key string"""
humble_downloads = []
current_order = hapi.get_order(key)
logger.display_message(
False, "Processing",
"{0} is product: {1}".format(key,
current_order.product.human_name))
for current_subproduct in current_order.subproducts or []:
for current_download in current_subproduct.downloads or []:
if not ConfigData.download_platforms.get(
current_download.platform, False):
continue
for current_dl_struct in current_download.download_structs:
if HumbleDownload.download_is_excluded(
current_download.platform,
current_dl_struct.filename):
print("skipping excluded download: " +
current_dl_struct.filename)
continue
hd = HumbleDownload(current_download, current_dl_struct,
current_order, current_subproduct, key)
if hd.is_valid():
humble_downloads.append(hd)
return humble_downloads
def batch_download(hapi, game_keys):
ProgressTracker.reset()
ProgressTracker.item_count_total = len(game_keys)
download_size_total = 0
item_count_total = 0
key_downloads = dict()
# Create initial list of Humble Downloads.
# Platforms that are turned off are filtered here, and the download
# size is computed. Checksums are also calculated for finished
# downloads. This initial query could be paralelized for speed gains
for key in game_keys[::-1]:
ProgressTracker.item_count_current += 1
logger.display_message(
False, "Processing",
"Retrieving order details for order %s (%d/%d)." %
(key, ProgressTracker.item_count_current,
ProgressTracker.item_count_total))
humble_downloads = HumbleDownload.needed_downloads_from_key(
hapi, key)
item_count_total += len(humble_downloads)
download_size_total += sum(dl.humble_file_size
for dl in humble_downloads)
logger.display_message(
False, "Processing", "Added %d downloads for order %s" %
(len(humble_downloads), key))
if len(humble_downloads) > 0:
key_downloads[key] = humble_downloads
ProgressTracker.reset()
ProgressTracker.item_count_total = item_count_total
ProgressTracker.download_size_total = download_size_total
# Now, download the files after updating the url in case they expired
for key in key_downloads:
HumbleDownload.update_download_list_url(hapi,
key_downloads.get(key))
for hd in key_downloads.get(key):
ProgressTracker.assign_download(hd)
ProgressTracker.display_summary()
logger.display_message(False, "Download", hd.status_message)
logger.display_message(
False, "Download",
"Downloading %s." % hd.humble_file_size_human)
hd.download_file()
if hd.humble_file_size is not None:
ProgressTracker.download_size_current += (
hd.humble_file_size)
ProgressTracker.item_count_current += 1
logger.display_message(False, "Processing", "Finished.")
def batch_download(hapi, game_keys):
ProgressTracker.reset()
ProgressTracker.item_count_total = len(game_keys)
download_size_total = 0
item_count_total = 0
key_downloads = dict()
# Create initial list of Humble Downloads.
# Platforms that are turned off are filtered here, and the download
# size is computed. Checksums are also calculated for finished
# downloads. This initial query could be paralelized for speed gains
for key in game_keys:
ProgressTracker.item_count_current += 1
logger.display_message(
False, "Processing",
"Retrieving order details for order %s (%d/%d)." %
(key, ProgressTracker.item_count_current,
ProgressTracker.item_count_total))
humble_downloads = HumbleDownload.needed_downloads_from_key(hapi,
key)
item_count_total += len(humble_downloads)
download_size_total += sum(
dl.humble_file_size for dl in humble_downloads)
logger.display_message(False, "Processing",
"Added %d downloads for order %s"
% (len(humble_downloads), key))
if len(humble_downloads) > 0:
key_downloads[key] = humble_downloads
ProgressTracker.reset()
ProgressTracker.item_count_total = item_count_total
ProgressTracker.download_size_total = download_size_total
# Now, download the files after updating the url in case they expired
for key in key_downloads:
HumbleDownload.update_download_list_url(
hapi, key_downloads.get(key))
for hd in key_downloads.get(key):
ProgressTracker.assign_download(hd)
ProgressTracker.display_summary()
logger.display_message(False, "Download", hd.status_message)
logger.display_message(
False, "Download",
"Downloading %s." % hd.humble_file_size_human)
hd.download_file()
if hd.humble_file_size is not None:
ProgressTracker.download_size_current += (
hd.humble_file_size)
ProgressTracker.item_count_current += 1
logger.display_message(False, "Processing", "Finished.")
def display_summary():
progress_message = "%d/%d DL: %s/%s (%s)" % (
ProgressTracker.item_count_current,
ProgressTracker.item_count_total,
ProgressTracker.format_filesize(
ProgressTracker.download_size_current),
ProgressTracker.format_filesize(
ProgressTracker.download_size_total),
ProgressTracker.format_percentage(
ProgressTracker.download_size_current,
ProgressTracker.download_size_total))
logger.display_message(False, "Progress", progress_message)
logger.display_message(
True, "Progress", "%s: %s: %s" %
(ProgressTracker.current_product,
ProgressTracker.current_subproduct,
ProgressTracker.current_download))
def download_file(self):
""" Downloads a file from the location specified in the provided
DownloadStruct.
"""
self.__create_directory()
if not ConfigData.resume_downloads:
self.remove()
Events.trigger(Events.EVENT_DOWNLOAD_START, self.filename)
try:
if ConfigData.resume_downloads and self.local_file_size > 0:
self.__resume_download()
else:
self.__start_download()
except Exception as ex:
logger.display_message(False, "Download", "Download of %s failed: %s" % (self.filename, ex))
Events.trigger(Events.EVENT_DOWNLOAD_END, self.filename)
def downloads_from_key(hapi, key):
"""Returns a list of HumbleDownload objetcts from a key string"""
humble_downloads = []
current_order = hapi.get_order(key)
logger.display_message(False, "Processing",
"{0} is product: {1}".format(key, current_order.product.human_name))
for current_subproduct in current_order.subproducts or []:
for current_download in current_subproduct.downloads or []:
if not ConfigData.download_platforms.get(
current_download.platform, False):
continue
for current_dl_struct in current_download.download_structs:
if HumbleDownload.download_is_excluded(current_download.platform, current_dl_struct.filename):
print("skipping excluded download: " +
current_dl_struct.filename)
# Check maximun file size
if (ConfigData.max_file_size and
current_dl_struct.file_size is not None and
current_dl_struct.file_size / 1048576 > ConfigData.max_file_size):
continue
# Check file extensions
if ConfigData.file_extensions:
fn_lower = current_dl_struct.filename.lower()
ext_ok = False
for ext in ConfigData.file_extensions:
if fn_lower.endswith(ext.lower()):
ext_ok = True
if not ext_ok:
continue
hd = HumbleDownload(current_download,
current_dl_struct,
current_order,
current_subproduct,
key)
if hd.is_valid():
humble_downloads.append(hd)
return humble_downloads
def print_download_start(filename):
logger.display_message(False, "Download", "%s: " % filename, False)
sys.stdout.flush()
def print_md5_start(filename):
logger.display_message(False, "Checksum", "%s: " % filename, False)
sys.stdout.flush()
def print_download_start(filename):
logger.display_message(False, "Download",
"{0}: {1:7.2f}% ".format(filename, 0), False)
sys.stdout.flush()
def print_md5_start(filename):
logger.display_message(False, "Checksum",
"{0}: {1:7.2f}% ".format(filename, 0), False)
sys.stdout.flush()
# Load the configuration from the YAML file...
try:
Configuration.load_configuration("/etc/hb_downloader.yaml")
except FileNotFoundError:
print("Configuration File not found in /etc")
print("Trying local instead...")
Configuration.load_configuration("hb-downloader-settings.yaml")
Configuration.parse_command_line()
Configuration.dump_configuration()
Configuration.push_configuration()
validation_status, message = Configuration.validate_configuration()
if not validation_status:
logger.display_message(False, "Error", message)
exit(
"Invalid configuration. Please check your command line arguments and "
"hb-downloader-settings.yaml.")
# Initialize the event handlers.
EventHandler.initialize()
hapi = HumbleApi(ConfigData.auth_sess_cookie)
if not hapi.check_login():
exit("Login to humblebundle.com failed."
" Please verify your authentication cookie")
logger.display_message(False, "Processing", "Downloading order list.")
game_keys = hapi.get_gamekeys()
def dump_configuration():
"""
Dumps the current configuration to the log when debug mode is
activated
:return: None
"""
# Shortcut the process if debugging is not turned on.
if not ConfigData.debug:
return
logger.display_message(True, "Config",
"write_md5=%s" % ConfigData.write_md5)
logger.display_message(True, "Config",
"read_md5=%s" % ConfigData.read_md5)
logger.display_message(True, "Config",
"force_md5=%s" % ConfigData.force_md5)
logger.display_message(True, "Config",
"ignore_md5=%s" % ConfigData.ignore_md5)
logger.display_message(True, "Config", "debug=%s" % ConfigData.debug)
logger.display_message(
True, "Config",
"download_location=%s" % ConfigData.download_location)
logger.display_message(True, "Config",
"chunksize=%s" % ConfigData.chunk_size)
logger.display_message(
True, "Config",
"resume_downloads=%s" % ConfigData.resume_downloads)
for platform in list(ConfigData.download_platforms.keys()):
logger.display_message(
True, "Config", "Platform %s=%s" %
(platform, ConfigData.download_platforms[platform]))
def dump_configuration():
"""
Dumps the current configuration to the log when debug mode is
activated
:return: None
"""
# Shortcut the process if debugging is not turned on.
if not ConfigData.debug:
return
logger.display_message(
True, "Config", "write_md5=%s" % ConfigData.write_md5)
logger.display_message(
True, "Config", "read_md5=%s" % ConfigData.read_md5)
logger.display_message(
True, "Config", "force_md5=%s" % ConfigData.force_md5)
logger.display_message(
True, "Config", "ignore_md5=%s" % ConfigData.ignore_md5)
logger.display_message(
True, "Config", "debug=%s" % ConfigData.debug)
logger.display_message(
True, "Config", "download_location=%s" %
ConfigData.download_location)
logger.display_message(
True, "Config", "chunksize=%s" % ConfigData.chunk_size)
logger.display_message(
True, "Config", "resume_downloads=%s" %
ConfigData.resume_downloads)
for platform in list(ConfigData.download_platforms.keys()):
logger.display_message(
True, "Config", "Platform %s=%s" %
(platform, ConfigData.download_platforms[platform]))
def dump_configuration():
"""
Dumps the current configuration to the log when debug mode is
activated
:return: None
"""
# Shortcut the process if debugging is not turned on.
if not ConfigData.debug:
return
logger.display_message(
True, "Config", "write_md5=%s" % ConfigData.write_md5)
logger.display_message(
True, "Config", "read_md5=%s" % ConfigData.read_md5)
logger.display_message(
True, "Config", "force_md5=%s" % ConfigData.force_md5)
logger.display_message(
True, "Config", "ignore_md5=%s" % ConfigData.ignore_md5)
logger.display_message(
True, "Config", "debug=%s" % ConfigData.debug)
logger.display_message(
True, "Config", "download_location=%s" %
ConfigData.download_location)
logger.display_message(
True, "Config", "folderstructure_OrderName=%s" %
ConfigData.folderstructure_OrderName)
logger.display_message(
True, "Config", "chunksize=%s" % ConfigData.chunk_size)
logger.display_message(
True, "Config", "resume_downloads=%s" %
ConfigData.resume_downloads)
logger.display_message(
True, "Config", "get_extra_file_info=%s" %
ConfigData.get_extra_file_info)
for platform in list(ConfigData.download_platforms.keys()):
logger.display_message(
True, "Config", "Platform %s=%s" %
(platform, ConfigData.download_platforms[platform]))
for atype in list(ConfigData.audio_types.keys()):
logger.display_message(
True, "Config", "Audio Types %s=%s" %
(atype, ConfigData.audio_types[atype]))
for etypes in list(ConfigData.ebook_types.keys()):
logger.display_message(
True, "Config", "ebook Types %s=%s" %
(etypes, ConfigData.ebook_types[etypes]))
# Load the configuration from the YAML file...
try:
Configuration.load_configuration("/etc/hb_downloader.yaml")
except FileNotFoundError:
print("Configuration File not found in /etc")
print("Trying local instead...")
Configuration.load_configuration("hb-downloader-settings.yaml")
Configuration.parse_command_line()
Configuration.dump_configuration()
Configuration.push_configuration()
validation_status, message = Configuration.validate_configuration()
if not validation_status:
logger.display_message(False, "Error", message)
exit("Invalid configuration. Please check your command line arguments and"
"hb-downloader-settings.yaml.")
# Initialize the event handlers.
EventHandler.initialize()
hapi = HumbleApi(ConfigData.auth_sess_cookie)
if not hapi.check_login():
exit("Login to humblebundle.com failed."
" Please verify your authentication cookie")
logger.display_message(False, "Processing", "Downloading order list.")
game_keys = hapi.get_gamekeys()
logger.display_message(False, "Processing", "%s orders found." %
