def display_summary(): """ Displays the current tracked download's progress. """ global start_time if start_time: elapsed = time.time() - start_time fasts = ProgressTracker.download_size_current / elapsed remaining = (ProgressTracker.download_size_total - ProgressTracker.download_size_current) / fasts else: start_time = time.time() remaining = None fasts = 1024**2 progress_message = "%d/%d DL: %s/%s (%s, %s)" % ( ProgressTracker.item_count_current, ProgressTracker.item_count_total, ProgressTracker.format_filesize( ProgressTracker.download_size_current), ProgressTracker.format_filesize( ProgressTracker.download_size_total), ProgressTracker.format_percentage( ProgressTracker.download_size_current, ProgressTracker.download_size_total), ProgressTracker.format_seconds( remaining, ProgressTracker.download_size_total, fasts)) logger.display_message(False, "Progress", progress_message) logger.display_message( True, "Progress", "%s: %s: %s" % (ProgressTracker.current_product, ProgressTracker.current_subproduct, ProgressTracker.current_download))
def downloads_from_key(hapi, key): """Returns a list of HumbleDownload objetcts from a key string""" humble_downloads = [] current_order = hapi.get_order(key) logger.display_message( False, "Processing", "{0} is product: {1}".format(key, current_order.product.human_name)) for current_subproduct in current_order.subproducts or []: for current_download in current_subproduct.downloads or []: if not ConfigData.download_platforms.get( current_download.platform, False): continue for current_dl_struct in current_download.download_structs: if HumbleDownload.download_is_excluded( current_download.platform, current_dl_struct.filename): print("skipping excluded download: " + current_dl_struct.filename) continue hd = HumbleDownload(current_download, current_dl_struct, current_order, current_subproduct, key) if hd.is_valid(): humble_downloads.append(hd) return humble_downloads
def batch_download(hapi, game_keys): ProgressTracker.reset() ProgressTracker.item_count_total = len(game_keys) download_size_total = 0 item_count_total = 0 key_downloads = dict() # Create initial list of Humble Downloads. # Platforms that are turned off are filtered here, and the download # size is computed. Checksums are also calculated for finished # downloads. This initial query could be paralelized for speed gains for key in game_keys[::-1]: ProgressTracker.item_count_current += 1 logger.display_message( False, "Processing", "Retrieving order details for order %s (%d/%d)." % (key, ProgressTracker.item_count_current, ProgressTracker.item_count_total)) humble_downloads = HumbleDownload.needed_downloads_from_key( hapi, key) item_count_total += len(humble_downloads) download_size_total += sum(dl.humble_file_size for dl in humble_downloads) logger.display_message( False, "Processing", "Added %d downloads for order %s" % (len(humble_downloads), key)) if len(humble_downloads) > 0: key_downloads[key] = humble_downloads ProgressTracker.reset() ProgressTracker.item_count_total = item_count_total ProgressTracker.download_size_total = download_size_total # Now, download the files after updating the url in case they expired for key in key_downloads: HumbleDownload.update_download_list_url(hapi, key_downloads.get(key)) for hd in key_downloads.get(key): ProgressTracker.assign_download(hd) ProgressTracker.display_summary() logger.display_message(False, "Download", hd.status_message) logger.display_message( False, "Download", "Downloading %s." % hd.humble_file_size_human) hd.download_file() if hd.humble_file_size is not None: ProgressTracker.download_size_current += ( hd.humble_file_size) ProgressTracker.item_count_current += 1 logger.display_message(False, "Processing", "Finished.")
def batch_download(hapi, game_keys): ProgressTracker.reset() ProgressTracker.item_count_total = len(game_keys) download_size_total = 0 item_count_total = 0 key_downloads = dict() # Create initial list of Humble Downloads. # Platforms that are turned off are filtered here, and the download # size is computed. Checksums are also calculated for finished # downloads. This initial query could be paralelized for speed gains for key in game_keys: ProgressTracker.item_count_current += 1 logger.display_message( False, "Processing", "Retrieving order details for order %s (%d/%d)." % (key, ProgressTracker.item_count_current, ProgressTracker.item_count_total)) humble_downloads = HumbleDownload.needed_downloads_from_key(hapi, key) item_count_total += len(humble_downloads) download_size_total += sum( dl.humble_file_size for dl in humble_downloads) logger.display_message(False, "Processing", "Added %d downloads for order %s" % (len(humble_downloads), key)) if len(humble_downloads) > 0: key_downloads[key] = humble_downloads ProgressTracker.reset() ProgressTracker.item_count_total = item_count_total ProgressTracker.download_size_total = download_size_total # Now, download the files after updating the url in case they expired for key in key_downloads: HumbleDownload.update_download_list_url( hapi, key_downloads.get(key)) for hd in key_downloads.get(key): ProgressTracker.assign_download(hd) ProgressTracker.display_summary() logger.display_message(False, "Download", hd.status_message) logger.display_message( False, "Download", "Downloading %s." % hd.humble_file_size_human) hd.download_file() if hd.humble_file_size is not None: ProgressTracker.download_size_current += ( hd.humble_file_size) ProgressTracker.item_count_current += 1 logger.display_message(False, "Processing", "Finished.")
def display_summary(): progress_message = "%d/%d DL: %s/%s (%s)" % ( ProgressTracker.item_count_current, ProgressTracker.item_count_total, ProgressTracker.format_filesize( ProgressTracker.download_size_current), ProgressTracker.format_filesize( ProgressTracker.download_size_total), ProgressTracker.format_percentage( ProgressTracker.download_size_current, ProgressTracker.download_size_total)) logger.display_message(False, "Progress", progress_message) logger.display_message( True, "Progress", "%s: %s: %s" % (ProgressTracker.current_product, ProgressTracker.current_subproduct, ProgressTracker.current_download))
def download_file(self): """ Downloads a file from the location specified in the provided DownloadStruct. """ self.__create_directory() if not ConfigData.resume_downloads: self.remove() Events.trigger(Events.EVENT_DOWNLOAD_START, self.filename) try: if ConfigData.resume_downloads and self.local_file_size > 0: self.__resume_download() else: self.__start_download() except Exception as ex: logger.display_message(False, "Download", "Download of %s failed: %s" % (self.filename, ex)) Events.trigger(Events.EVENT_DOWNLOAD_END, self.filename)
def downloads_from_key(hapi, key): """Returns a list of HumbleDownload objetcts from a key string""" humble_downloads = [] current_order = hapi.get_order(key) logger.display_message(False, "Processing", "{0} is product: {1}".format(key, current_order.product.human_name)) for current_subproduct in current_order.subproducts or []: for current_download in current_subproduct.downloads or []: if not ConfigData.download_platforms.get( current_download.platform, False): continue for current_dl_struct in current_download.download_structs: if HumbleDownload.download_is_excluded(current_download.platform, current_dl_struct.filename): print("skipping excluded download: " + current_dl_struct.filename) # Check maximun file size if (ConfigData.max_file_size and current_dl_struct.file_size is not None and current_dl_struct.file_size / 1048576 > ConfigData.max_file_size): continue # Check file extensions if ConfigData.file_extensions: fn_lower = current_dl_struct.filename.lower() ext_ok = False for ext in ConfigData.file_extensions: if fn_lower.endswith(ext.lower()): ext_ok = True if not ext_ok: continue hd = HumbleDownload(current_download, current_dl_struct, current_order, current_subproduct, key) if hd.is_valid(): humble_downloads.append(hd) return humble_downloads
def print_download_start(filename): logger.display_message(False, "Download", "%s: " % filename, False) sys.stdout.flush()
def print_md5_start(filename): logger.display_message(False, "Checksum", "%s: " % filename, False) sys.stdout.flush()
def print_download_start(filename): logger.display_message(False, "Download", "{0}: {1:7.2f}% ".format(filename, 0), False) sys.stdout.flush()
def print_md5_start(filename): logger.display_message(False, "Checksum", "{0}: {1:7.2f}% ".format(filename, 0), False) sys.stdout.flush()
# Load the configuration from the YAML file... try: Configuration.load_configuration("/etc/hb_downloader.yaml") except FileNotFoundError: print("Configuration File not found in /etc") print("Trying local instead...") Configuration.load_configuration("hb-downloader-settings.yaml") Configuration.parse_command_line() Configuration.dump_configuration() Configuration.push_configuration() validation_status, message = Configuration.validate_configuration() if not validation_status: logger.display_message(False, "Error", message) exit( "Invalid configuration. Please check your command line arguments and " "hb-downloader-settings.yaml.") # Initialize the event handlers. EventHandler.initialize() hapi = HumbleApi(ConfigData.auth_sess_cookie) if not hapi.check_login(): exit("Login to humblebundle.com failed." " Please verify your authentication cookie") logger.display_message(False, "Processing", "Downloading order list.") game_keys = hapi.get_gamekeys()
def dump_configuration(): """ Dumps the current configuration to the log when debug mode is activated :return: None """ # Shortcut the process if debugging is not turned on. if not ConfigData.debug: return logger.display_message(True, "Config", "write_md5=%s" % ConfigData.write_md5) logger.display_message(True, "Config", "read_md5=%s" % ConfigData.read_md5) logger.display_message(True, "Config", "force_md5=%s" % ConfigData.force_md5) logger.display_message(True, "Config", "ignore_md5=%s" % ConfigData.ignore_md5) logger.display_message(True, "Config", "debug=%s" % ConfigData.debug) logger.display_message( True, "Config", "download_location=%s" % ConfigData.download_location) logger.display_message(True, "Config", "chunksize=%s" % ConfigData.chunk_size) logger.display_message( True, "Config", "resume_downloads=%s" % ConfigData.resume_downloads) for platform in list(ConfigData.download_platforms.keys()): logger.display_message( True, "Config", "Platform %s=%s" % (platform, ConfigData.download_platforms[platform]))
def dump_configuration(): """ Dumps the current configuration to the log when debug mode is activated :return: None """ # Shortcut the process if debugging is not turned on. if not ConfigData.debug: return logger.display_message( True, "Config", "write_md5=%s" % ConfigData.write_md5) logger.display_message( True, "Config", "read_md5=%s" % ConfigData.read_md5) logger.display_message( True, "Config", "force_md5=%s" % ConfigData.force_md5) logger.display_message( True, "Config", "ignore_md5=%s" % ConfigData.ignore_md5) logger.display_message( True, "Config", "debug=%s" % ConfigData.debug) logger.display_message( True, "Config", "download_location=%s" % ConfigData.download_location) logger.display_message( True, "Config", "chunksize=%s" % ConfigData.chunk_size) logger.display_message( True, "Config", "resume_downloads=%s" % ConfigData.resume_downloads) for platform in list(ConfigData.download_platforms.keys()): logger.display_message( True, "Config", "Platform %s=%s" % (platform, ConfigData.download_platforms[platform]))
def dump_configuration(): """ Dumps the current configuration to the log when debug mode is activated :return: None """ # Shortcut the process if debugging is not turned on. if not ConfigData.debug: return logger.display_message( True, "Config", "write_md5=%s" % ConfigData.write_md5) logger.display_message( True, "Config", "read_md5=%s" % ConfigData.read_md5) logger.display_message( True, "Config", "force_md5=%s" % ConfigData.force_md5) logger.display_message( True, "Config", "ignore_md5=%s" % ConfigData.ignore_md5) logger.display_message( True, "Config", "debug=%s" % ConfigData.debug) logger.display_message( True, "Config", "download_location=%s" % ConfigData.download_location) logger.display_message( True, "Config", "folderstructure_OrderName=%s" % ConfigData.folderstructure_OrderName) logger.display_message( True, "Config", "chunksize=%s" % ConfigData.chunk_size) logger.display_message( True, "Config", "resume_downloads=%s" % ConfigData.resume_downloads) logger.display_message( True, "Config", "get_extra_file_info=%s" % ConfigData.get_extra_file_info) for platform in list(ConfigData.download_platforms.keys()): logger.display_message( True, "Config", "Platform %s=%s" % (platform, ConfigData.download_platforms[platform])) for atype in list(ConfigData.audio_types.keys()): logger.display_message( True, "Config", "Audio Types %s=%s" % (atype, ConfigData.audio_types[atype])) for etypes in list(ConfigData.ebook_types.keys()): logger.display_message( True, "Config", "ebook Types %s=%s" % (etypes, ConfigData.ebook_types[etypes]))
# Load the configuration from the YAML file... try: Configuration.load_configuration("/etc/hb_downloader.yaml") except FileNotFoundError: print("Configuration File not found in /etc") print("Trying local instead...") Configuration.load_configuration("hb-downloader-settings.yaml") Configuration.parse_command_line() Configuration.dump_configuration() Configuration.push_configuration() validation_status, message = Configuration.validate_configuration() if not validation_status: logger.display_message(False, "Error", message) exit("Invalid configuration. Please check your command line arguments and" "hb-downloader-settings.yaml.") # Initialize the event handlers. EventHandler.initialize() hapi = HumbleApi(ConfigData.auth_sess_cookie) if not hapi.check_login(): exit("Login to humblebundle.com failed." " Please verify your authentication cookie") logger.display_message(False, "Processing", "Downloading order list.") game_keys = hapi.get_gamekeys() logger.display_message(False, "Processing", "%s orders found." %