class DownloadManager(DownloadCore, ThreadManager): #herencia multiple """ DownloadCore: .Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas. .Atributos heredados: self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads .Metodos heredados: - ThreadManager: .Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas. .Atributos heredados: self.thread_downloads (DICT) .Metodos heredados: get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status """ def __init__(self): """""" DownloadCore.__init__(self) #inicializar download_core.py ThreadManager.__init__(self) #inicializar thread_manager.py self.global_slots = Slots() #slots.py def start_all(self, id_order_list): """""" self.queue_downloads.update(self.stopped_downloads) self.stopped_downloads.clear() self.reorder_queue(id_order_list) for download_item in self.queue_downloads.values(): download_item.reset_fail_count() self.download_starter(download_item) def stop_all(self, filter_host_list=None): """""" if filter_host_list is None: filter_host_list = [] for id_item, download_item in self.active_downloads.iteritems(): if download_item.host not in filter_host_list: self.stop_thread(self.get_thread(id_item)) for id_item, download_item in self.queue_downloads.items(): if download_item.host not in filter_host_list: self.stopped_downloads[id_item] = download_item del self.queue_downloads[id_item] def start_download(self, id_item): #iniciar descarga. Boton play. """""" try: download_item = self.stopped_downloads.pop(id_item) except KeyError: return False else: self.downloader_init([download_item, ], download_item.path) return True def stop_download(self, id_item): #detener descarga. Boton stop. """""" try: download_item = self.active_downloads[id_item] except KeyError: try: download_item = self.queue_downloads.pop(id_item) self.stopped_downloads[id_item] = download_item except KeyError: return False else: return True else: self.stop_thread(self.get_thread(id_item)) return True def delete_download(self, id_items_list): """""" for id_item in id_items_list: is_active = False try: download_item = self.active_downloads.pop(id_item) is_active = True except KeyError: try: download_item = self.stopped_downloads.pop(id_item) except KeyError: try: download_item = self.queue_downloads.pop(id_item) except KeyError as err: #will crash after this and that's ok. logger.exception(err) th = None if is_active: th = self.get_thread(id_item) self.stop_thread(th) self.delete_thread(id_item) self.global_slots.remove_slot() self.next_download() threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start() def remove_file(self, download_item, th): """""" if th is not None: th.join() try: os.remove(os.path.join(download_item.path, download_item.name)) except Exception as err: logger.warning(err) def get_items_update(self): """ Roba ciclos. """ result_list = [] for id_item, download_item in self.active_downloads.items(): item_data = self.get_thread_status(id_item) #Metodo de threadManager heredado if item_data is not None: download_item.update(*item_data) limit_exceeded = self.is_limit_exceeded(download_item.id) else: download_item.status = cons.STATUS_ERROR limit_exceeded = False status = download_item.status result_list.append(download_item) if status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR): if status == cons.STATUS_FINISHED: self.complete_downloads[id_item] = download_item elif status == cons.STATUS_ERROR: logger.warning("status error: {0}".format(download_item.host)) download_item.fail_count += 1 if download_item.fail_count > conf.get_retries_limit(): download_item.status = cons.STATUS_STOPPED self.stopped_downloads[id_item] = download_item else: download_item.status = cons.STATUS_QUEUE self.queue_downloads[id_item] = download_item else: #stopped self.stopped_downloads[id_item] = download_item self.delete_thread(id_item) #metodo de threadmanager heredado del self.active_downloads[id_item] self.global_slots.remove_slot() #remove the slot, so the next download can start. self.next_download() if status == cons.STATUS_FINISHED: events.trigger_download_complete(download_item) if not self.active_downloads and status != cons.STATUS_STOPPED: events.trigger_all_downloads_complete() elif limit_exceeded: #cons.STATUS_ERROR events.trigger_limit_exceeded() return result_list def downloader_init(self, item_list, save_to_path): #start_thread. Crea el thread para el link dado. """ Crea los threads para la descarga de cada archivo. """ #if not self.active_downloads: #after this method completes, there will be one active download at least. #events.trigger_downloading_process_pre_start() for download_item in item_list: #in self.pending_downloads: download_item.set_path(save_to_path) download_item.reset_fail_count() self.queue_downloads[download_item.id] = download_item self.download_starter(download_item) def next_download(self): """ Init next download on queue. This is called when an active download is stopped or has finished. """ for download_item in self.queue_downloads.values(): #iniciar la proxima descarga en la cola, o la proxima... if self.global_slots.available_slot(): self.download_starter(download_item) #iniciar descarga si es posible (si ya no se esta bajando del mismo host) else: break def download_starter(self, download_item): """""" if self.global_slots.available_slot(): slot = True if host_accounts.get_account(download_item.host) is None: #si no es premium entrar. if not self.is_host_slot_available(download_item.host): slot = False if slot: self.global_slots.add_slot() self.add_thread(download_item.id, download_item.name, download_item.path, download_item.link, download_item.host, download_item.chunks) #crear thread y comenzar descarga. Metodo de threadmanager heredado self.active_downloads[download_item.id] = download_item del self.queue_downloads[download_item.id] def is_host_slot_available(self, host): """""" count = 0 host_slots = plugins_parser.get_plugin_item(host).get_slots_limit() if host_slots > 0: #-1 or 0 means unlimited slots for download_item in self.active_downloads.itervalues(): if host == download_item.host: count += 1 if count >= host_slots: return False return True def new_slot_limit(self, new_limit): """""" current_limit = self.global_slots.get_limit() self.global_slots.set_limit(new_limit) if new_limit > current_limit: self.next_download()
class DownloadManager(DownloadCore, ThreadManager): """ DownloadCore: .Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas. .Atributos heredados: self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads .Metodos heredados: - ThreadManager: .Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas. .Atributos heredados: self.thread_downloads (DICT) .Metodos heredados: get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status """ def __init__(self): """""" DownloadCore.__init__(self) #download_core.py ThreadManager.__init__(self) #thread_manager.py self.global_slots = Slots() #slots.py def start_all(self, id_order_list): """""" self.queue_downloads.update(self.stopped_downloads) self.stopped_downloads.clear() self.reorder_queue(id_order_list) for download_item in self.queue_downloads.values(): download_item.reset_fail_count() self.download_starter(download_item) def stop_all(self, filter_host_list=None): """""" filter_host_list = filter_host_list or [] for id_item, download_item in self.active_downloads.iteritems(): if download_item.host not in filter_host_list: self.stop_thread(id_item) for id_item, download_item in self.queue_downloads.items(): if download_item.host not in filter_host_list: self.stopped_downloads[id_item] = download_item del self.queue_downloads[id_item] def start_download(self, id_item): """""" try: download_item = self.stopped_downloads.pop(id_item) except KeyError: return False else: self.downloader_init([download_item, ], download_item.path) return True def stop_download(self, id_item): """""" try: download_item = self.active_downloads[id_item] except KeyError: try: download_item = self.queue_downloads.pop(id_item) self.stopped_downloads[id_item] = download_item except KeyError: return False else: return True else: self.stop_thread(id_item) return True def delete_download(self, id_items_list, remove_file=False): """""" for id_item in id_items_list: is_active = False try: download_item = self.active_downloads.pop(id_item) is_active = True except KeyError: try: download_item = self.stopped_downloads.pop(id_item) except KeyError: try: download_item = self.queue_downloads.pop(id_item) except KeyError: try: download_item = self.complete_downloads.pop(id_item) except KeyError: #bug: error on remove complete item from the gui. raise if is_active: th = self.get_thread(id_item) self.stop_thread(id_item) self.delete_thread(id_item) self.global_slots.remove_slot() self.next_download() else: th = None if remove_file: threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start() def remove_file(self, download_item, th): """""" if th is not None: th.join() try: os.remove(os.path.join(download_item.path, download_item.name)) except Exception as err: logger.warning(err) def update_active_downloads(self): """ Roba ciclos. This may change the active_downloads dict, you should get a dict copy before calling this method. """ for id_item, download_item in self.active_downloads.items(): item_data = self.get_thread_update(id_item) #threadmanager download_item.update(*item_data) limit_exceeded = self.is_limit_exceeded(id_item) #threadmanager old_status = download_item.status if old_status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR): if old_status == cons.STATUS_STOPPED: self.stopped_downloads[id_item] = download_item elif old_status == cons.STATUS_FINISHED: self.complete_downloads[id_item] = download_item else: #status == cons.STATUS_ERROR download_item.fail_count += 1 if download_item.fail_count > conf.get_retries_limit(): download_item.status = cons.STATUS_STOPPED self.stopped_downloads[id_item] = download_item else: download_item.status = cons.STATUS_QUEUE self.queue_downloads[id_item] = download_item self.delete_thread(id_item) #threadmanager del self.active_downloads[id_item] self.global_slots.remove_slot() self.next_download() if old_status == cons.STATUS_FINISHED: events.download_complete.emit(download_item) if not self.active_downloads and old_status != cons.STATUS_STOPPED: events.all_downloads_complete.emit() if limit_exceeded and self.active_downloads and old_status == cons.STATUS_ERROR: events.limit_exceeded.emit() def update_download_item(self, download_item): pass def downloader_init(self, item_list, path): """ Crea los threads para la descarga de cada archivo. """ #if not self.active_downloads: #after this method completes, there will be one active download at least. #events.trigger_downloading_process_pre_start() for download_item in item_list: download_item.path = path download_item.fail_count = 0 self.queue_downloads[download_item.id] = download_item self.download_starter(download_item) def next_download(self): """ Init next download on queue. This is called when an active download is stopped or has finished. """ for download_item in self.queue_downloads.values(): if self.global_slots.available_slot(): self.download_starter(download_item) else: break def download_starter(self, download_item): """""" if self.global_slots.available_slot(): slot = True if host_accounts.get_account(download_item.host) is None: #not premium. if not self.is_host_slot_available(download_item.host): slot = False if slot: self.global_slots.add_slot() self.create_thread(download_item) #threadmanager self.active_downloads[download_item.id] = download_item del self.queue_downloads[download_item.id] def is_host_slot_available(self, host): """""" count = 0 host_slots = plugins_parser.get_plugin_item(host).get_slots_limit() if host_slots > 0: #-1 or 0 means unlimited slots for download_item in self.active_downloads.itervalues(): if host == download_item.host: count += 1 if count >= host_slots: return False return True def new_slot_limit(self, new_limit): """""" current_limit = self.global_slots.get_limit() self.global_slots.set_limit(new_limit) if new_limit > current_limit: self.next_download()
class AddDownloadsManager: """""" def __init__(self): """""" self.__slots = Slots(limit=20) #checker stots self.__pending_downloads = OrderedDict() #{id_item: download_item, } self.__checking_downloads = {} #{id_item: download_item, } self.__ready_downloads = {} #{id_item: download_item, } checked_downloads self.__thread_checking_downloads = {} #{id_item: th, } def get_checking_downloads(self): return self.__checking_downloads.copy() def get_all_checking_downloads(self): all_downloads = {} all_downloads.update(self.__pending_downloads) all_downloads.update(self.__checking_downloads) all_downloads.update(self.__ready_downloads) return all_downloads def clear_pending(self): """ Erase pending_downloads dicts """ self.__pending_downloads.clear() self.__checking_downloads.clear() self.__ready_downloads.clear() self.__thread_checking_downloads.clear() self.__slots.set_slots(slots=0) def create_download_item(self, file_name, link, copy_link=True): """""" host = misc.get_host(link) if plugins_parser.services_dict.get(host, None) is None: host = cons.UNSUPPORTED download_item = DownloadItem(file_name, host, link, can_copy_link=copy_link) self.__pending_downloads[download_item.id] = download_item return download_item def start_checking(self): """""" for id_item, download_item in self.__pending_downloads.items(): if self.__slots.add_slot(): th = LinkChecker(download_item.link) th.start() self.__thread_checking_downloads[id_item] = th self.__checking_downloads[id_item] = download_item del self.__pending_downloads[id_item] else: break def update_checking_downloads(self): """ This may change the checking_downloads dict, you should get a dict copy before calling this method. """ for id_item, download_item in self.__checking_downloads.items(): th = self.__thread_checking_downloads[id_item] if not th.is_alive(): download_item.link_status = th.link_status download_item.size = th.size download_item.status_msg = th.status_msg if download_item.name == cons.UNKNOWN: #may be downloading download_item.name = th.file_name self.__ready_downloads[id_item] = download_item del self.__checking_downloads[id_item] del self.__thread_checking_downloads[id_item] self.__slots.remove_slot() self.start_checking() def recheck_items(self): """""" for id_item, download_item in self.__ready_downloads.items(): if download_item.link_status not in (cons.LINK_CHECKING, cons.LINK_ALIVE): download_item.link_status = cons.LINK_CHECKING #safe self.__pending_downloads[id_item] = download_item del self.__ready_downloads[id_item] self.start_checking() def pop_checking_items(self, id_item_list): """""" result_list = [] for id_item in id_item_list: # add this items. try: download_item = self.__checking_downloads.pop(id_item) except KeyError: try: download_item = self.__ready_downloads.pop(id_item) except KeyError: try: download_item = self.__pending_downloads.pop(id_item) except KeyError: raise else: del self.__thread_checking_downloads[id_item] self.__slots.remove_slot() #self.start_checking() result_list.append(download_item) self.start_checking() return result_list
class DownloadManager(DownloadCore, ThreadManager): #herencia multiple """ DownloadCore: .Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas. .Atributos heredados: self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads .Metodos heredados: - ThreadManager: .Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas. .Atributos heredados: self.thread_downloads (DICT) .Metodos heredados: get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status """ def __init__(self): """""" DownloadCore.__init__(self) #inicializar download_core.py ThreadManager.__init__(self) #inicializar thread_manager.py self.global_slots = Slots() #slots.py def start_all(self, id_order_list): """""" self.queue_downloads.update(self.stopped_downloads) self.stopped_downloads.clear() self.reorder_queue(id_order_list) for download_item in self.queue_downloads.values(): download_item.reset_fail_count() self.download_starter(download_item) def stop_all(self, filter_host_list=None): """""" if filter_host_list is None: filter_host_list = [] for id_item, download_item in self.active_downloads.iteritems(): if download_item.host not in filter_host_list: self.stop_thread(self.get_thread(id_item)) for id_item, download_item in self.queue_downloads.items(): if download_item.host not in filter_host_list: self.stopped_downloads[id_item] = download_item del self.queue_downloads[id_item] def start_download(self, id_item): #iniciar descarga. Boton play. """""" try: download_item = self.stopped_downloads.pop(id_item) except KeyError: return False else: self.downloader_init([ download_item, ], download_item.path) return True def stop_download(self, id_item): #detener descarga. Boton stop. """""" try: download_item = self.active_downloads[id_item] except KeyError: try: download_item = self.queue_downloads.pop(id_item) self.stopped_downloads[id_item] = download_item except KeyError: return False else: return True else: self.stop_thread(self.get_thread(id_item)) return True def delete_download(self, id_items_list): """""" for id_item in id_items_list: is_active = False try: download_item = self.active_downloads.pop(id_item) is_active = True except KeyError: try: download_item = self.stopped_downloads.pop(id_item) except KeyError: try: download_item = self.queue_downloads.pop(id_item) except KeyError as err: #will crash after this and that's ok. logger.exception(err) th = None if is_active: th = self.get_thread(id_item) self.stop_thread(th) self.delete_thread(id_item) self.global_slots.remove_slot() self.next_download() threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start() def remove_file(self, download_item, th): """""" if th is not None: th.join() try: os.remove(os.path.join(download_item.path, download_item.name)) except Exception as err: logger.warning(err) def get_items_update(self): """ Roba ciclos. """ result_list = [] for id_item, download_item in self.active_downloads.items(): item_data = self.get_thread_status( id_item) #Metodo de threadManager heredado if item_data is not None: download_item.update(*item_data) limit_exceeded = self.is_limit_exceeded(download_item.id) else: download_item.status = cons.STATUS_ERROR limit_exceeded = False status = download_item.status result_list.append(download_item) if status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR): if status == cons.STATUS_FINISHED: self.complete_downloads[id_item] = download_item elif status == cons.STATUS_ERROR: logger.warning("status error: {0}".format( download_item.host)) download_item.fail_count += 1 if download_item.fail_count > conf.get_retries_limit(): download_item.status = cons.STATUS_STOPPED self.stopped_downloads[id_item] = download_item else: download_item.status = cons.STATUS_QUEUE self.queue_downloads[id_item] = download_item else: #stopped self.stopped_downloads[id_item] = download_item self.delete_thread(id_item) #metodo de threadmanager heredado del self.active_downloads[id_item] self.global_slots.remove_slot( ) #remove the slot, so the next download can start. self.next_download() if status == cons.STATUS_FINISHED: events.trigger_download_complete(download_item) if not self.active_downloads and status != cons.STATUS_STOPPED: events.trigger_all_downloads_complete() elif limit_exceeded: #cons.STATUS_ERROR events.trigger_limit_exceeded() return result_list def downloader_init( self, item_list, save_to_path): #start_thread. Crea el thread para el link dado. """ Crea los threads para la descarga de cada archivo. """ #if not self.active_downloads: #after this method completes, there will be one active download at least. #events.trigger_downloading_process_pre_start() for download_item in item_list: #in self.pending_downloads: download_item.set_path(save_to_path) download_item.reset_fail_count() self.queue_downloads[download_item.id] = download_item self.download_starter(download_item) def next_download(self): """ Init next download on queue. This is called when an active download is stopped or has finished. """ for download_item in self.queue_downloads.values( ): #iniciar la proxima descarga en la cola, o la proxima... if self.global_slots.available_slot(): self.download_starter( download_item ) #iniciar descarga si es posible (si ya no se esta bajando del mismo host) else: break def download_starter(self, download_item): """""" if self.global_slots.available_slot(): slot = True if host_accounts.get_account( download_item.host) is None: #si no es premium entrar. if not self.is_host_slot_available(download_item.host): slot = False if slot: self.global_slots.add_slot() self.add_thread( download_item.id, download_item.name, download_item.path, download_item.link, download_item.host, download_item.chunks ) #crear thread y comenzar descarga. Metodo de threadmanager heredado self.active_downloads[download_item.id] = download_item del self.queue_downloads[download_item.id] def is_host_slot_available(self, host): """""" count = 0 host_slots = plugins_parser.get_plugin_item(host).get_slots_limit() if host_slots > 0: #-1 or 0 means unlimited slots for download_item in self.active_downloads.itervalues(): if host == download_item.host: count += 1 if count >= host_slots: return False return True def new_slot_limit(self, new_limit): """""" current_limit = self.global_slots.get_limit() self.global_slots.set_limit(new_limit) if new_limit > current_limit: self.next_download()
class AddDownloadsManager: """""" def __init__(self): """""" self.__slots = Slots(limit=20) #checker stots self.__pending_downloads = OrderedDict() #{id_item: download_item, } self.__checking_downloads = {} #{id_item: download_item, } self.__ready_downloads = { } #{id_item: download_item, } checked_downloads self.__thread_checking_downloads = {} #{id_item: th, } def clear_pending(self): """ Erase pending_downloads dicts """ self.__pending_downloads.clear() self.__checking_downloads.clear() self.__ready_downloads.clear() self.__thread_checking_downloads.clear() self.__slots.set_slots(slots=0) def create_download_item(self, file_name, size, link, copy_link=True): """""" host = misc.get_host(link) download_item = DownloadItem(file_name, host, size, link, can_copy_link=copy_link) self.__pending_downloads[download_item.id] = download_item return download_item def start_checking(self): """""" for id_item, download_item in self.__pending_downloads.items(): if self.__slots.add_slot(): th = LinkChecker(download_item.link) th.start() self.__thread_checking_downloads[id_item] = th self.__checking_downloads[id_item] = download_item del self.__pending_downloads[id_item] else: break def get_checking_update(self): """""" result_list = [] for id_item, download_item in self.__checking_downloads.items(): result_list.append(download_item) th = self.__thread_checking_downloads[id_item] if not th.is_alive(): download_item.host = th.host download_item.link_status = th.link_status download_item.size = th.size download_item.status_msg = th.status_msg if download_item.name == cons.UNKNOWN: #may be downloading download_item.name = th.file_name self.__ready_downloads[id_item] = download_item del self.__checking_downloads[id_item] del self.__thread_checking_downloads[id_item] self.__slots.remove_slot() self.start_checking() return result_list def recheck_items(self): """""" for id_item, download_item in self.__ready_downloads.items(): if download_item.link_status not in (cons.LINK_CHECKING, cons.LINK_ALIVE): download_item.link_status = cons.LINK_CHECKING #safe self.__pending_downloads[id_item] = download_item del self.__ready_downloads[id_item] self.start_checking() def get_added_items( self, id_add_list): #get download_items (added) from pending. """""" result_list = [] for id_item in id_add_list: #add this items. try: download_item = self.__checking_downloads.pop(id_item) del self.__thread_checking_downloads[id_item] self.__slots.remove_slot() except: try: download_item = self.__ready_downloads.pop(id_item) except: try: download_item = self.__pending_downloads.pop( id_item ) #so we only keep the non-added items in the pending_dict except Exception as err: download_item = None logger.warning(err) if download_item is not None: result_list.append(download_item) return result_list