class DownloadManager(DownloadCore, ThreadManager): #herencia multiple
"""
DownloadCore:
.Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas.
.Atributos heredados:
self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads
.Metodos heredados:
-
ThreadManager:
.Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas.
.Atributos heredados:
self.thread_downloads (DICT)
.Metodos heredados:
get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status
"""
def __init__(self):
""""""
DownloadCore.__init__(self) #inicializar download_core.py
ThreadManager.__init__(self) #inicializar thread_manager.py
self.global_slots = Slots() #slots.py
def start_all(self, id_order_list):
""""""
self.queue_downloads.update(self.stopped_downloads)
self.stopped_downloads.clear()
self.reorder_queue(id_order_list)
for download_item in self.queue_downloads.values():
download_item.reset_fail_count()
self.download_starter(download_item)
def stop_all(self, filter_host_list=None):
""""""
if filter_host_list is None:
filter_host_list = []
for id_item, download_item in self.active_downloads.iteritems():
if download_item.host not in filter_host_list:
self.stop_thread(self.get_thread(id_item))
for id_item, download_item in self.queue_downloads.items():
if download_item.host not in filter_host_list:
self.stopped_downloads[id_item] = download_item
del self.queue_downloads[id_item]
def start_download(self, id_item): #iniciar descarga. Boton play.
""""""
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
return False
else:
self.downloader_init([download_item, ], download_item.path)
return True
def stop_download(self, id_item): #detener descarga. Boton stop.
""""""
try:
download_item = self.active_downloads[id_item]
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
self.stopped_downloads[id_item] = download_item
except KeyError:
return False
else:
return True
else:
self.stop_thread(self.get_thread(id_item))
return True
def delete_download(self, id_items_list):
""""""
for id_item in id_items_list:
is_active = False
try:
download_item = self.active_downloads.pop(id_item)
is_active = True
except KeyError:
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
except KeyError as err: #will crash after this and that's ok.
logger.exception(err)
th = None
if is_active:
th = self.get_thread(id_item)
self.stop_thread(th)
self.delete_thread(id_item)
self.global_slots.remove_slot()
self.next_download()
threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start()
def remove_file(self, download_item, th):
""""""
if th is not None:
th.join()
try:
os.remove(os.path.join(download_item.path, download_item.name))
except Exception as err:
logger.warning(err)
def get_items_update(self):
"""
Roba ciclos.
"""
result_list = []
for id_item, download_item in self.active_downloads.items():
item_data = self.get_thread_status(id_item) #Metodo de threadManager heredado
if item_data is not None:
download_item.update(*item_data)
limit_exceeded = self.is_limit_exceeded(download_item.id)
else:
download_item.status = cons.STATUS_ERROR
limit_exceeded = False
status = download_item.status
result_list.append(download_item)
if status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR):
if status == cons.STATUS_FINISHED:
self.complete_downloads[id_item] = download_item
elif status == cons.STATUS_ERROR:
logger.warning("status error: {0}".format(download_item.host))
download_item.fail_count += 1
if download_item.fail_count > conf.get_retries_limit():
download_item.status = cons.STATUS_STOPPED
self.stopped_downloads[id_item] = download_item
else:
download_item.status = cons.STATUS_QUEUE
self.queue_downloads[id_item] = download_item
else: #stopped
self.stopped_downloads[id_item] = download_item
self.delete_thread(id_item) #metodo de threadmanager heredado
del self.active_downloads[id_item]
self.global_slots.remove_slot() #remove the slot, so the next download can start.
self.next_download()
if status == cons.STATUS_FINISHED:
events.trigger_download_complete(download_item)
if not self.active_downloads and status != cons.STATUS_STOPPED:
events.trigger_all_downloads_complete()
elif limit_exceeded: #cons.STATUS_ERROR
events.trigger_limit_exceeded()
return result_list
def downloader_init(self, item_list, save_to_path): #start_thread. Crea el thread para el link dado.
"""
Crea los threads para la descarga de cada archivo.
"""
#if not self.active_downloads: #after this method completes, there will be one active download at least.
#events.trigger_downloading_process_pre_start()
for download_item in item_list: #in self.pending_downloads:
download_item.set_path(save_to_path)
download_item.reset_fail_count()
self.queue_downloads[download_item.id] = download_item
self.download_starter(download_item)
def next_download(self):
"""
Init next download on queue. This is called when an active download is stopped or has finished.
"""
for download_item in self.queue_downloads.values(): #iniciar la proxima descarga en la cola, o la proxima...
if self.global_slots.available_slot():
self.download_starter(download_item) #iniciar descarga si es posible (si ya no se esta bajando del mismo host)
else:
break
def download_starter(self, download_item):
""""""
if self.global_slots.available_slot():
slot = True
if host_accounts.get_account(download_item.host) is None: #si no es premium entrar.
if not self.is_host_slot_available(download_item.host):
slot = False
if slot:
self.global_slots.add_slot()
self.add_thread(download_item.id, download_item.name, download_item.path, download_item.link, download_item.host, download_item.chunks) #crear thread y comenzar descarga. Metodo de threadmanager heredado
self.active_downloads[download_item.id] = download_item
del self.queue_downloads[download_item.id]
def is_host_slot_available(self, host):
""""""
count = 0
host_slots = plugins_parser.get_plugin_item(host).get_slots_limit()
if host_slots > 0: #-1 or 0 means unlimited slots
for download_item in self.active_downloads.itervalues():
if host == download_item.host:
count += 1
if count >= host_slots:
return False
return True
def new_slot_limit(self, new_limit):
""""""
current_limit = self.global_slots.get_limit()
self.global_slots.set_limit(new_limit)
if new_limit > current_limit:
self.next_download()
class DownloadManager(DownloadCore, ThreadManager):
"""
DownloadCore:
.Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas.
.Atributos heredados:
self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads
.Metodos heredados:
-
ThreadManager:
.Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas.
.Atributos heredados:
self.thread_downloads (DICT)
.Metodos heredados:
get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status
"""
def __init__(self):
""""""
DownloadCore.__init__(self) #download_core.py
ThreadManager.__init__(self) #thread_manager.py
self.global_slots = Slots() #slots.py
def start_all(self, id_order_list):
""""""
self.queue_downloads.update(self.stopped_downloads)
self.stopped_downloads.clear()
self.reorder_queue(id_order_list)
for download_item in self.queue_downloads.values():
download_item.reset_fail_count()
self.download_starter(download_item)
def stop_all(self, filter_host_list=None):
""""""
filter_host_list = filter_host_list or []
for id_item, download_item in self.active_downloads.iteritems():
if download_item.host not in filter_host_list:
self.stop_thread(id_item)
for id_item, download_item in self.queue_downloads.items():
if download_item.host not in filter_host_list:
self.stopped_downloads[id_item] = download_item
del self.queue_downloads[id_item]
def start_download(self, id_item):
""""""
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
return False
else:
self.downloader_init([download_item, ], download_item.path)
return True
def stop_download(self, id_item):
""""""
try:
download_item = self.active_downloads[id_item]
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
self.stopped_downloads[id_item] = download_item
except KeyError:
return False
else:
return True
else:
self.stop_thread(id_item)
return True
def delete_download(self, id_items_list, remove_file=False):
""""""
for id_item in id_items_list:
is_active = False
try:
download_item = self.active_downloads.pop(id_item)
is_active = True
except KeyError:
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
except KeyError:
try:
download_item = self.complete_downloads.pop(id_item)
except KeyError:
#bug: error on remove complete item from the gui.
raise
if is_active:
th = self.get_thread(id_item)
self.stop_thread(id_item)
self.delete_thread(id_item)
self.global_slots.remove_slot()
self.next_download()
else:
th = None
if remove_file:
threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start()
def remove_file(self, download_item, th):
""""""
if th is not None:
th.join()
try:
os.remove(os.path.join(download_item.path, download_item.name))
except Exception as err:
logger.warning(err)
def update_active_downloads(self):
"""
Roba ciclos.
This may change the active_downloads dict, you should get a dict copy before calling this method.
"""
for id_item, download_item in self.active_downloads.items():
item_data = self.get_thread_update(id_item) #threadmanager
download_item.update(*item_data)
limit_exceeded = self.is_limit_exceeded(id_item) #threadmanager
old_status = download_item.status
if old_status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR):
if old_status == cons.STATUS_STOPPED:
self.stopped_downloads[id_item] = download_item
elif old_status == cons.STATUS_FINISHED:
self.complete_downloads[id_item] = download_item
else: #status == cons.STATUS_ERROR
download_item.fail_count += 1
if download_item.fail_count > conf.get_retries_limit():
download_item.status = cons.STATUS_STOPPED
self.stopped_downloads[id_item] = download_item
else:
download_item.status = cons.STATUS_QUEUE
self.queue_downloads[id_item] = download_item
self.delete_thread(id_item) #threadmanager
del self.active_downloads[id_item]
self.global_slots.remove_slot()
self.next_download()
if old_status == cons.STATUS_FINISHED:
events.download_complete.emit(download_item)
if not self.active_downloads and old_status != cons.STATUS_STOPPED:
events.all_downloads_complete.emit()
if limit_exceeded and self.active_downloads and old_status == cons.STATUS_ERROR:
events.limit_exceeded.emit()
def update_download_item(self, download_item):
pass
def downloader_init(self, item_list, path):
"""
Crea los threads para la descarga de cada archivo.
"""
#if not self.active_downloads: #after this method completes, there will be one active download at least.
#events.trigger_downloading_process_pre_start()
for download_item in item_list:
download_item.path = path
download_item.fail_count = 0
self.queue_downloads[download_item.id] = download_item
self.download_starter(download_item)
def next_download(self):
"""
Init next download on queue. This is called when an active download is stopped or has finished.
"""
for download_item in self.queue_downloads.values():
if self.global_slots.available_slot():
self.download_starter(download_item)
else:
break
def download_starter(self, download_item):
""""""
if self.global_slots.available_slot():
slot = True
if host_accounts.get_account(download_item.host) is None: #not premium.
if not self.is_host_slot_available(download_item.host):
slot = False
if slot:
self.global_slots.add_slot()
self.create_thread(download_item) #threadmanager
self.active_downloads[download_item.id] = download_item
del self.queue_downloads[download_item.id]
def is_host_slot_available(self, host):
""""""
count = 0
host_slots = plugins_parser.get_plugin_item(host).get_slots_limit()
if host_slots > 0: #-1 or 0 means unlimited slots
for download_item in self.active_downloads.itervalues():
if host == download_item.host:
count += 1
if count >= host_slots:
return False
return True
def new_slot_limit(self, new_limit):
""""""
current_limit = self.global_slots.get_limit()
self.global_slots.set_limit(new_limit)
if new_limit > current_limit:
self.next_download()
class AddDownloadsManager:
""""""
def __init__(self):
""""""
self.__slots = Slots(limit=20) #checker stots
self.__pending_downloads = OrderedDict() #{id_item: download_item, }
self.__checking_downloads = {} #{id_item: download_item, }
self.__ready_downloads = {} #{id_item: download_item, } checked_downloads
self.__thread_checking_downloads = {} #{id_item: th, }
def get_checking_downloads(self):
return self.__checking_downloads.copy()
def get_all_checking_downloads(self):
all_downloads = {}
all_downloads.update(self.__pending_downloads)
all_downloads.update(self.__checking_downloads)
all_downloads.update(self.__ready_downloads)
return all_downloads
def clear_pending(self):
"""
Erase pending_downloads dicts
"""
self.__pending_downloads.clear()
self.__checking_downloads.clear()
self.__ready_downloads.clear()
self.__thread_checking_downloads.clear()
self.__slots.set_slots(slots=0)
def create_download_item(self, file_name, link, copy_link=True):
""""""
host = misc.get_host(link)
if plugins_parser.services_dict.get(host, None) is None:
host = cons.UNSUPPORTED
download_item = DownloadItem(file_name, host, link, can_copy_link=copy_link)
self.__pending_downloads[download_item.id] = download_item
return download_item
def start_checking(self):
""""""
for id_item, download_item in self.__pending_downloads.items():
if self.__slots.add_slot():
th = LinkChecker(download_item.link)
th.start()
self.__thread_checking_downloads[id_item] = th
self.__checking_downloads[id_item] = download_item
del self.__pending_downloads[id_item]
else:
break
def update_checking_downloads(self):
"""
This may change the checking_downloads dict, you should get a dict copy before calling this method.
"""
for id_item, download_item in self.__checking_downloads.items():
th = self.__thread_checking_downloads[id_item]
if not th.is_alive():
download_item.link_status = th.link_status
download_item.size = th.size
download_item.status_msg = th.status_msg
if download_item.name == cons.UNKNOWN: #may be downloading
download_item.name = th.file_name
self.__ready_downloads[id_item] = download_item
del self.__checking_downloads[id_item]
del self.__thread_checking_downloads[id_item]
self.__slots.remove_slot()
self.start_checking()
def recheck_items(self):
""""""
for id_item, download_item in self.__ready_downloads.items():
if download_item.link_status not in (cons.LINK_CHECKING, cons.LINK_ALIVE):
download_item.link_status = cons.LINK_CHECKING #safe
self.__pending_downloads[id_item] = download_item
del self.__ready_downloads[id_item]
self.start_checking()
def pop_checking_items(self, id_item_list):
""""""
result_list = []
for id_item in id_item_list: # add this items.
try:
download_item = self.__checking_downloads.pop(id_item)
except KeyError:
try:
download_item = self.__ready_downloads.pop(id_item)
except KeyError:
try:
download_item = self.__pending_downloads.pop(id_item)
except KeyError:
raise
else:
del self.__thread_checking_downloads[id_item]
self.__slots.remove_slot()
#self.start_checking()
result_list.append(download_item)
self.start_checking()
return result_list
class DownloadManager(DownloadCore, ThreadManager): #herencia multiple
"""
DownloadCore:
.Contiene las listas con los items (tipos de clase, DownloadItem) de descarga, y los metodos para modificar esas listas.
.Atributos heredados:
self.active_downloads, self.queue_downloads, self.complete_downloads, self.stopped_downloads
.Metodos heredados:
-
ThreadManager:
.Contiene el diccionario con los threads (de descarga, clase Downloader) instanciados de las descargas activas.
.Atributos heredados:
self.thread_downloads (DICT)
.Metodos heredados:
get_thread, add_thread, delete_thread, stop_thread, stop_all, get_thread_status
"""
def __init__(self):
""""""
DownloadCore.__init__(self) #inicializar download_core.py
ThreadManager.__init__(self) #inicializar thread_manager.py
self.global_slots = Slots() #slots.py
def start_all(self, id_order_list):
""""""
self.queue_downloads.update(self.stopped_downloads)
self.stopped_downloads.clear()
self.reorder_queue(id_order_list)
for download_item in self.queue_downloads.values():
download_item.reset_fail_count()
self.download_starter(download_item)
def stop_all(self, filter_host_list=None):
""""""
if filter_host_list is None:
filter_host_list = []
for id_item, download_item in self.active_downloads.iteritems():
if download_item.host not in filter_host_list:
self.stop_thread(self.get_thread(id_item))
for id_item, download_item in self.queue_downloads.items():
if download_item.host not in filter_host_list:
self.stopped_downloads[id_item] = download_item
del self.queue_downloads[id_item]
def start_download(self, id_item): #iniciar descarga. Boton play.
""""""
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
return False
else:
self.downloader_init([
download_item,
], download_item.path)
return True
def stop_download(self, id_item): #detener descarga. Boton stop.
""""""
try:
download_item = self.active_downloads[id_item]
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
self.stopped_downloads[id_item] = download_item
except KeyError:
return False
else:
return True
else:
self.stop_thread(self.get_thread(id_item))
return True
def delete_download(self, id_items_list):
""""""
for id_item in id_items_list:
is_active = False
try:
download_item = self.active_downloads.pop(id_item)
is_active = True
except KeyError:
try:
download_item = self.stopped_downloads.pop(id_item)
except KeyError:
try:
download_item = self.queue_downloads.pop(id_item)
except KeyError as err: #will crash after this and that's ok.
logger.exception(err)
th = None
if is_active:
th = self.get_thread(id_item)
self.stop_thread(th)
self.delete_thread(id_item)
self.global_slots.remove_slot()
self.next_download()
threading.Thread(group=None,
target=self.remove_file,
name=None,
args=(download_item, th)).start()
def remove_file(self, download_item, th):
""""""
if th is not None:
th.join()
try:
os.remove(os.path.join(download_item.path, download_item.name))
except Exception as err:
logger.warning(err)
def get_items_update(self):
"""
Roba ciclos.
"""
result_list = []
for id_item, download_item in self.active_downloads.items():
item_data = self.get_thread_status(
id_item) #Metodo de threadManager heredado
if item_data is not None:
download_item.update(*item_data)
limit_exceeded = self.is_limit_exceeded(download_item.id)
else:
download_item.status = cons.STATUS_ERROR
limit_exceeded = False
status = download_item.status
result_list.append(download_item)
if status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED,
cons.STATUS_ERROR):
if status == cons.STATUS_FINISHED:
self.complete_downloads[id_item] = download_item
elif status == cons.STATUS_ERROR:
logger.warning("status error: {0}".format(
download_item.host))
download_item.fail_count += 1
if download_item.fail_count > conf.get_retries_limit():
download_item.status = cons.STATUS_STOPPED
self.stopped_downloads[id_item] = download_item
else:
download_item.status = cons.STATUS_QUEUE
self.queue_downloads[id_item] = download_item
else: #stopped
self.stopped_downloads[id_item] = download_item
self.delete_thread(id_item) #metodo de threadmanager heredado
del self.active_downloads[id_item]
self.global_slots.remove_slot(
) #remove the slot, so the next download can start.
self.next_download()
if status == cons.STATUS_FINISHED:
events.trigger_download_complete(download_item)
if not self.active_downloads and status != cons.STATUS_STOPPED:
events.trigger_all_downloads_complete()
elif limit_exceeded: #cons.STATUS_ERROR
events.trigger_limit_exceeded()
return result_list
def downloader_init(
self, item_list,
save_to_path): #start_thread. Crea el thread para el link dado.
"""
Crea los threads para la descarga de cada archivo.
"""
#if not self.active_downloads: #after this method completes, there will be one active download at least.
#events.trigger_downloading_process_pre_start()
for download_item in item_list: #in self.pending_downloads:
download_item.set_path(save_to_path)
download_item.reset_fail_count()
self.queue_downloads[download_item.id] = download_item
self.download_starter(download_item)
def next_download(self):
"""
Init next download on queue. This is called when an active download is stopped or has finished.
"""
for download_item in self.queue_downloads.values(
): #iniciar la proxima descarga en la cola, o la proxima...
if self.global_slots.available_slot():
self.download_starter(
download_item
) #iniciar descarga si es posible (si ya no se esta bajando del mismo host)
else:
break
def download_starter(self, download_item):
""""""
if self.global_slots.available_slot():
slot = True
if host_accounts.get_account(
download_item.host) is None: #si no es premium entrar.
if not self.is_host_slot_available(download_item.host):
slot = False
if slot:
self.global_slots.add_slot()
self.add_thread(
download_item.id, download_item.name, download_item.path,
download_item.link, download_item.host,
download_item.chunks
) #crear thread y comenzar descarga. Metodo de threadmanager heredado
self.active_downloads[download_item.id] = download_item
del self.queue_downloads[download_item.id]
def is_host_slot_available(self, host):
""""""
count = 0
host_slots = plugins_parser.get_plugin_item(host).get_slots_limit()
if host_slots > 0: #-1 or 0 means unlimited slots
for download_item in self.active_downloads.itervalues():
if host == download_item.host:
count += 1
if count >= host_slots:
return False
return True
def new_slot_limit(self, new_limit):
""""""
current_limit = self.global_slots.get_limit()
self.global_slots.set_limit(new_limit)
if new_limit > current_limit:
self.next_download()
class AddDownloadsManager:
""""""
def __init__(self):
""""""
self.__slots = Slots(limit=20) #checker stots
self.__pending_downloads = OrderedDict() #{id_item: download_item, }
self.__checking_downloads = {} #{id_item: download_item, }
self.__ready_downloads = {
} #{id_item: download_item, } checked_downloads
self.__thread_checking_downloads = {} #{id_item: th, }
def clear_pending(self):
"""
Erase pending_downloads dicts
"""
self.__pending_downloads.clear()
self.__checking_downloads.clear()
self.__ready_downloads.clear()
self.__thread_checking_downloads.clear()
self.__slots.set_slots(slots=0)
def create_download_item(self, file_name, size, link, copy_link=True):
""""""
host = misc.get_host(link)
download_item = DownloadItem(file_name,
host,
size,
link,
can_copy_link=copy_link)
self.__pending_downloads[download_item.id] = download_item
return download_item
def start_checking(self):
""""""
for id_item, download_item in self.__pending_downloads.items():
if self.__slots.add_slot():
th = LinkChecker(download_item.link)
th.start()
self.__thread_checking_downloads[id_item] = th
self.__checking_downloads[id_item] = download_item
del self.__pending_downloads[id_item]
else:
break
def get_checking_update(self):
""""""
result_list = []
for id_item, download_item in self.__checking_downloads.items():
result_list.append(download_item)
th = self.__thread_checking_downloads[id_item]
if not th.is_alive():
download_item.host = th.host
download_item.link_status = th.link_status
download_item.size = th.size
download_item.status_msg = th.status_msg
if download_item.name == cons.UNKNOWN: #may be downloading
download_item.name = th.file_name
self.__ready_downloads[id_item] = download_item
del self.__checking_downloads[id_item]
del self.__thread_checking_downloads[id_item]
self.__slots.remove_slot()
self.start_checking()
return result_list
def recheck_items(self):
""""""
for id_item, download_item in self.__ready_downloads.items():
if download_item.link_status not in (cons.LINK_CHECKING,
cons.LINK_ALIVE):
download_item.link_status = cons.LINK_CHECKING #safe
self.__pending_downloads[id_item] = download_item
del self.__ready_downloads[id_item]
self.start_checking()
def get_added_items(
self, id_add_list): #get download_items (added) from pending.
""""""
result_list = []
for id_item in id_add_list: #add this items.
try:
download_item = self.__checking_downloads.pop(id_item)
del self.__thread_checking_downloads[id_item]
self.__slots.remove_slot()
except:
try:
download_item = self.__ready_downloads.pop(id_item)
except:
try:
download_item = self.__pending_downloads.pop(
id_item
) #so we only keep the non-added items in the pending_dict
except Exception as err:
download_item = None
logger.warning(err)
if download_item is not None:
result_list.append(download_item)
return result_list
