def filling_list_of_letters(self): try: name_of_folder = self.Main_window.Folders_box.currentText() except Exception: name_of_folder = 'INBOX' list_of_folders = self.Imap.parse_list_of_folders() if len(list_of_folders) == 0: self.Grid.addWidget( QtWidgets.QLabel('Problems with decode folders'), 0, 0) for item in list_of_folders: if item[0] == name_of_folder: self.Imap.select_folder(item[1]) break try: count_of_letters = int(self.Imap.get_count_of_letters()) except Exception: self.Imap.select_folder('INBOX') print('не упал') count_of_letters = int(self.Imap.get_count_of_letters()) print('упал') if count_of_letters == 0: self.Grid.addWidget(QtWidgets.QLabel('No letters'), 0, 0) else: trd = DownloadThread(self.Imap, count_of_letters) self.trd = trd trd.start() trd.finished.connect(self.finished) self.set_disabled(True)
def main(): '''登录微博''' paramDict = read_config() if not login(paramDict['username'], paramDict['password']): exit() '''与数据库建立连接和指针''' pool = PooledDB(MySQLdb, int(paramDict['threadnum']), host = paramDict['dbhost'], user = paramDict['dbuser'], passwd = paramDict['dbpasswd'], db = paramDict['dbname']) conn = pool.connection() cur = conn.cursor() '''读取未爬取的链接列表放入队列''' urlQLock = threading.Lock() tableName = 'users' sql = 'select id, uid from %s where isCrawled = 0' % tableName cur.execute(sql) result = cur.fetchall() urlQ = Queue(len(result)) for entry in result: urlQ.put(entry) '''建立线程''' for i in xrange(int(paramDict['threadnum'])): thr = DownloadThread(pool, urlQ, urlQLock) threadPool.append(thr) thr.start() '''检查是否存在结束的线程,若有,则重新建立新的线程''' while True: try: sleep(60) '''当队列为空时,跳出循环''' if not urlQ.qsize(): break if threading.activeCount() < int(paramDict['threadnum']) + 1: '''检查哪个线程已经结束,将其清除''' i = 0 for thr in threadPool: if not thr.isAlive(): thr.clear() del threadPool[i] newThr = DownloadThread(pool, urlQ, urlQLock) threadPool.append(newThr) newThr.start() else: i += 1 except: print sys.exc_info()[0] for thr in threadPool: thr.end() break print 'Main thread end!'
class DownloadManager(object): XML_TAG = 'download-manager' DEFAULT_FILENAME = "download-manager.xml" # Posted when the download manager stops downloading because # the queue is empty. DONE_NOTIFICATION = "DownloadManagerDoneNotification" # Posted when the download manager stopped after a call to the # stop method. Note that this notification is NOT posted when # the manager stops due to an empty queue. STOPPED_NOTIFICATION = "DownloadManagerStoppedNotification" # Posted when the downloading of a new object has commenced. NEXT_DOWNLOAD_NOTIFICATION = "DownloadManagerNextDownloadNotification" # Posted when new output in self.output is available. NEW_OUTPUT_NOTIFICATION = "DownloadManagerNewOutputNotification" def __init__(self, filename=DEFAULT_FILENAME): self.queue = [] # Collection of DownloadConfiguration instances self.done = [] # Cellection of ... instances self.active = None # DownloadConfiguration instance or None # List of tuples (text, type) where text is a string and type # is 0 for stdout and 1 for stderr. self.output = [] self.__downloadThread = None self.__shouldStop = False self.filename = filename self.__lock = RLock() # Saving and loading def synchronize(self): """Writes and saves the download status.""" with self.__lock: tb = ET.TreeBuilder() tb.start(self.XML_TAG) # Queue tb.start('queue') for dc in self.queue: dc.addToTreeBuilder(tb) if self.active != None: self.active.addToTreeBuilder(tb) tb.end('queue') # Done tb.start('done') for dc in self.done: dc.addToTreeBuilder(tb) tb.end('done') root = tb.close() tree = ET.ElementTree(root) tree.write(self.filename) def load(self): """Loads the download status from the file.""" with self.__lock: doc = MD.parse(self.filename) root = doc.documentElement if root.tagName != self.XML_TAG: raise UnsupportedFormat() cn = root.childNodes for c in cn: if c.nodeType != MD.Node.ELEMENT_NODE: continue if c.tagName == 'queue': self.__parseQueue(c) if c.tagName == 'done': self.__parseDone(c) def __parseQueue(self, queueElem): cn = queueElem.childNodes for c in cn: if c.nodeType != MD.Node.ELEMENT_NODE: continue if c.tagName == DownloadConfiguration.XML_TAG: dc = DownloadConfiguration.fromXMLElement(c) if dc != None: self.queue.append(dc) def __parseDone(self, doneElem): cn = doneElem.childNodes for c in cn: if c.nodeType != MD.Node.ELEMENT_NODE: continue if c.tagName == DownloadConfiguration.XML_TAG: dc = DownloadConfiguration.fromXMLElement(c) if dc != None: self.done.append(dc) # Managing queue def addToQueue(self, dc): """Adds a download configuration to the queue.""" with self.__lock: self.queue.append(dc) self.synchronize() def removeFromQueue(self, index, alertParent=None): """Removes a download configuration from the queue. index - The index of the download configuration alertParent - The parent of the confirmation alert if you want to show one to the user. If you want to remove the download configuration without confiromation, pass None. """ with self.__lock: if index >= len(self.queue): return if alertParent != None: self.__showRemovalConfirmation(index, alertParent) else: dc = self.queue[index] self.__removalConfiguration = dc self.__continueRemoval() def clearDone(self): """Removes all configurations from the done list.""" with self.__lock: self.done.clear() self.synchronize() def __showRemovalConfirmation(self, index, alertParent): with self.__lock: if index >= len(self.queue): return dc = self.queue[index] self.__removalConfiguration = dc title = "Are you sure you want to delete the selected " title+= "download configuration?" msg = "This operation cannot be undone." a = MessageAlert(alertParent, title, msg) b = Button("OK", self.__continueRemoval, Button.SHORTCUT_ENTER) a.addButton(b) b = Button("Cancel", self.__endRemovalConfirmation, 'c') a.addButton(b) self.__removalAlert = a alertParent.beginModalScreen(a) def __endRemovalConfirmation(self): self.__removalAlert.parent.endModalScreen(self.__removalAlert) self.__removalAlert = None def __continueRemoval(self): with self.__lock: try: self.queue.remove(self.__removalConfiguration) self.synchronize() except ValueError: pass self.__removalAlert.parent.endModalScreen(self.__removalAlert) self.__removalAlert = None # Downloading Objects def isDownloading(self): with self.__lock: return self.active != None def startDownloading(self): with self.__lock: self.__shouldStop = False if self.active != None: return self.__downloadNextItem() def stopDownloading(self): with self.__lock: self.__shouldStop = True def __downloadNextItem(self): with self.__lock: if self.__shouldStop == True: self.__shouldStop = False self.active = None notifName = DownloadThread.DONE_NOTIFICATION Notification.removeObserver(self, notifName) self.__notifyStopped() return if len(self.queue) == 0: self.active = None notifName = DownloadThread.DONE_NOTIFICATION Notification.removeObserver(self, notifName) self.__notifyDone() return # Start the download-thread notifName = DownloadThread.DONE_NOTIFICATION Notification.addObserver(self, notifName) notifName = DownloadThread.NEW_OUTPUT_NOTIFICATION Notification.addObserver(self, notifName) self.active = self.queue[0] del self.queue[0] self.__downloadThread = DownloadThread(self.active) self.__downloadThread.start() self.__notifyNextDownload() def __handleDownloadThreadDoneNotification(self, notif): if notif.sender != self.__downloadThread: return if self.active != None: with self.__lock: self.done.append(self.active) self.active = None self.synchronize() self.__downloadThread = None self.__downloadNextItem() def __handleDownloadThreadNewOutputNotification(self, notif): if notif.sender != self.__downloadThread: return self.output = notif.sender.output self.__notifyNewOutput() # Notification def __notifyDone(self): n = Notification(self.DONE_NOTIFICATION, self) Notification.post(n) def __notifyStopped(self): n = Notification(self.STOPPED_NOTIFICATION, self) Notification.post(n) def __notifyNextDownload(self): n = Notification(self.NEXT_DOWNLOAD_NOTIFICATION, self) Notification.post(n) def __notifyNewOutput(self): n = Notification(self.NEW_OUTPUT_NOTIFICATION, self) Notification.post(n) def handleNotification(self, notif): if notif.name == DownloadThread.DONE_NOTIFICATION: self.__handleDownloadThreadDoneNotification(notif) if notif.name == DownloadThread.NEW_OUTPUT_NOTIFICATION: self.__handleDownloadThreadNewOutputNotification(notif)
class DownloadTask(QObject): status_changed = pyqtSignal(str) aborted = pyqtSignal() def __init__(self, url=None, target=None): super().__init__() self.url = url if url is not None else default_url self.target = target if target is not None else target_prefix + self._available_target_for( url) self.status = 'Waiting' self.thread_ = None self.stopping = False def __str__(self): return 'Download task [URL = \'{}\', target = \'{}\', status = \'{}\']'.format( self.url, self.target, self.status) def start(self): if self.stopping: return self.thread_ = DownloadThread(self.url, self.target) self.thread_.completed.connect(self._on_completed) self.thread_.aborted.connect(self._on_aborted) self.thread_.failed.connect(self._on_failed) self.thread_.progressed.connect(self._on_progressed) self.thread_.start() def stop(self): if self.stopping: return self.stopping = True if self.thread_.isRunning(): self.thread_.to_abort = True else: self._on_aborted(False) def _on_completed(self): self._change_status('Completed') def _on_aborted(self, need_removal): if need_removal: self._remove_target() self._change_status('Aborted') self.stopping = False self.aborted.emit() def _on_failed(self, err_str): if err_str: self._change_status('Failed ({})'.format(err_str)) else: self._change_status('Failed') self._remove_target() def _on_progressed(self, progress): self._change_status('{:.2f}%'.format(progress * 100)) def _available_target_for(self, url): res = url[url.rfind('/') + len('/'):] separator_pos = res.rfind('.') if separator_pos != -1: filename, extension = res[:separator_pos], res[separator_pos + len('.'):] else: filename, extension = res, '' n = 0 while os.path.exists(res): n += 1 res = self._make_res(filename, extension, n) return res def _make_res(self, filename, extension, n): if n: return filename + ' (' + str(n) + ').' + extension return filename + '.' + extension def _change_status(self, status): self.status = status self.status_changed.emit(status) def _remove_target(self): if os.path.exists(self.target): os.remove(self.target)
def startDownload(self, downloadInfo): """ 通过DonwloadInfo模型中的数据进行下载 """ name = downloadInfo.title + ".torrent" downloadThread = DownloadThread(downloadInfo = downloadInfo, name = name, sem = self.sem) downloadThread.start()
class ListOfLettersWidget(QtWidgets.QWidget): def __init__(self, imap, main_window): super().__init__() self.Imap = imap self.Main_window = main_window self.List_of_marked = [] self.Index_of_current_letter = 0 self.initUI() def initUI(self): grid = QtWidgets.QGridLayout() self.setLayout(grid) self.Grid = grid self.filling_list_of_letters() self.Main_window.show_more_button.clicked.connect( self.show_more_clicked) def show_more_clicked(self): index = self.Index_of_current_letter count = int(self.Imap.get_count_of_letters()) - index - 1 if count <= 0: return self.trd = DownloadThread(self.Imap, count) self.trd.start() self.trd.finished.connect(self.finished) self.set_disabled(True) def filling_list_of_letters(self): try: name_of_folder = self.Main_window.Folders_box.currentText() except Exception: name_of_folder = 'INBOX' list_of_folders = self.Imap.parse_list_of_folders() if len(list_of_folders) == 0: self.Grid.addWidget( QtWidgets.QLabel('Problems with decode folders'), 0, 0) for item in list_of_folders: if item[0] == name_of_folder: self.Imap.select_folder(item[1]) break try: count_of_letters = int(self.Imap.get_count_of_letters()) except Exception: self.Imap.select_folder('INBOX') print('не упал') count_of_letters = int(self.Imap.get_count_of_letters()) print('упал') if count_of_letters == 0: self.Grid.addWidget(QtWidgets.QLabel('No letters'), 0, 0) else: trd = DownloadThread(self.Imap, count_of_letters) self.trd = trd trd.start() trd.finished.connect(self.finished) self.set_disabled(True) def finished(self): result = self.trd.result.copy() self.set_disabled(False) for i in range(len(result)): self.Grid.addWidget(LetterWidget(result[i], self), i + self.Index_of_current_letter, 0) self.Index_of_current_letter += len(result) self.Main_window.autorization_window.hide() self.Main_window.show() def set_disabled(self, flag): for i in range(self.Grid.count()): self.Grid.itemAt(i).widget().setDisabled(flag) for i in range(self.Main_window.Grid.count()): self.Main_window.Grid.itemAt(i).widget().setDisabled(flag)
def download(self): try: self.d_size = long(self.info['Content-Length']) print "File size :", self.d_size except: print "Resume support not available." print "Exiting..." sys.exit(1) try: if self.info['Accept-Ranges'] != 'bytes': print 'It seems that server does not support byte ranges.I don\'t know how to handle this, so I am EXITING. If you know what can be done then please inform author at itrishansh[at]gmail[dot]com.' sys.exit(1) except: print "Server doesn\'t support ranges." self.parts = [] end = 0 while (end + self.part_size) < self.d_size: self.parts.append((str(end), str(end + self.part_size - 1))) #print (str(end), str(end + self.part_size -1)) end = end + self.part_size #print end if end < self.d_size: self.parts.append((str(end), '')) self.nparts = len(self.parts) #print self.parts queueLock = threading.Lock() workQueue = Queue.Queue(len(self.parts)) threads = [] threadID = 1 # Create new threads while threadID <= self.n_threads: thread = DownloadThread(threadID, 'Thread-' + str(threadID), self.name, workQueue, queueLock, self.exit, self.url) thread.start() threads.append(thread) threadID += 1 # Fill the queue queueLock.acquire() i = 0 for part in self.parts: workQueue.put(part + (i, )) i += 1 queueLock.release() del i # Wait for queue to empty while not workQueue.empty(): time.sleep(1) # Notify threads it's time to exit #print "Modifying exitFlag" self.exitFlag = 1 # Wait for all threads to complete for t in threads: t.join() print 'Merging downloaded parts' self.merge() print "Exiting Main Thread"
def start_download_img(url, file_name): t = DownloadThread(arg=(url, file_name)) t.setDaemon(True) t.start()