Пример #1
0
 def filling_list_of_letters(self):
     try:
         name_of_folder = self.Main_window.Folders_box.currentText()
     except Exception:
         name_of_folder = 'INBOX'
     list_of_folders = self.Imap.parse_list_of_folders()
     if len(list_of_folders) == 0:
         self.Grid.addWidget(
             QtWidgets.QLabel('Problems with decode folders'), 0, 0)
     for item in list_of_folders:
         if item[0] == name_of_folder:
             self.Imap.select_folder(item[1])
             break
     try:
         count_of_letters = int(self.Imap.get_count_of_letters())
     except Exception:
         self.Imap.select_folder('INBOX')
         print('не упал')
         count_of_letters = int(self.Imap.get_count_of_letters())
         print('упал')
     if count_of_letters == 0:
         self.Grid.addWidget(QtWidgets.QLabel('No letters'), 0, 0)
     else:
         trd = DownloadThread(self.Imap, count_of_letters)
         self.trd = trd
         trd.start()
         trd.finished.connect(self.finished)
         self.set_disabled(True)
Пример #2
0
 def start(self):
     if self.stopping:
         return
     self.thread_ = DownloadThread(self.url, self.target)
     self.thread_.completed.connect(self._on_completed)
     self.thread_.aborted.connect(self._on_aborted)
     self.thread_.failed.connect(self._on_failed)
     self.thread_.progressed.connect(self._on_progressed)
     self.thread_.start()
Пример #3
0
 def show_more_clicked(self):
     index = self.Index_of_current_letter
     count = int(self.Imap.get_count_of_letters()) - index - 1
     if count <= 0:
         return
     self.trd = DownloadThread(self.Imap, count)
     self.trd.start()
     self.trd.finished.connect(self.finished)
     self.set_disabled(True)
Пример #4
0
    def __downloadNextItem(self):
        with self.__lock:
            if self.__shouldStop == True:
                self.__shouldStop = False
                self.active = None

                notifName = DownloadThread.DONE_NOTIFICATION
                Notification.removeObserver(self, notifName)
                self.__notifyStopped()
                return

            if len(self.queue) == 0:
                self.active = None

                notifName = DownloadThread.DONE_NOTIFICATION
                Notification.removeObserver(self, notifName)
                self.__notifyDone()
                return

            # Start the download-thread
            notifName = DownloadThread.DONE_NOTIFICATION
            Notification.addObserver(self, notifName)

            notifName = DownloadThread.NEW_OUTPUT_NOTIFICATION
            Notification.addObserver(self, notifName)

            self.active = self.queue[0]
            del self.queue[0]
            self.__downloadThread = DownloadThread(self.active)
            self.__downloadThread.start()

            self.__notifyNextDownload()
Пример #5
0
class DownloadManager(object):
    XML_TAG = 'download-manager'
    DEFAULT_FILENAME = "download-manager.xml"

    # Posted when the download manager stops downloading because
    # the queue is empty.
    DONE_NOTIFICATION = "DownloadManagerDoneNotification"

    # Posted when the download manager stopped after a call to the
    # stop method. Note that this notification is NOT posted when
    # the manager stops due to an empty queue.
    STOPPED_NOTIFICATION = "DownloadManagerStoppedNotification"

    # Posted when the downloading of a new object has commenced.
    NEXT_DOWNLOAD_NOTIFICATION = "DownloadManagerNextDownloadNotification"

    # Posted when new output in self.output is available.
    NEW_OUTPUT_NOTIFICATION = "DownloadManagerNewOutputNotification"
    
    def __init__(self, filename=DEFAULT_FILENAME):
        self.queue = [] # Collection of DownloadConfiguration instances
        self.done = [] # Cellection of ... instances
        self.active = None # DownloadConfiguration instance or None

        # List of tuples (text, type) where text is a string and type
        # is 0 for stdout and 1 for stderr.
        self.output = []

        self.__downloadThread = None
        self.__shouldStop = False

        self.filename = filename

        self.__lock = RLock()

    # Saving and loading
    def synchronize(self):
        """Writes and saves the download status."""
        with self.__lock:
            tb = ET.TreeBuilder()
            tb.start(self.XML_TAG)

            # Queue
            tb.start('queue')
            for dc in self.queue:
                dc.addToTreeBuilder(tb)
            if self.active != None:
                self.active.addToTreeBuilder(tb)
            tb.end('queue')

            # Done
            tb.start('done')
            for dc in self.done:
                dc.addToTreeBuilder(tb)
            tb.end('done')

            root = tb.close()
            tree = ET.ElementTree(root)
            tree.write(self.filename)
            
            
    def load(self):
        """Loads the download status from the file."""
        with self.__lock:
            doc = MD.parse(self.filename)
            root = doc.documentElement
            if root.tagName != self.XML_TAG:
                raise UnsupportedFormat()

            cn = root.childNodes
            for c in cn:
                if c.nodeType != MD.Node.ELEMENT_NODE:
                    continue
                if c.tagName == 'queue':
                    self.__parseQueue(c)
                if c.tagName == 'done':
                    self.__parseDone(c)


    def __parseQueue(self, queueElem):
        cn = queueElem.childNodes
        for c in cn:
            if c.nodeType != MD.Node.ELEMENT_NODE:
                continue
            if c.tagName == DownloadConfiguration.XML_TAG:
                dc = DownloadConfiguration.fromXMLElement(c)
                if dc != None:
                    self.queue.append(dc)

    def __parseDone(self, doneElem):
        cn = doneElem.childNodes
        for c in cn:
            if c.nodeType != MD.Node.ELEMENT_NODE:
                continue
            if c.tagName == DownloadConfiguration.XML_TAG:
                dc = DownloadConfiguration.fromXMLElement(c)
                if dc != None:
                    self.done.append(dc)

    # Managing queue
    def addToQueue(self, dc):
        """Adds a download configuration to the queue."""
        with self.__lock:
            self.queue.append(dc)
            self.synchronize()

    def removeFromQueue(self, index, alertParent=None):
        """Removes a download configuration from the queue.

        index - The index of the download configuration
        alertParent - The parent of the confirmation alert if you
           want to show one to the user. If you want to remove the
           download configuration without confiromation, pass None.
        """
        with self.__lock:
            if index >= len(self.queue):
                return

            if alertParent != None:
                self.__showRemovalConfirmation(index, alertParent)
            else:
                dc = self.queue[index]
                self.__removalConfiguration = dc
                self.__continueRemoval()

    def clearDone(self):
        """Removes all configurations from the done list."""
        with self.__lock:
            self.done.clear()
            self.synchronize()

    def __showRemovalConfirmation(self, index, alertParent):
        with self.__lock:
            if index >= len(self.queue):
                return

            dc = self.queue[index]
            self.__removalConfiguration = dc

            title = "Are you sure you want to delete the selected "
            title+= "download configuration?"
            msg = "This operation cannot be undone."

            a = MessageAlert(alertParent, title, msg)
            b = Button("OK", self.__continueRemoval, Button.SHORTCUT_ENTER)
            a.addButton(b)
            b = Button("Cancel", self.__endRemovalConfirmation, 'c')
            a.addButton(b)
            
            self.__removalAlert = a
            alertParent.beginModalScreen(a)

    def __endRemovalConfirmation(self):
        self.__removalAlert.parent.endModalScreen(self.__removalAlert)
        self.__removalAlert = None

    def __continueRemoval(self):
        with self.__lock:
            try:
                self.queue.remove(self.__removalConfiguration)
                self.synchronize()
            except ValueError:
                pass
            
        self.__removalAlert.parent.endModalScreen(self.__removalAlert)
        self.__removalAlert = None


    # Downloading Objects
    def isDownloading(self):
        with self.__lock:
            return self.active != None

    def startDownloading(self):
        with self.__lock:
            self.__shouldStop = False
            if self.active != None:
                return

            self.__downloadNextItem()

    def stopDownloading(self):
        with self.__lock:
            self.__shouldStop = True

    def __downloadNextItem(self):
        with self.__lock:
            if self.__shouldStop == True:
                self.__shouldStop = False
                self.active = None

                notifName = DownloadThread.DONE_NOTIFICATION
                Notification.removeObserver(self, notifName)
                self.__notifyStopped()
                return

            if len(self.queue) == 0:
                self.active = None

                notifName = DownloadThread.DONE_NOTIFICATION
                Notification.removeObserver(self, notifName)
                self.__notifyDone()
                return

            # Start the download-thread
            notifName = DownloadThread.DONE_NOTIFICATION
            Notification.addObserver(self, notifName)

            notifName = DownloadThread.NEW_OUTPUT_NOTIFICATION
            Notification.addObserver(self, notifName)

            self.active = self.queue[0]
            del self.queue[0]
            self.__downloadThread = DownloadThread(self.active)
            self.__downloadThread.start()

            self.__notifyNextDownload()

    def __handleDownloadThreadDoneNotification(self, notif):
        if notif.sender != self.__downloadThread:
            return

        if self.active != None:
            with self.__lock:
                self.done.append(self.active)
                self.active = None
                self.synchronize()

        self.__downloadThread = None
        self.__downloadNextItem()

    def __handleDownloadThreadNewOutputNotification(self, notif):
        if notif.sender != self.__downloadThread:
            return

        self.output = notif.sender.output
        self.__notifyNewOutput()

    # Notification
    def __notifyDone(self):
        n = Notification(self.DONE_NOTIFICATION, self)
        Notification.post(n)

    def __notifyStopped(self):
        n = Notification(self.STOPPED_NOTIFICATION, self)
        Notification.post(n)

    def __notifyNextDownload(self):
        n = Notification(self.NEXT_DOWNLOAD_NOTIFICATION, self)
        Notification.post(n)

    def __notifyNewOutput(self):
        n = Notification(self.NEW_OUTPUT_NOTIFICATION, self)
        Notification.post(n)
        
    def handleNotification(self, notif):
        if notif.name == DownloadThread.DONE_NOTIFICATION:
            self.__handleDownloadThreadDoneNotification(notif)

        if notif.name == DownloadThread.NEW_OUTPUT_NOTIFICATION:
            self.__handleDownloadThreadNewOutputNotification(notif)
Пример #6
0
    def createThread(self):
        """create a download thread"""

        thread = DownloadThread(self)
        self.threads.append(thread)
Пример #7
0
class DownloadTask(QObject):
    status_changed = pyqtSignal(str)
    aborted = pyqtSignal()

    def __init__(self, url=None, target=None):
        super().__init__()
        self.url = url if url is not None else default_url
        self.target = target if target is not None else target_prefix + self._available_target_for(
            url)
        self.status = 'Waiting'
        self.thread_ = None
        self.stopping = False

    def __str__(self):
        return 'Download task [URL = \'{}\', target = \'{}\', status = \'{}\']'.format(
            self.url, self.target, self.status)

    def start(self):
        if self.stopping:
            return
        self.thread_ = DownloadThread(self.url, self.target)
        self.thread_.completed.connect(self._on_completed)
        self.thread_.aborted.connect(self._on_aborted)
        self.thread_.failed.connect(self._on_failed)
        self.thread_.progressed.connect(self._on_progressed)
        self.thread_.start()

    def stop(self):
        if self.stopping:
            return
        self.stopping = True
        if self.thread_.isRunning():
            self.thread_.to_abort = True
        else:
            self._on_aborted(False)

    def _on_completed(self):
        self._change_status('Completed')

    def _on_aborted(self, need_removal):
        if need_removal:
            self._remove_target()
        self._change_status('Aborted')
        self.stopping = False
        self.aborted.emit()

    def _on_failed(self, err_str):
        if err_str:
            self._change_status('Failed ({})'.format(err_str))
        else:
            self._change_status('Failed')
        self._remove_target()

    def _on_progressed(self, progress):
        self._change_status('{:.2f}%'.format(progress * 100))

    def _available_target_for(self, url):
        res = url[url.rfind('/') + len('/'):]
        separator_pos = res.rfind('.')
        if separator_pos != -1:
            filename, extension = res[:separator_pos], res[separator_pos +
                                                           len('.'):]
        else:
            filename, extension = res, ''
        n = 0
        while os.path.exists(res):
            n += 1
            res = self._make_res(filename, extension, n)
        return res

    def _make_res(self, filename, extension, n):
        if n:
            return filename + ' (' + str(n) + ').' + extension
        return filename + '.' + extension

    def _change_status(self, status):
        self.status = status
        self.status_changed.emit(status)

    def _remove_target(self):
        if os.path.exists(self.target):
            os.remove(self.target)
Пример #8
0
 def startDownload(self, downloadInfo):
     """ 通过DonwloadInfo模型中的数据进行下载 """
     name = downloadInfo.title + ".torrent"
     downloadThread = DownloadThread(downloadInfo = downloadInfo, name = name, sem = self.sem)
     downloadThread.start()
Пример #9
0
class ListOfLettersWidget(QtWidgets.QWidget):
    def __init__(self, imap, main_window):
        super().__init__()
        self.Imap = imap
        self.Main_window = main_window
        self.List_of_marked = []
        self.Index_of_current_letter = 0
        self.initUI()

    def initUI(self):
        grid = QtWidgets.QGridLayout()
        self.setLayout(grid)
        self.Grid = grid
        self.filling_list_of_letters()
        self.Main_window.show_more_button.clicked.connect(
            self.show_more_clicked)

    def show_more_clicked(self):
        index = self.Index_of_current_letter
        count = int(self.Imap.get_count_of_letters()) - index - 1
        if count <= 0:
            return
        self.trd = DownloadThread(self.Imap, count)
        self.trd.start()
        self.trd.finished.connect(self.finished)
        self.set_disabled(True)

    def filling_list_of_letters(self):
        try:
            name_of_folder = self.Main_window.Folders_box.currentText()
        except Exception:
            name_of_folder = 'INBOX'
        list_of_folders = self.Imap.parse_list_of_folders()
        if len(list_of_folders) == 0:
            self.Grid.addWidget(
                QtWidgets.QLabel('Problems with decode folders'), 0, 0)
        for item in list_of_folders:
            if item[0] == name_of_folder:
                self.Imap.select_folder(item[1])
                break
        try:
            count_of_letters = int(self.Imap.get_count_of_letters())
        except Exception:
            self.Imap.select_folder('INBOX')
            print('не упал')
            count_of_letters = int(self.Imap.get_count_of_letters())
            print('упал')
        if count_of_letters == 0:
            self.Grid.addWidget(QtWidgets.QLabel('No letters'), 0, 0)
        else:
            trd = DownloadThread(self.Imap, count_of_letters)
            self.trd = trd
            trd.start()
            trd.finished.connect(self.finished)
            self.set_disabled(True)

    def finished(self):
        result = self.trd.result.copy()
        self.set_disabled(False)
        for i in range(len(result)):
            self.Grid.addWidget(LetterWidget(result[i], self),
                                i + self.Index_of_current_letter, 0)
        self.Index_of_current_letter += len(result)
        self.Main_window.autorization_window.hide()
        self.Main_window.show()

    def set_disabled(self, flag):
        for i in range(self.Grid.count()):
            self.Grid.itemAt(i).widget().setDisabled(flag)
        for i in range(self.Main_window.Grid.count()):
            self.Main_window.Grid.itemAt(i).widget().setDisabled(flag)
Пример #10
0
def main():
	'''登录微博'''
	paramDict = read_config()
	if not login(paramDict['username'], paramDict['password']):
		exit()

	'''与数据库建立连接和指针'''
	pool = PooledDB(MySQLdb, int(paramDict['threadnum']),  host = paramDict['dbhost'], user = paramDict['dbuser'], passwd = paramDict['dbpasswd'], db = paramDict['dbname'])
	conn = pool.connection()
	cur = conn.cursor()

	'''读取未爬取的链接列表放入队列'''
	urlQLock = threading.Lock()
	tableName = 'users'
	sql = 'select id, uid from %s where isCrawled = 0' % tableName
	cur.execute(sql)
	result = cur.fetchall()
	urlQ = Queue(len(result))
	for entry in result:
		urlQ.put(entry)

	'''建立线程'''
	for i in xrange(int(paramDict['threadnum'])):
		thr = DownloadThread(pool, urlQ, urlQLock)
		threadPool.append(thr)
		thr.start()
	 
	'''检查是否存在结束的线程,若有,则重新建立新的线程'''
	while True:
		try:
			sleep(60)
			'''当队列为空时,跳出循环'''
			if not urlQ.qsize():
				break
			if threading.activeCount() < int(paramDict['threadnum']) + 1:
				'''检查哪个线程已经结束,将其清除'''
				i = 0
				for thr in threadPool:
					if not thr.isAlive():
						thr.clear()
						del threadPool[i]
						newThr = DownloadThread(pool, urlQ, urlQLock)
						threadPool.append(newThr)
						newThr.start()
					else:
						i += 1
		except:
			print sys.exc_info()[0]
			for thr in threadPool:
				thr.end()
			break
	print 'Main thread end!'
Пример #11
0
    def download(self):
        try:
            self.d_size = long(self.info['Content-Length'])
            print "File size :", self.d_size
        except:
            print "Resume support not available."
            print "Exiting..."
            sys.exit(1)

        try:
            if self.info['Accept-Ranges'] != 'bytes':
                print 'It seems that server does not support byte ranges.I don\'t know how to handle this, so I am EXITING. If you know what can be done then please inform author at itrishansh[at]gmail[dot]com.'
                sys.exit(1)
        except:
            print "Server doesn\'t support ranges."
        self.parts = []
        end = 0
        while (end + self.part_size) < self.d_size:
            self.parts.append((str(end), str(end + self.part_size - 1)))
            #print (str(end), str(end + self.part_size -1))
            end = end + self.part_size
            #print end
        if end < self.d_size:
            self.parts.append((str(end), ''))
        self.nparts = len(self.parts)

        #print self.parts
        queueLock = threading.Lock()
        workQueue = Queue.Queue(len(self.parts))
        threads = []
        threadID = 1

        # Create new threads
        while threadID <= self.n_threads:
            thread = DownloadThread(threadID, 'Thread-' + str(threadID),
                                    self.name, workQueue, queueLock, self.exit,
                                    self.url)
            thread.start()
            threads.append(thread)
            threadID += 1

        # Fill the queue
        queueLock.acquire()
        i = 0
        for part in self.parts:
            workQueue.put(part + (i, ))
            i += 1
        queueLock.release()
        del i

        # Wait for queue to empty
        while not workQueue.empty():
            time.sleep(1)

        # Notify threads it's time to exit
        #print "Modifying exitFlag"
        self.exitFlag = 1

        # Wait for all threads to complete
        for t in threads:
            t.join()

        print 'Merging downloaded parts'
        self.merge()

        print "Exiting Main Thread"
Пример #12
0
def start_download_img(url, file_name):
    t = DownloadThread(arg=(url, file_name))
    t.setDaemon(True)
    t.start()