コード例 #1
0
    def queryNodes(self, indexes=None, apimodule=False, options=None):
        if not (self.mainWindow.tree.selectedCount()
                or self.mainWindow.allnodesCheckbox.isChecked() or
                (indexes is not None)):
            return False

        #Show progress window
        progress = ProgressBar("Fetching Data", parent=self.mainWindow)

        try:
            apimodule, options = self.getQueryOptions(apimodule, options)
            indexes = self.getIndexes(options, indexes, progress)

            # Update progress window
            self.mainWindow.logmessage("Start fetching data.")
            totalnodes = 0
            hasindexes = True
            progress.setMaximum(totalnodes)
            self.mainWindow.tree.treemodel.nodecounter = 0

            #Init status messages
            statuscount = defaultdict(int)
            errorcount = 0
            ratelimitcount = 0
            allowedstatus = [
                'fetched (200)', 'downloaded (200)', 'fetched (202)'
            ]

            try:
                #Spawn Threadpool
                threadpool = ApiThreadPool(apimodule)
                threadpool.spawnThreads(options.get("threads", 1))

                #Process Logging/Input/Output Queue
                while True:
                    try:
                        #Logging (sync logs in threads with main thread)
                        msg = threadpool.getLogMessage()
                        if msg is not None:
                            self.mainWindow.logmessage(msg)

                        # Jobs in: packages of 100 at a time
                        jobsin = 0
                        while hasindexes and (jobsin < 100):
                            index = next(indexes, False)
                            if index:
                                jobsin += 1
                                totalnodes += 1
                                if index.isValid():
                                    job = self.prepareJob(index, options)
                                    threadpool.addJob(job)
                            else:
                                threadpool.applyJobs()
                                progress.setRemaining(threadpool.getJobCount())
                                progress.resetRate()
                                hasindexes = False
                                progress.removeInfo('input')
                                self.mainWindow.logmessage(
                                    "Added {} node(s) to queue.".format(
                                        totalnodes))

                        if jobsin > 0:
                            progress.setMaximum(totalnodes)

                        #Jobs out
                        job = threadpool.getJob()

                        #-Finished all nodes (sentinel)...
                        if job is None:
                            break

                        #-Finished one node...
                        elif 'progress' in job:
                            progresskey = 'nodeprogress' + str(
                                job.get('threadnumber', ''))

                            # Update single progress
                            if 'current' in job:
                                percent = int((job.get('current', 0) * 100.0 /
                                               job.get('total', 1)))
                                progress.showInfo(
                                    progresskey,
                                    "{}% of current node processed.".format(
                                        percent))
                            elif 'page' in job:
                                if job.get('page', 0) > 1:
                                    progress.showInfo(
                                        progresskey,
                                        "{} page(s) of current node processed."
                                        .format(job.get('page', 0)))

                            # Update total progress
                            else:
                                progress.removeInfo(progresskey)
                                if not threadpool.suspended:
                                    progress.step()

                        #-Add data...
                        elif 'data' in job and (not progress.wasCanceled):
                            if not job['nodeindex'].isValid():
                                continue

                            # Add data
                            treeindex = job['nodeindex']
                            treenode = treeindex.internalPointer()

                            newcount = treenode.appendNodes(
                                job['data'], job['options'], True)
                            if options.get('expand', False):
                                self.mainWindow.tree.setExpanded(
                                    treeindex, True)

                            # Count status and errors
                            status = job['options'].get('querystatus', 'empty')
                            statuscount[status] += 1
                            errorcount += int(not status in allowedstatus)

                            # Detect rate limit
                            ratelimit = job['options'].get('ratelimit', False)
                            #ratelimit = ratelimit or (not newcount)
                            ratelimitcount += int(ratelimit)
                            autoretry = (ratelimitcount) or (
                                status == "request error")

                            # Clear errors when everything is ok
                            if not threadpool.suspended and (
                                    status
                                    in allowedstatus) and (not ratelimit):
                                #threadpool.clearRetry()
                                errorcount = 0
                                ratelimitcount = 0

                            # Suspend on error or ratelimit
                            elif (errorcount >=
                                  options['errors']) or (ratelimitcount > 0):
                                threadpool.suspendJobs()

                                if ratelimit:
                                    msg = "You reached the rate limit of the API."
                                else:
                                    msg = "{} consecutive errors occurred.\nPlease check your settings.".format(
                                        errorcount)

                                timeout = 60 * 5  # 5 minutes

                                # Adjust progress
                                progress.showError(msg, timeout, autoretry)
                                self.mainWindow.tree.treemodel.commitNewNodes()

                            # Add job for retry
                            if not status in allowedstatus:
                                threadpool.addError(job)

                            # Show info
                            progress.showInfo(
                                status,
                                "{} response(s) with status: {}".format(
                                    statuscount[status], status))
                            progress.showInfo(
                                'newnodes', "{} new node(s) created".format(
                                    self.mainWindow.tree.treemodel.nodecounter)
                            )
                            progress.showInfo(
                                'threads', "{} active thread(s)".format(
                                    threadpool.getThreadCount()))
                            progress.setRemaining(threadpool.getJobCount())

                            # Custom info from modules
                            info = job['options'].get('info', {})
                            for name, value in info.items():
                                progress.showInfo(name, value)

                        # Abort
                        elif progress.wasCanceled:
                            progress.showInfo(
                                'cancel',
                                "Disconnecting from stream, may take some time."
                            )
                            threadpool.stopJobs()

                        # Retry
                        elif progress.wasResumed:
                            if progress.wasRetried:
                                threadpool.retryJobs()
                            else:
                                threadpool.clearRetry()
                                # errorcount = 0
                                # ratelimitcount = 0
                                threadpool.resumeJobs()

                            progress.setRemaining(threadpool.getJobCount())
                            progress.hideError()

                        # Continue
                        elif not threadpool.suspended:
                            threadpool.resumeJobs()

                        # Finished with pending errors
                        if not threadpool.hasJobs(
                        ) and threadpool.hasErrorJobs():
                            msg = "All nodes finished but you have {} pending errors. Skip or retry?".format(
                                threadpool.getErrorJobsCount())
                            autoretry = False
                            timeout = 60 * 5  # 5 minutes
                            progress.showError(msg, timeout, autoretry)

                        # Finished
                        if not threadpool.hasJobs():
                            progress.showInfo(
                                'cancel',
                                "Work finished, shutting down threads.")
                            threadpool.stopJobs()

                        #-Waiting...
                        progress.computeRate()
                        time.sleep(1.0 / 1000.0)
                    finally:
                        QApplication.processEvents()

            finally:
                request_summary = [
                    str(val) + " x " + key for key, val in statuscount.items()
                ]
                request_summary = ", ".join(request_summary)
                request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'

                self.mainWindow.logmessage(
                    "{}, {} new node(s) created. Summary of responses: {}.".
                    format(request_end,
                           self.mainWindow.tree.treemodel.nodecounter,
                           request_summary))

                self.mainWindow.tree.treemodel.commitNewNodes()
        except Exception as e:
            self.mainWindow.logmessage(
                "Error in scheduler, fetching aborted: {}.".format(str(e)))
        finally:
            progress.close()
            return not progress.wasCanceled
コード例 #2
0
    def queryNodes(self, indexes=None, apimodule=False, options=False):
        if not (self.mainWindow.tree.selectedCount()
                or self.mainWindow.allnodesCheckbox.isChecked() or
                (indexes is not None)):
            return (False)

        #Show progress window
        progress = ProgressBar("Fetching Data", parent=self.mainWindow)

        try:
            #Get global options
            globaloptions = {}
            globaloptions['threads'] = self.mainWindow.threadsEdit.value()
            globaloptions['speed'] = self.mainWindow.speedEdit.value()
            globaloptions['errors'] = self.mainWindow.errorEdit.value()
            globaloptions[
                'expand'] = self.mainWindow.autoexpandCheckbox.isChecked()
            globaloptions[
                'logrequests'] = self.mainWindow.logCheckbox.isChecked()
            globaloptions[
                'saveheaders'] = self.mainWindow.headersCheckbox.isChecked()
            globaloptions[
                'allnodes'] = self.mainWindow.allnodesCheckbox.isChecked()
            objecttypes = self.mainWindow.typesEdit.text().replace(
                ' ', '').split(',')
            level = self.mainWindow.levelEdit.value() - 1

            #Get selected nodes
            if indexes is None:
                select_all = globaloptions['allnodes']
                select_filter = {'level': level, 'objecttype': objecttypes}
                indexes = self.mainWindow.tree.selectedIndexesAndChildren(
                    False, select_filter, select_all)
            elif isinstance(indexes, list):
                indexes = iter(indexes)

            # if (len(indexes) == 0):
            #     return (False)

            #Update progress window
            #self.mainWindow.logmessage("Start fetching data for {} node(s).".format(len(indexes)))
            self.mainWindow.logmessage("Start fetching data.")
            totalnodes = 0
            hasindexes = True
            progress.setMaximum(totalnodes)
            self.mainWindow.tree.treemodel.nodecounter = 0

            #Init status messages
            statuscount = {}
            errorcount = 0
            ratelimitcount = 0
            allowedstatus = [
                'fetched (200)', 'downloaded (200)', 'fetched (202)', 'stream'
            ]  #,'error (400)'

            if apimodule == False:
                apimodule = self.mainWindow.RequestTabs.currentWidget()
            if options == False:
                options = apimodule.getOptions()

            options.update(globaloptions)

            try:
                #Spawn Threadpool
                threadpool = ApiThreadPool(apimodule)
                threadpool.spawnThreads(options.get("threads", 1))

                #Init input Queue
                #indexes = deque(list(indexes))

                #Process Logging/Input/Output Queue
                while True:
                    try:
                        #Logging (sync logs in threads with main thread)
                        msg = threadpool.getLogMessage()
                        if msg is not None:
                            self.mainWindow.logmessage(msg)

                        # Jobs in: packages of 100 at a time
                        jobsin = 0
                        while (hasindexes and (jobsin < 100)):

                            index = next(indexes, False)
                            if index:
                                jobsin += 1
                                totalnodes += 1
                                if index.isValid():
                                    treenode = index.internalPointer()
                                    job = {
                                        'nodeindex': index,
                                        'nodedata': deepcopy(treenode.data),
                                        'options': deepcopy(options)
                                    }
                                    threadpool.addJob(job)
                            else:
                                threadpool.applyJobs()
                                progress.setRemaining(threadpool.getJobCount())
                                progress.resetRate()
                                hasindexes = False
                                self.mainWindow.logmessage(
                                    "Added {} node(s) to queue.".format(
                                        totalnodes))

                        if jobsin > 0:
                            progress.setMaximum(totalnodes)

                        #Jobs out
                        job = threadpool.getJob()

                        #-Finished all nodes (sentinel)...
                        if job is None:
                            break

                        #-Finished one node...
                        elif 'progress' in job:
                            progresskey = 'nodeprogress' + str(
                                job.get('threadnumber', ''))

                            # Update single progress
                            if 'current' in job:
                                percent = int((job.get('current', 0) * 100.0 /
                                               job.get('total', 1)))
                                progress.showInfo(
                                    progresskey,
                                    "{}% of current node processed.".format(
                                        percent))
                            elif 'page' in job:
                                if job.get('page', 0) > 1:
                                    progress.showInfo(
                                        progresskey,
                                        "{} page(s) of current node processed."
                                        .format(job.get('page', 0)))

                            # Update total progress
                            else:
                                progress.removeInfo(progresskey)
                                if not threadpool.suspended:
                                    progress.step()

                        #-Add data...
                        elif 'data' in job and (not progress.wasCanceled):
                            if not job['nodeindex'].isValid():
                                continue

                            # Add data
                            treeindex = job['nodeindex']
                            treenode = treeindex.internalPointer()
                            treenode.appendNodes(job['data'], job['options'],
                                                 job['headers'], True)
                            if options.get('expand', False):
                                self.mainWindow.tree.setExpanded(
                                    treeindex, True)

                            # Count status
                            status = job['options'].get('querystatus', 'empty')

                            if not status in statuscount:
                                statuscount[status] = 1
                            else:
                                statuscount[status] = statuscount[status] + 1

                            # Collect errors for automatic retry
                            if not status in allowedstatus:
                                threadpool.addError(job)
                                errorcount += 1

                            # Detect rate limit
                            ratelimit = job['options'].get('ratelimit', False)

                            if ratelimit:
                                ratelimitcount += 1

                            # Clear errors
                            if not threadpool.suspended and (
                                    status in allowedstatus) and not ratelimit:
                                threadpool.clearRetry()
                                errorcount = 0
                                ratelimitcount = 0

                            # Suspend on error
                            elif (errorcount >
                                  (globaloptions['errors'] - 1)) or (
                                      ratelimitcount > 0):
                                threadpool.suspendJobs()

                                if ratelimit:
                                    msg = "You reached the rate limit of the API."
                                else:
                                    msg = "{} consecutive errors occurred.\nPlease check your settings.".format(
                                        errorcount)

                                timeout = 60 * 5  #5 minutes

                                # Adjust progress
                                progress.setRemaining(
                                    threadpool.getJobCount() +
                                    threadpool.getRetryCount())
                                progress.showError(msg, timeout,
                                                   ratelimitcount > 0)
                                self.mainWindow.tree.treemodel.commitNewNodes()

                            # Show info
                            progress.showInfo(
                                status,
                                "{} response(s) with status: {}".format(
                                    statuscount[status], status))
                            progress.showInfo(
                                'newnodes', "{} new node(s) created".format(
                                    self.mainWindow.tree.treemodel.nodecounter)
                            )
                            progress.showInfo(
                                'threads', "{} active thread(s)".format(
                                    threadpool.getThreadCount()))
                            progress.setRemaining(threadpool.getJobCount())

                            # Custom info from modules
                            info = job['options'].get('info', {})
                            for name, value in info.items():
                                progress.showInfo(name, value)

                        # Abort
                        elif progress.wasCanceled:
                            progress.showInfo(
                                'cancel',
                                "Disconnecting from stream, may take some time."
                            )
                            threadpool.stopJobs()

                        # Retry
                        elif progress.wasResumed:
                            if progress.wasRetried:
                                threadpool.retryJobs()
                            else:
                                threadpool.clearRetry()
                                errorcount = 0
                                ratelimitcount = 0
                                threadpool.resumeJobs()

                            progress.setRemaining(threadpool.getJobCount())
                            progress.hideError()

                        # Continue
                        elif not threadpool.suspended:
                            threadpool.resumeJobs()

                        # Finished
                        if not threadpool.hasJobs():
                            progress.showInfo(
                                'cancel',
                                "Work finished, shutting down threads.")
                            threadpool.stopJobs()

                        #-Waiting...
                        progress.computeRate()
                        time.sleep(1.0 / 1000.0)
                    finally:
                        QApplication.processEvents()

            finally:
                request_summary = [
                    str(val) + " x " + key for key, val in statuscount.items()
                ]
                request_summary = ", ".join(request_summary)
                request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'

                self.mainWindow.logmessage(
                    "{}, {} new node(s) created. Summary of responses: {}.".
                    format(request_end,
                           self.mainWindow.tree.treemodel.nodecounter,
                           request_summary))

                self.mainWindow.tree.treemodel.commitNewNodes()
        finally:
            progress.close()
コード例 #3
0
ファイル: actions.py プロジェクト: strohne/Facepager
    def queryNodes(self, indexes=None, apimodule=False, options=False):
        if not self.actionQuery.isEnabled() or not ((self.mainWindow.tree.selectedCount() > 0) or (indexes is not None)):
            return (False)

        #Show progress window
        progress = ProgressBar("Fetching Data", parent=self.mainWindow)

        try:
            #Get global options
            globaloptions = {}
            globaloptions['threads'] = self.mainWindow.threadsEdit.value()
            globaloptions['speed'] = self.mainWindow.speedEdit.value()
            globaloptions['errors'] = self.mainWindow.errorEdit.value()
            globaloptions['expand'] = self.mainWindow.autoexpandCheckbox.isChecked()
            globaloptions['logrequests'] = self.mainWindow.logCheckbox.isChecked()
            globaloptions['saveheaders'] = self.mainWindow.headersCheckbox.isChecked()
            objecttypes = self.mainWindow.typesEdit.text().replace(' ','').split(',')
            level = self.mainWindow.levelEdit.value() - 1

            #Get selected nodes
            if indexes is None:
                indexes = self.mainWindow.tree.selectedIndexesAndChildren(False, {'level': level,
                                                                                  'objecttype':objecttypes})

            if (len(indexes) == 0):
                return (False)

            #Update progress window
            self.mainWindow.logmessage("Start fetching data for {} node(s).".format(len(indexes)))
            progress.setMaximum(len(indexes))
            self.mainWindow.tree.treemodel.nodecounter = 0

            #Init status messages
            statuscount = {}
            errorcount = 0
            ratelimitcount = 0
            allowedstatus = ['fetched (200)','downloaded (200)','fetched (202)','stream'] #,'error (400)'


            if apimodule == False:
                apimodule = self.mainWindow.RequestTabs.currentWidget()
            if options == False:
                options = apimodule.getOptions()

            options.update(globaloptions)

            try:
                #Spawn Threadpool
                threadpool = ApiThreadPool(apimodule)
                threadpool.spawnThreads(options.get("threads", 1))

                #Init input Queue
                indexes = deque(indexes)


                #Process Logging/Input/Output Queue
                while True:
                    try:
                        #Logging (sync logs in threads with main thread)
                        msg = threadpool.getLogMessage()
                        if msg is not None:
                            self.mainWindow.logmessage(msg)

                        #Jobs in
                        if (len(indexes) > 0):
                            index = indexes.popleft()
                            if index.isValid():
                                treenode = index.internalPointer()
                                job = {'nodeindex': index,
                                       'nodedata': deepcopy(treenode.data),
                                       'options': deepcopy(options)}
                                threadpool.addJob(job)

                            if len(indexes) == 0:
                                threadpool.applyJobs()
                                progress.setRemaining(threadpool.getJobCount())
                                progress.resetRate()

                        #Jobs out
                        job = threadpool.getJob()

                        #-Finished all nodes (sentinel)...
                        if job is None:
                            break

                        #-Finished one node...
                        elif 'progress' in job:
                            progresskey = 'nodeprogress' + str(job.get('threadnumber', ''))

                            # Update single progress
                            if 'current' in job:
                                percent = int((job.get('current',0) * 100.0 / job.get('total',1))) 
                                progress.showInfo(progresskey, "{}% of current node processed.".format(percent))
                            elif 'page' in job:
                                if job.get('page', 0) > 1:
                                    progress.showInfo(progresskey, "{} page(s) of current node processed.".format(job.get('page',0)))

                            # Update total progress
                            else:
                                progress.removeInfo(progresskey)
                                if not threadpool.suspended:
                                    progress.step()

                        #-Add data...
                        elif 'data' in job and (not progress.wasCanceled):
                            if not job['nodeindex'].isValid():
                                continue

                            # Add data
                            treeindex = job['nodeindex']
                            treenode = treeindex.internalPointer()
                            treenode.appendNodes(job['data'], job['options'], job['headers'], True)
                            if options.get('expand',False):
                                 self.mainWindow.tree.setExpanded(treeindex,True)

                            # Count status
                            status = job['options'].get('querystatus', 'empty')

                            if not status in statuscount:
                                statuscount[status] = 1
                            else:
                                statuscount[status] = statuscount[status]+1

                            # Collect errors for automatic retry
                            if not status in allowedstatus:
                                threadpool.addError(job)
                                errorcount += 1

                            # Detect rate limit
                            ratelimit = job['options'].get('ratelimit', False)

                            if ratelimit:
                                ratelimitcount += 1

                            # Clear errors
                            if not threadpool.suspended and (status in allowedstatus) and not ratelimit:
                                threadpool.clearRetry()
                                errorcount = 0
                                ratelimitcount = 0


                            # Suspend on error
                            elif (errorcount > (globaloptions['errors']-1)) or (ratelimitcount > 0):
                                threadpool.suspendJobs()

                                if ratelimit:
                                    msg = "You reached the rate limit of the API."
                                else:
                                    msg = "{} consecutive errors occurred.\nPlease check your settings.".format(errorcount)

                                timeout = 60 * 5 #5 minutes

                                # Adjust progress
                                progress.setRemaining(threadpool.getJobCount() + threadpool.getRetryCount())
                                progress.showError(msg, timeout, ratelimitcount > 0)
                                self.mainWindow.tree.treemodel.commitNewNodes()

                            # Show info
                            progress.showInfo(status,"{} response(s) with status: {}".format(statuscount[status],status))
                            progress.showInfo('newnodes',"{} new node(s) created".format(self.mainWindow.tree.treemodel.nodecounter))
                            progress.showInfo('threads',"{} active thread(s)".format(threadpool.getThreadCount()))
                            progress.setRemaining(threadpool.getJobCount())

                            # Custom info from modules
                            info = job['options'].get('info', {})
                            for name, value in info.items():
                                progress.showInfo(name, value)

                        # Abort
                        elif progress.wasCanceled:
                            progress.showInfo('cancel', "Disconnecting from stream, may take some time.")
                            threadpool.stopJobs()

                        # Retry
                        elif progress.wasResumed:
                            if progress.wasRetried:
                                threadpool.retryJobs()
                            else:
                                threadpool.clearRetry()
                                threadpool.resumeJobs()

                            progress.setRemaining(threadpool.getJobCount())
                            progress.hideError()

                        # Continue
                        elif not threadpool.suspended:
                            threadpool.resumeJobs()

                        # Finished
                        if not threadpool.hasJobs():
                            progress.showInfo('cancel', "Work finished, shutting down threads.")
                            threadpool.stopJobs()

                        #-Waiting...
                        progress.computeRate()
                        time.sleep(1.0 / 1000.0)
                    finally:
                        QApplication.processEvents()

            finally:
                request_summary = [str(val)+" x "+key for key,val in statuscount.items()]
                request_summary = ", ".join(request_summary)
                request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'

                self.mainWindow.logmessage("{}, {} new node(s) created. Summary of responses: {}.".format(request_end, self.mainWindow.tree.treemodel.nodecounter,request_summary))

                self.mainWindow.tree.treemodel.commitNewNodes()
        finally:
            progress.close()