def queryNodes(self, indexes=None, apimodule=False, options=None):
if not (self.mainWindow.tree.selectedCount()
or self.mainWindow.allnodesCheckbox.isChecked() or
(indexes is not None)):
return False
#Show progress window
progress = ProgressBar("Fetching Data", parent=self.mainWindow)
try:
# Get global options
globaloptions = {}
globaloptions['threads'] = self.mainWindow.threadsEdit.value()
globaloptions['speed'] = self.mainWindow.speedEdit.value()
globaloptions['errors'] = self.mainWindow.errorEdit.value()
globaloptions[
'expand'] = self.mainWindow.autoexpandCheckbox.isChecked()
globaloptions[
'logrequests'] = self.mainWindow.logCheckbox.isChecked()
globaloptions[
'saveheaders'] = self.mainWindow.headersCheckbox.isChecked()
globaloptions[
'allnodes'] = self.mainWindow.allnodesCheckbox.isChecked()
globaloptions['resume'] = self.mainWindow.resumeCheckbox.isChecked(
)
# Get module option
if isinstance(apimodule, str):
apimodule = self.mainWindow.getModule(apimodule)
if apimodule == False:
apimodule = self.mainWindow.RequestTabs.currentWidget()
apimodule.getProxies(True)
if options is None:
options = apimodule.getOptions()
else:
options = options.copy()
options.update(globaloptions)
#Get selected nodes
if indexes is None:
objecttypes = self.mainWindow.typesEdit.text().replace(
' ', '').split(',')
level = self.mainWindow.levelEdit.value() - 1
select_all = globaloptions['allnodes']
select_filter = {'level': level, '!objecttype': objecttypes}
self.progressUpdate = datetime.now()
def updateProgress(current, total, level=0):
if datetime.now() >= self.progressUpdate:
progress.showInfo(
'input', "Adding nodes to queue ({}/{}).".format(
current, total))
QApplication.processEvents()
self.progressUpdate = datetime.now() + timedelta(
milliseconds=60)
return not progress.wasCanceled
conditions = {
'filter': select_filter,
'selectall': select_all,
'options': options
}
indexes = self.mainWindow.tree.selectedIndexesAndChildren(
conditions, updateProgress)
elif isinstance(indexes, list):
indexes = iter(indexes)
# Update progress window
self.mainWindow.logmessage("Start fetching data.")
totalnodes = 0
hasindexes = True
progress.setMaximum(totalnodes)
self.mainWindow.tree.treemodel.nodecounter = 0
#Init status messages
statuscount = defaultdict(int)
errorcount = 0
ratelimitcount = 0
allowedstatus = [
'fetched (200)', 'downloaded (200)', 'fetched (202)'
]
try:
#Spawn Threadpool
threadpool = ApiThreadPool(apimodule)
threadpool.spawnThreads(options.get("threads", 1))
#Process Logging/Input/Output Queue
while True:
try:
#Logging (sync logs in threads with main thread)
msg = threadpool.getLogMessage()
if msg is not None:
self.mainWindow.logmessage(msg)
# Jobs in: packages of 100 at a time
jobsin = 0
while hasindexes and (jobsin < 100):
index = next(indexes, False)
if index:
jobsin += 1
totalnodes += 1
if index.isValid():
# Copy node data and options
treenode = index.internalPointer()
node_data = deepcopy(treenode.data)
node_options = deepcopy(options)
node_options[
'lastdata'] = treenode.lastdata if hasattr(
treenode, 'lastdata') else None
# Add job
job = {
'nodeindex': index,
'nodedata': node_data,
'options': node_options
}
threadpool.addJob(job)
else:
threadpool.applyJobs()
progress.setRemaining(threadpool.getJobCount())
progress.resetRate()
hasindexes = False
progress.removeInfo('input')
self.mainWindow.logmessage(
"Added {} node(s) to queue.".format(
totalnodes))
if jobsin > 0:
progress.setMaximum(totalnodes)
#Jobs out
job = threadpool.getJob()
#-Finished all nodes (sentinel)...
if job is None:
break
#-Finished one node...
elif 'progress' in job:
progresskey = 'nodeprogress' + str(
job.get('threadnumber', ''))
# Update single progress
if 'current' in job:
percent = int((job.get('current', 0) * 100.0 /
job.get('total', 1)))
progress.showInfo(
progresskey,
"{}% of current node processed.".format(
percent))
elif 'page' in job:
if job.get('page', 0) > 1:
progress.showInfo(
progresskey,
"{} page(s) of current node processed."
.format(job.get('page', 0)))
# Update total progress
else:
progress.removeInfo(progresskey)
if not threadpool.suspended:
progress.step()
#-Add data...
elif 'data' in job and (not progress.wasCanceled):
if not job['nodeindex'].isValid():
continue
# Add data
treeindex = job['nodeindex']
treenode = treeindex.internalPointer()
treenode.appendNodes(job['data'], job['options'],
job['headers'], True)
if options.get('expand', False):
self.mainWindow.tree.setExpanded(
treeindex, True)
# Count status
status = job['options'].get('querystatus', 'empty')
statuscount[status] += 1
# Collect errors for automatic retry
if not status in allowedstatus:
threadpool.addError(job)
errorcount += 1
# Detect rate limit
ratelimit = job['options'].get('ratelimit', False)
ratelimitcount += int(ratelimit)
autoretry = (ratelimitcount) or (
status == "request error")
# Clear errors when everything is ok
if not threadpool.suspended and (
status
in allowedstatus) and (not ratelimit):
#threadpool.clearRetry()
errorcount = 0
ratelimitcount = 0
# Suspend on error or ratelimit
elif (errorcount >= globaloptions['errors']) or (
ratelimitcount > 0):
threadpool.suspendJobs()
if ratelimit:
msg = "You reached the rate limit of the API."
else:
msg = "{} consecutive errors occurred.\nPlease check your settings.".format(
errorcount)
timeout = 60 * 5 # 5 minutes
# Adjust progress
progress.showError(msg, timeout, autoretry)
self.mainWindow.tree.treemodel.commitNewNodes()
# Show info
progress.showInfo(
status,
"{} response(s) with status: {}".format(
statuscount[status], status))
progress.showInfo(
'newnodes', "{} new node(s) created".format(
self.mainWindow.tree.treemodel.nodecounter)
)
progress.showInfo(
'threads', "{} active thread(s)".format(
threadpool.getThreadCount()))
progress.setRemaining(threadpool.getJobCount())
# Custom info from modules
info = job['options'].get('info', {})
for name, value in info.items():
progress.showInfo(name, value)
# Abort
elif progress.wasCanceled:
progress.showInfo(
'cancel',
"Disconnecting from stream, may take some time."
)
threadpool.stopJobs()
# Retry
elif progress.wasResumed:
if progress.wasRetried:
threadpool.retryJobs()
else:
threadpool.clearRetry()
# errorcount = 0
# ratelimitcount = 0
threadpool.resumeJobs()
progress.setRemaining(threadpool.getJobCount())
progress.hideError()
# Continue
elif not threadpool.suspended:
threadpool.resumeJobs()
# Finished with pending errors
if not threadpool.hasJobs(
) and threadpool.hasErrorJobs():
msg = "All nodes finished but you have {} pending errors. Skip or retry?".format(
threadpool.getErrorJobsCount())
autoretry = False
timeout = 60 * 5 # 5 minutes
progress.showError(msg, timeout, autoretry)
# Finished
if not threadpool.hasJobs():
progress.showInfo(
'cancel',
"Work finished, shutting down threads.")
threadpool.stopJobs()
#-Waiting...
progress.computeRate()
time.sleep(1.0 / 1000.0)
finally:
QApplication.processEvents()
finally:
request_summary = [
str(val) + " x " + key for key, val in statuscount.items()
]
request_summary = ", ".join(request_summary)
request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'
self.mainWindow.logmessage(
"{}, {} new node(s) created. Summary of responses: {}.".
format(request_end,
self.mainWindow.tree.treemodel.nodecounter,
request_summary))
self.mainWindow.tree.treemodel.commitNewNodes()
except Exception as e:
self.mainWindow.logmessage(
"Error in scheduler, fetching aborted: {}.".format(str(e)))
finally:
progress.close()
return not progress.wasCanceled
def queryNodes(self, indexes=None, apimodule=False, options=None):
if not (self.mainWindow.tree.selectedCount()
or self.mainWindow.allnodesCheckbox.isChecked() or
(indexes is not None)):
return False
#Show progress window
progress = ProgressBar("Fetching Data", parent=self.mainWindow)
try:
apimodule, options = self.getQueryOptions(apimodule, options)
indexes = self.getIndexes(options, indexes, progress)
# Update progress window
self.mainWindow.logmessage("Start fetching data.")
totalnodes = 0
hasindexes = True
progress.setMaximum(totalnodes)
self.mainWindow.tree.treemodel.nodecounter = 0
#Init status messages
statuscount = defaultdict(int)
errorcount = 0
ratelimitcount = 0
allowedstatus = [
'fetched (200)', 'downloaded (200)', 'fetched (202)'
]
try:
#Spawn Threadpool
threadpool = ApiThreadPool(apimodule)
threadpool.spawnThreads(options.get("threads", 1))
#Process Logging/Input/Output Queue
while True:
try:
#Logging (sync logs in threads with main thread)
msg = threadpool.getLogMessage()
if msg is not None:
self.mainWindow.logmessage(msg)
# Jobs in: packages of 100 at a time
jobsin = 0
while hasindexes and (jobsin < 100):
index = next(indexes, False)
if index:
jobsin += 1
totalnodes += 1
if index.isValid():
job = self.prepareJob(index, options)
threadpool.addJob(job)
else:
threadpool.applyJobs()
progress.setRemaining(threadpool.getJobCount())
progress.resetRate()
hasindexes = False
progress.removeInfo('input')
self.mainWindow.logmessage(
"Added {} node(s) to queue.".format(
totalnodes))
if jobsin > 0:
progress.setMaximum(totalnodes)
#Jobs out
job = threadpool.getJob()
#-Finished all nodes (sentinel)...
if job is None:
break
#-Finished one node...
elif 'progress' in job:
progresskey = 'nodeprogress' + str(
job.get('threadnumber', ''))
# Update single progress
if 'current' in job:
percent = int((job.get('current', 0) * 100.0 /
job.get('total', 1)))
progress.showInfo(
progresskey,
"{}% of current node processed.".format(
percent))
elif 'page' in job:
if job.get('page', 0) > 1:
progress.showInfo(
progresskey,
"{} page(s) of current node processed."
.format(job.get('page', 0)))
# Update total progress
else:
progress.removeInfo(progresskey)
if not threadpool.suspended:
progress.step()
#-Add data...
elif 'data' in job and (not progress.wasCanceled):
if not job['nodeindex'].isValid():
continue
# Add data
treeindex = job['nodeindex']
treenode = treeindex.internalPointer()
newcount = treenode.appendNodes(
job['data'], job['options'], True)
if options.get('expand', False):
self.mainWindow.tree.setExpanded(
treeindex, True)
# Count status and errors
status = job['options'].get('querystatus', 'empty')
statuscount[status] += 1
errorcount += int(not status in allowedstatus)
# Detect rate limit
ratelimit = job['options'].get('ratelimit', False)
#ratelimit = ratelimit or (not newcount)
ratelimitcount += int(ratelimit)
autoretry = (ratelimitcount) or (
status == "request error")
# Clear errors when everything is ok
if not threadpool.suspended and (
status
in allowedstatus) and (not ratelimit):
#threadpool.clearRetry()
errorcount = 0
ratelimitcount = 0
# Suspend on error or ratelimit
elif (errorcount >=
options['errors']) or (ratelimitcount > 0):
threadpool.suspendJobs()
if ratelimit:
msg = "You reached the rate limit of the API."
else:
msg = "{} consecutive errors occurred.\nPlease check your settings.".format(
errorcount)
timeout = 60 * 5 # 5 minutes
# Adjust progress
progress.showError(msg, timeout, autoretry)
self.mainWindow.tree.treemodel.commitNewNodes()
# Add job for retry
if not status in allowedstatus:
threadpool.addError(job)
# Show info
progress.showInfo(
status,
"{} response(s) with status: {}".format(
statuscount[status], status))
progress.showInfo(
'newnodes', "{} new node(s) created".format(
self.mainWindow.tree.treemodel.nodecounter)
)
progress.showInfo(
'threads', "{} active thread(s)".format(
threadpool.getThreadCount()))
progress.setRemaining(threadpool.getJobCount())
# Custom info from modules
info = job['options'].get('info', {})
for name, value in info.items():
progress.showInfo(name, value)
# Abort
elif progress.wasCanceled:
progress.showInfo(
'cancel',
"Disconnecting from stream, may take some time."
)
threadpool.stopJobs()
# Retry
elif progress.wasResumed:
if progress.wasRetried:
threadpool.retryJobs()
else:
threadpool.clearRetry()
# errorcount = 0
# ratelimitcount = 0
threadpool.resumeJobs()
progress.setRemaining(threadpool.getJobCount())
progress.hideError()
# Continue
elif not threadpool.suspended:
threadpool.resumeJobs()
# Finished with pending errors
if not threadpool.hasJobs(
) and threadpool.hasErrorJobs():
msg = "All nodes finished but you have {} pending errors. Skip or retry?".format(
threadpool.getErrorJobsCount())
autoretry = False
timeout = 60 * 5 # 5 minutes
progress.showError(msg, timeout, autoretry)
# Finished
if not threadpool.hasJobs():
progress.showInfo(
'cancel',
"Work finished, shutting down threads.")
threadpool.stopJobs()
#-Waiting...
progress.computeRate()
time.sleep(1.0 / 1000.0)
finally:
QApplication.processEvents()
finally:
request_summary = [
str(val) + " x " + key for key, val in statuscount.items()
]
request_summary = ", ".join(request_summary)
request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'
self.mainWindow.logmessage(
"{}, {} new node(s) created. Summary of responses: {}.".
format(request_end,
self.mainWindow.tree.treemodel.nodecounter,
request_summary))
self.mainWindow.tree.treemodel.commitNewNodes()
except Exception as e:
self.mainWindow.logmessage(
"Error in scheduler, fetching aborted: {}.".format(str(e)))
finally:
progress.close()
return not progress.wasCanceled