class MainW(QtGui.QMainWindow): """ Mainw to host and run one single thread """ def __init__(self): QtGui.QMainWindow.__init__(self) self.mutex = QMutex() self.thr = GuiThread(process=self.fun, period=0.1) self.thr2 = GuiThread(process=self.fun2, period=0.1) self.buff = CircIoBuffer(byte_size=15*len('rafal miecznik')) self.thr.start() self.thr2.start() time.sleep(2) print self.buff.read() for d in dir(self.mutex): print d def fun(self): print self.mutex.tryLock() self.buff.write(' rafal') self.mutex.unlock() #print self.thr.currentThreadId() def fun2(self): self.mutex.lock() self.buff.write(' miecznik')
class ThreadSerial(QThread): dataUpdate = pyqtSignal(str) def __init__(self, envs): super(ThreadSerial, self).__init__() self.serial = envs.get('SERIAL') self.mutex = QMutex() self.exit = False def run(self): debug("[+] Listening {}".format(self.serial.port)) while not self.exit: if self.mutex.tryLock(100): try: data = self.serial.read_all() finally: self.mutex.unlock() if data: self.dataUpdate.emit(data) time.sleep(0.001) def pause(self, timeout=1000): return self.mutex.tryLock(timeout) def resume(self): self.mutex.unlock() def read_no_wait(self): data = self.serial.read_all() self.dataUpdate.emit(data) return data def write_no_wait(self, data): self.serial.write(unicode(data).encode('utf-8')) def write(self, data): if self.mutex.tryLock(100): try: self.serial.write(unicode(data).encode('utf-8')) finally: self.mutex.unlock() def terminate(self, timeout=1000): debug("[-] Terminate {}".format(self.serial.port)) self.exit = True if not self.wait(timeout): QThread.terminate(self) self.serial.close()
class ImporterThread(QThread): def __init__(self, framework, parent=None): QThread.__init__(self, parent) self.framework = framework self.qlock = QMutex() self.cursor = None QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport, Qt.DirectConnection) self.exec_() def quitHandler(self): if self.cursor: self.cursor.close() self.exit(0) def startedHandler(self): pass def runImport(self, importers, proxy_file, source, callback): self.importers = importers self.proxy_filelist = [proxy_file] self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def runImportList(self, importers, proxy_filelist, source, callback): self.importers = importers self.proxy_filelist = proxy_filelist self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def handle_runImport(self): if self.qlock.tryLock(): try: for proxy_file in self.proxy_filelist: try: self.framework.debug_log('attempting import of %s' % (str(proxy_file))) self.importers.process_import(str(proxy_file), self.framework, self.source) except Exception as ex: self.framework.report_exception(ex) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('runImportFinished()'))
class ImporterThread(QThread): def __init__(self, framework, parent = None): QThread.__init__(self, parent) self.framework = framework self.qlock = QMutex() self.cursor = None QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport, Qt.DirectConnection) self.exec_() def quitHandler(self): if self.cursor: self.cursor.close() self.exit(0) def startedHandler(self): pass def runImport(self, importers, proxy_file, source, callback): self.importers = importers self.proxy_filelist = [proxy_file] self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def runImportList(self, importers, proxy_filelist, source, callback): self.importers = importers self.proxy_filelist = proxy_filelist self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def handle_runImport(self): if self.qlock.tryLock(): try: for proxy_file in self.proxy_filelist: try: self.framework.debug_log('attempting import of %s' % (str(proxy_file))) self.importers.process_import(str(proxy_file), self.framework, self.source) except Exception as ex: self.framework.report_exception(ex) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('runImportFinished()'))
class AnalyzerThread(QThread): def __init__(self, framework, parent=None): QThread.__init__(self, parent) self.framework = framework self.callback_object = None self.qtimer = QTimer() self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.handle_quit) QObject.connect(self, SIGNAL('started()'), self.handle_started) self.Data = None self.read_cursor = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.read_cursor = self.Data.allocate_thread_cursor() self.cursor = self.Data.allocate_thread_cursor() def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.read_cursor: self.read_cursor.close() self.Data.release_thread_cursor(self.read_cursor) self.read_cursor = None if self.cursor: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('do_runAnalysis()'), self.handle_runAnalysis, Qt.DirectConnection) self.exec_() def runAnalysis(self, callback_object): self.callback_object = callback_object QTimer.singleShot(50, self, SIGNAL('do_runAnalysis()')) def handle_quit(self): self.framework.debug_log('AnalyzerThread quit...') self.close_cursor() self.exit(0) def handle_started(self): self.framework.debug_log('AnalyzerThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def handle_runAnalysis(self): fullanalysistext = '' if not self.qlock.tryLock(): self.framework.debug_log('failed to acquire lock for analyzers') else: try: fullanalysistext = self.analyze_content() except Exception as e: self.framework.report_exception(e) finally: self.qlock.unlock() if self.callback_object: self.callback_object.emit(SIGNAL('runAnalysisFinished(QString)'), str(fullanalysistext)) def result_type_to_string(self, resulttype): """Reads type and package information from a result, and returns it as a string""" resultclass = resulttype.__class__ return "".join((resultclass.__module__, ".", resultclass.__name__)) def analyze_content(self): """ Perform analysis on the captured content""" #TODO: NEW DB THREAD TO HOLD RESPONSES, ANOTHER FOR WRITING RESULTS scopeController = self.framework.getScopeController() response = self.Data.read_all_responses(self.read_cursor) response_IDs = [] for row in response: dbrow = [m or '' for m in row] Id = dbrow[ResponsesTable.ID] url = dbrow[ResponsesTable.URL] if scopeController.isUrlInScope(url, url): response_IDs.append(Id) #Instantiate all found analyzers analyzerobjects = AnalyzerList(self.framework) analyzerobjects.instantiate_analyzers() analysisrunid = self.Data.analysis_start(self.cursor) #TODO - Consider threading from here down for x in analyzerobjects: #dbconfig=self.Data.get_config_value(self.read_cursor, 'ANALYSIS', str(x.__class__)) #print "dbconfig=%s"%dbconfig print("class=%s" % x.__class__) #x.setConfiguration(dbconfig) x.preanalysis() x.initResultsData() resultinstance = x.getResults() x.analyzerinstanceid = self.Data.analysis_add_analyzer_instance( self.cursor, analysisrunid, str(x.__class__).translate('<>'), x.friendlyname, x.desc, self.result_type_to_string(resultinstance)) fullanalysistext = StringIO() for Id in response_IDs: transaction = self.framework.get_request_response(Id) for analyzer in analyzerobjects: try: analyzer.analyzeTransaction(transaction, analyzer.getResults()) tempanalysisresults = analyzer.getResults() #If there were results for this page, add them to the DB if transaction.Id in tempanalysisresults.pages: pageresultset = self.Data.analysis_add_resultset( self.cursor, analyzer.analyzerinstanceid, transaction.Id, False, transaction.responseUrl, self.result_type_to_string( tempanalysisresults.pages[transaction.Id])) for result in tempanalysisresults.pages[ transaction.Id].results: self.Data.analysis_add_singleresult( self.cursor, pageresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) for key, value in list(tempanalysisresults.pages[ transaction.Id].stats.items()): self.Data.analysis_add_stat( self.cursor, pageresultset, key, value) except Exception as e: # TODO: add real debugging support self.framework.debug_log('Transaction ID: ' + str(transaction.Id)) self.framework.report_exception(e) #Post Analysis for analyzer in analyzerobjects: results = analyzer.getResults() analyzer.postanalysis(results) for context in list(results.overall.keys()): overallresultset = self.Data.analysis_add_resultset( self.cursor, analyzer.analyzerinstanceid, None, True, context, self.result_type_to_string(results.overall[context])) for result in results.overall[context].results: self.Data.analysis_add_singleresult( self.cursor, overallresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) #print "WRITING:",self.result_type_to_string(result) for key, value in list(results.overall[context].stats.items()): self.Data.analysis_add_stat(self.cursor, overallresultset, key, value) self.Data.commit() #Output results to analysis tab #for analyzer in analyzerobjects: #fullanalysistext.write(analyzer.getResults().toHTML()) return ''
class SiteMapThread(QThread): def __init__(self, framework, treeViewModel, parent=None): QThread.__init__(self, parent) self.framework = framework self.treeViewModel = treeViewModel self.qtimer = QTimer() self.qlock = QMutex() self.fillAll = False QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) self.re_set_cookie = re.compile(r'^Set-Cookie2?:\s*(.+)$', re.I | re.M) self.re_host_name = re.compile(r'^Host:\s*(.+?)$', re.I | re.M) self.lastId = 0 self.Data = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.cursor = self.Data.allocate_thread_cursor() self.populateSiteMap(True) def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.cursor and self.Data: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('updateSiteMap()'), self.doUpdateSiteMap, Qt.DirectConnection) self.framework.subscribe_response_data_added(self.doUpdateSiteMap) self.exec_() def quitHandler(self): self.framework.debug_log('SiteMapThread quit...') self.close_cursor() self.exit(0) def startedHandler(self): self.framework.debug_log('SiteMapThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def populateSiteMap(self, fillAll): self.fillAll = fillAll QTimer.singleShot(50, self, SIGNAL('updateSiteMap()')) def doUpdateSiteMap(self): if not self.qlock.tryLock(): return try: if self.fillAll: self.fillAll = False self.treeViewModel.clearModel() self.lastId = 0 rows = self.Data.get_sitemap_info(self.cursor, self.lastId) global_cookie_jar = self.framework.get_global_cookie_jar() count = 0 for row in rows: count += 1 if 0 == (count % 100): self.yieldCurrentThread() rowItems = [m or '' for m in list(row)] Id = str(rowItems[0]) try: self.lastId = int(Id) except ValueError: pass # XXX: review all for bytes usage if isinstance(rowItems[1], bytes): url = str(rowItems[1], 'utf-8', 'ignore') else: url = str(rowItems[1]) status = str(rowItems[2]) response_headers = str(rowItems[3]) request_headers = str(rowItems[4]) # TODO: make configurable if status in ('400', '404', '500', '501'): continue # TODO: m = self.re_set_cookie.search(response_headers) if m: setCookies = m.group(1) cookieList = QNetworkCookie.parseCookies(setCookies) global_cookie_jar.setCookiesFromUrl( cookieList, QUrl.fromEncoded(url)) parsed = urlparse.urlsplit(url) hostname = '' if not parsed.hostname: m = self.re_host_name.search(request_headers) if m: hostname = m.group(1).rstrip() else: hostname = parsed.hostname hostname = hostname.lower() hostloc = urlparse.urlunsplit( (parsed.scheme, parsed.netloc, '/', '', '')) rootNode = self.treeViewModel.findOrAddNode(hostname) hostLocNode = rootNode.findOrAddNode(self.treeViewModel, hostloc) pathval = parsed.path # add directories parentNode = hostLocNode parentNode.setResponseId(None, hostloc) lastSlash = 0 slash = 0 while True: slash = pathval.find('/', slash + 1) if slash < 0: break dirname = pathval[lastSlash + 1:slash + 1] parentNode = parentNode.findOrAddNode( self.treeViewModel, dirname) parentNode.setResponseId( None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval[0:slash + 1], '', ''))) lastSlash = slash # add file element if lastSlash + 1 < len(pathval): filename = pathval[lastSlash + 1:] parentNode = parentNode.findOrAddNode( self.treeViewModel, filename) parentNode.setResponseId( None, urlparse.urlunsplit( (parsed.scheme, parsed.netloc, pathval, '', ''))) # add query if parsed.query: parentNode = parentNode.findOrAddNode( self.treeViewModel, '?' + parsed.query) # store the latest Id # TODO: should determine best candidate to display parentNode.setResponseId(Id, url) finally: self.qlock.unlock()
class VcsStatusMonitorThread(QThread): """ Class implementing the VCS status monitor thread base class. @signal vcsStatusMonitorData(QStringList) emitted to update the VCS status @signal vcsStatusMonitorStatus(QString, QString) emitted to signal the status of the monitoring thread (ok, nok, op) and a status message """ def __init__(self, interval, projectDir, vcs, parent = None): """ Constructor @param interval new interval in seconds (integer) @param projectDir project directory to monitor (string or QString) @param vcs reference to the version control object @param parent reference to the parent object (QObject) """ QThread.__init__(self, parent) self.setObjectName("VcsStatusMonitorThread") self.setTerminationEnabled(True) self.projectDir = QString(projectDir) self.vcs = vcs self.interval = interval self.autoUpdate = False self.statusList = QStringList() self.reportedStates = {} self.shouldUpdate = False self.monitorMutex = QMutex() self.monitorCondition = QWaitCondition() self.__stopIt = False def run(self): """ Protected method implementing the tasks action. """ while not self.__stopIt: # perform the checking task self.statusList.clear() self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("wait"), self.trUtf8("Waiting for lock")) try: locked = self.vcs.vcsExecutionMutex.tryLock(5000) except TypeError: locked = self.vcs.vcsExecutionMutex.tryLock() if locked: try: self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("op"), self.trUtf8("Checking repository status")) res, statusMsg = self._performMonitor() finally: self.vcs.vcsExecutionMutex.unlock() if res: status = QString("ok") else: status = QString("nok") self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("send"), self.trUtf8("Sending data")) self.emit(SIGNAL("vcsStatusMonitorData(QStringList)"), QStringList(self.statusList)) self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), status, statusMsg) else: self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("timeout"), self.trUtf8("Timed out waiting for lock")) if self.autoUpdate and self.shouldUpdate: try: self.vcs.vcsUpdate(self.projectDir, True) continue # check again except TypeError: pass # compatibility for older VCS plugins self.shouldUpdate = False # wait until interval has expired checking for a stop condition self.monitorMutex.lock() if not self.__stopIt: self.monitorCondition.wait(self.monitorMutex, self.interval * 1000) self.monitorMutex.unlock() self.exit() def setInterval(self, interval): """ Public method to change the monitor interval. @param interval new interval in seconds (integer) """ locked = self.monitorMutex.tryLock() self.interval = interval self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def getInterval(self): """ Public method to get the monitor interval. @return interval in seconds (integer) """ return self.interval def setAutoUpdate(self, auto): """ Public method to enable the auto update function. @param auto status of the auto update function (boolean) """ self.autoUpdate = auto def getAutoUpdate(self): """ Public method to retrieve the status of the auto update function. @return status of the auto update function (boolean) """ return self.autoUpdate def checkStatus(self): """ Public method to wake up the status monitor thread. """ locked = self.monitorMutex.tryLock() self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def stop(self): """ Public method to stop the monitor thread. """ locked = self.monitorMutex.tryLock() self.__stopIt = True self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def clearCachedState(self, name): """ Public method to clear the cached VCS state of a file/directory. @param name name of the entry to be cleared (QString or string) """ project = e4App().getObject("Project") key = project.getRelativePath(unicode(name)) try: del self.reportedStates[key] except KeyError: pass def _performMonitor(self): """ Protected method implementing the real monitoring action. This method must be overridden and populate the statusList member variable with a list of strings giving the status in the first column and the path relative to the project directory starting with the third column. The allowed status flags are: <ul> <li>"A" path was added but not yet comitted</li> <li>"M" path has local changes</li> <li>"O" path was removed</li> <li>"R" path was deleted and then re-added</li> <li>"U" path needs an update</li> <li>"Z" path contains a conflict</li> <li>" " path is back at normal</li> </ul> @return tuple of flag indicating successful operation (boolean) and a status message in case of non successful operation (QString) """ raise RuntimeError('Not implemented')
class SiteMapThread(QThread): def __init__(self, framework, treeViewModel, parent=None): QThread.__init__(self, parent) self.framework = framework self.treeViewModel = treeViewModel self.qtimer = QTimer() self.qlock = QMutex() self.fillAll = False QObject.connect(self, SIGNAL("quit()"), self.quitHandler) QObject.connect(self, SIGNAL("started()"), self.startedHandler) self.re_set_cookie = re.compile(r"^Set-Cookie2?:\s*(.+)$", re.I | re.M) self.re_host_name = re.compile(r"^Host:\s*(.+?)$", re.I | re.M) self.lastId = 0 self.Data = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.cursor = self.Data.allocate_thread_cursor() self.populateSiteMap(True) def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.cursor and self.Data: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL("updateSiteMap()"), self.doUpdateSiteMap, Qt.DirectConnection) self.framework.subscribe_response_data_added(self.doUpdateSiteMap) self.exec_() def quitHandler(self): self.framework.debug_log("SiteMapThread quit...") self.close_cursor() self.exit(0) def startedHandler(self): self.framework.debug_log("SiteMapThread started...") self.framework.subscribe_database_events(self.db_attach, self.db_detach) def populateSiteMap(self, fillAll): self.fillAll = fillAll QTimer.singleShot(50, self, SIGNAL("updateSiteMap()")) def doUpdateSiteMap(self): if not self.qlock.tryLock(): return try: if self.fillAll: self.fillAll = False self.treeViewModel.clearModel() self.lastId = 0 rows = self.Data.get_sitemap_info(self.cursor, self.lastId) global_cookie_jar = self.framework.get_global_cookie_jar() count = 0 for row in rows: count += 1 if 0 == (count % 100): self.yieldCurrentThread() rowItems = [m or "" for m in list(row)] Id = str(rowItems[0]) try: self.lastId = int(Id) except ValueError: pass # XXX: review all for bytes usage if isinstance(rowItems[1], bytes): url = str(rowItems[1], "utf-8", "ignore") else: url = str(rowItems[1]) status = str(rowItems[2]) response_headers = str(rowItems[3]) request_headers = str(rowItems[4]) # TODO: make configurable if status in ("400", "404", "500", "501"): continue # TODO: m = self.re_set_cookie.search(response_headers) if m: setCookies = m.group(1) cookieList = QNetworkCookie.parseCookies(setCookies) global_cookie_jar.setCookiesFromUrl(cookieList, QUrl.fromEncoded(url)) parsed = urlparse.urlsplit(url) hostname = "" if not parsed.hostname: m = self.re_host_name.search(request_headers) if m: hostname = m.group(1).rstrip() else: hostname = parsed.hostname hostname = hostname.lower() hostloc = urlparse.urlunsplit((parsed.scheme, parsed.netloc, "/", "", "")) rootNode = self.treeViewModel.findOrAddNode(hostname) hostLocNode = rootNode.findOrAddNode(self.treeViewModel, hostloc) pathval = parsed.path # add directories parentNode = hostLocNode parentNode.setResponseId(None, hostloc) lastSlash = 0 slash = 0 while True: slash = pathval.find("/", slash + 1) if slash < 0: break dirname = pathval[lastSlash + 1 : slash + 1] parentNode = parentNode.findOrAddNode(self.treeViewModel, dirname) parentNode.setResponseId( None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval[0 : slash + 1], "", "")) ) lastSlash = slash # add file element if lastSlash + 1 < len(pathval): filename = pathval[lastSlash + 1 :] parentNode = parentNode.findOrAddNode(self.treeViewModel, filename) parentNode.setResponseId(None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval, "", ""))) # add query if parsed.query: parentNode = parentNode.findOrAddNode(self.treeViewModel, "?" + parsed.query) # store the latest Id # TODO: should determine best candidate to display parentNode.setResponseId(Id, url) finally: self.qlock.unlock()
class SVDViewer(QWidget): svdSelectEvent = QtCore.pyqtSignal() def __init__(self, array, labels, **options): QWidget.__init__(self) assert isinstance(array, np.ndarray),\ "Got a %s instead of an ndarray" % type(array) # figure out our size self.width = self.size().width() self.height = self.size().height() self.labels = labels self.magnitudes = None # can be assigned by external information self.array = np.asarray(array) self.orig_array = self.array.copy() self.npoints = self.array.shape[0] self.scale = self.calculate_scale() if options.get('jitter', True): self.add_jitter() self.layers = [] self.k = self.array.shape[1] self.projection = Projection(self.k) # create our internal display self.painter = QPainter() self.selected_index = None self.mouseX = 0 self.mouseY = 0 self.buttons = 0 # avoid racing self.paint_lock = QMutex() self.timer = QTimer(self) self.timer_ticks = 0 self.timer.setInterval(30) self.timer.timeout.connect(self.timerEvent) self.timer.start() self.setMouseTracking(True) self.default_colors = self.components_to_colors(self.array)[:] self.update_colors() # initialize the mapping from projection coordinates to screen # coordinates self.reset_view() def age_timer(self): self.timer_ticks += 1 if self.timer_ticks == TIMER_MAX: self.timer.stop() def activate_timer(self): if self.timer_ticks >= TIMER_MAX: self.timer.start() self.timer_ticks = 0 def stop_timer(self): self.timer.stop() def __del__(self): self.stop_timer() def calculate_scale(self): """ Find roughly the median of axis coordinates, determining a reasonable zoom level for the initial view. Make sure it's non-zero. """ coords = [c for c in np.abs(self.array.flatten()) if c > 0] + [1.0] coords.sort() return coords[len(coords)//2] def add_jitter(self): self.jitter = np.exp(np.random.normal(size=self.array.shape) / 50.0) self.array *= self.jitter def insert_layer(self, pos, layertype, *args): """ Add a layer of visualization to this widget, by specifying the subclass of Layer and whatever arguments its constructor takes. """ self.layers.insert(pos, layertype(self, *args)) def add_layer(self, layertype, *args): """ Add a layer of visualization to this widget, by specifying the subclass of Layer and whatever arguments its constructor takes. """ self.layers.append(layertype(self, *args)) def reset_view(self): self.screen_center = np.array([0., 0.]) self.screen_size = np.array([self.scale/10, self.scale/10]) self.projection.reset_projection() self.set_default_axes() self.update_screenpts() self.update() @staticmethod def make_svdview(matrix, svdmatrix, magnitudes=None, canonical=None): widget = SVDViewer(svdmatrix, svdmatrix.row_labels) if magnitudes is None: magnitudes = np.array([np.linalg.norm(vec) for vec in svdmatrix]) widget.magnitudes = magnitudes widget.setup_standard_layers() widget.set_default_axes() if canonical is None: canonical = [] for c in canonical: #svdmatrix[svdmatrix.row_index(c)] *= 4 magnitudes[svdmatrix.row_index(c)] *= 2 widget.insert_layer(1, CanonicalLayer, canonical) widget.insert_layer(2, LinkLayer, matrix) widget.insert_layer(3, NetworkLayer, 6) return widget @staticmethod def make_colors(matrix, svdmatrix): widget = SVDViewer(svdmatrix, svdmatrix.col_labels) widget.setup_standard_layers() from csc.concepttools.colors import text_color colors = [text_color(text) for text in widget.labels] widget.default_colors = np.clip(np.array(colors), 55, 230) widget.update_colors() return widget @staticmethod def make(array, labels): widget = SVDViewer(array, labels) widget.setup_standard_layers() return widget def setup_standard_layers(self): self.add_layer(PixelRenderingLayer) self.add_layer(PointLayer) self.add_layer(LabelLayer, 1000, 2000) self.add_layer(SelectionLayer) self.add_layer(SimilarityLayer) self.add_layer(RotationLayer) self.add_layer(PanZoomLayer) def set_default_axes(self): self.set_axis_to_pc(0, 1) self.set_axis_to_pc(1, 2) def set_default_x_axis(self): self.set_axis_to_pc(0, 1) def set_default_y_axis(self): self.set_axis_to_pc(1, 2) def set_axis_to_pc(self, axis, pc): """ Sets an axis to a particular principal component. """ pcvec = np.zeros((self.array.shape[1],)) pcvec[pc] = 1.0 if axis == 0: self.projection.set_x_axis(pcvec) elif axis == 1: self.projection.set_y_axis(pcvec) self.activate_timer() def set_axis_to_text(self, axis, text): if isinstance(text, QString): text = unicode(text) if not text: return if text in self.labels: index = self.labels.index(text) if axis == 0: self.projection.set_x_axis(self.array[index,:]) elif axis == 1: self.projection.set_y_axis(self.array[index,:]) self.activate_timer() else: print repr(text), "not in label list" def is_point_on_screen(self, coords): return np.all((coords >= np.int32([0, 0])) & (coords < np.int32([self.width, self.height])), axis=-1) def is_on_screen(self, x, y): return x >= 0 and x < self.width and y >= 0 and y < self.height def pixel_size(self): """ What is the diameter of a single pixel at the current zoom level? If the x and y scales end up being different, take the geometric mean. """ xsize = self.screen_size[0] / self.width ysize = self.screen_size[1] / self.height return np.sqrt(xsize*ysize) def components_to_screen(self, coords): return self.projection_to_screen(self.projection.components_to_projection(coords)) def projection_to_screen(self, coords): zoomed = (coords - self.screen_center) / self.screen_size screen = (zoomed + np.array([self.width, self.height])/2) return np.int32(screen * FLIP_Y + [0, self.height]) def screen_to_projection(self, screen_coords): zoomed = ((screen_coords - [0, self.height])*FLIP_Y) - np.array([self.width, self.height])/2 return (zoomed * self.screen_size) + self.screen_center def components_to_colors(self, coords): while coords.shape[1] < 5: coords = np.concatenate([coords, -coords, coords], axis=1) return np.clip(np.int32(coords[...,3:6]*80/self.scale + 160), 50, 230) def update_screenpts(self): self.screenpts = self.components_to_screen(self.array) def update_colors(self): self.colors = self.default_colors def constrain_to_screen(self, points): return np.clip(points, np.int32([1., 1.]), np.int32([self.width-2, self.height-2])).T def distances_from_mouse(self, coords): mouse = np.int32([self.mouseX, self.mouseY]) offsets = self.screenpts - mouse return np.sqrt(np.sum(offsets*offsets, axis=1)) def get_nearest_point(self): return np.argmin(self.distances_from_mouse(self.array)) def select_nearest_point(self): self.selected_index = self.get_nearest_point() self.selectEvent(self.selected_index) def selected_vector(self): return self.array[self.selected_index] def selected_label(self): return self.labels[self.selected_index] def selectEvent(self, index): for layer in self.layers: layer.selectEvent(index) self.svdSelectEvent.emit() def focus_on_point(self, text): index = self.labels.index(text) if index is None: return coords = self.projection.components_to_projection(self.array[index]) if not self.is_point_on_screen(coords): self.screen_center = coords self.selected_index = index self.selectEvent(index) def paintEvent(self, event): if self.paint_lock.tryLock(): QWidget.paintEvent(self, event) self.painter.begin(self) self.painter.setRenderHint(QPainter.Antialiasing, True) self.painter.setRenderHint(QPainter.TextAntialiasing, True) try: for layer in self.layers: layer.draw(self.painter) finally: self.painter.end() self.paint_lock.unlock() def resizeEvent(self, sizeEvent): self.width = sizeEvent.size().width() self.height = sizeEvent.size().height() for layer in self.layers: layer.resize(self.width, self.height) def mouseMoveEvent(self, mouseEvent): point = mouseEvent.pos() self.mouseX = point.x() self.mouseY = point.y() for layer in self.layers: layer.mouseMoveEvent(mouseEvent) if self.leftMouseDown() or self.rightMouseDown(): self.activate_timer() elif self.timer_ticks >= TIMER_MAX: self.update() def timerEvent(self): self.projection.timerEvent() for layer in self.layers: layer.timerEvent() self.update_screenpts() self.update() self.age_timer() def updateMouseButtons(self, event): self.buttons = event.buttons() self.modifiers = event.modifiers() def leftMouseDown(self): ''' Checks if the left mouse button was pressed at the time of the last event. Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows). ''' return (self.buttons & Qt.LeftButton) and not (self.modifiers & RIGHT_BUTTON_MODIFIER) def rightMouseDown(self): ''' Checks if the right mouse button was pressed at the time of the last event. Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows). ''' return (self.buttons & Qt.RightButton) or (self.buttons & Qt.LeftButton and self.modifiers & RIGHT_BUTTON_MODIFIER) def mousePressEvent(self, mouseEvent): self.updateMouseButtons(mouseEvent) for layer in self.layers: layer.mousePressEvent(mouseEvent) self.activate_timer() def mouseReleaseEvent(self, mouseEvent): self.updateMouseButtons(mouseEvent) for layer in self.layers: layer.mouseReleaseEvent(mouseEvent) self.activate_timer() def wheelEvent(self, mouseEvent): for layer in self.layers: layer.wheelEvent(mouseEvent) self.activate_timer() def dropLabel(self, index, label): self.refreshData(self, index) def refreshData(self): self.update_screenpts() self.default_colors = self.components_to_colors(self.array)[:] self.update_colors() self.update() def get_svg_figure(self): figs = [] for layer in self.layers: fig = layer.drawSVG() if fig is not None: figs.append(fig) return svgfig.Fig(*figs) def write_svg(self, filename=None): if filename is None: filename = 'luminoso.svg' out = open(filename, 'w') out.write("<svg>\n") out.write(self.get_svg_figure().SVG().xml()) out.write("\n</svg>\n") out.close()
class QuickAnalysisThread(QThread): def __init__(self, framework, parent=None): QThread.__init__(self, parent) self.framework = framework self.callback_object = None self.qtimer = QTimer() self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.handle_quit) QObject.connect(self, SIGNAL('started()'), self.handle_started) self.Data = None self.read_cursor = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.read_cursor = self.Data.allocate_thread_cursor() def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.read_cursor: self.read_cursor.close() self.Data.release_thread_cursor(self.read_cursor) self.read_cursor = None def run(self): QObject.connect(self, SIGNAL('do_runQuickAnalysis()'), self.handle_runQuickAnalysis, Qt.DirectConnection) self.exec_() def runQuickAnalysis(self, python_code, callback_object): self.python_code = python_code self.callback_object = callback_object QTimer.singleShot(50, self, SIGNAL('do_runQuickAnalysis()')) def handle_quit(self): self.framework.debug_log('QuickAnalysisThread quit...') self.close_cursor() self.exit(0) def handle_started(self): self.framework.debug_log('QuickAnalysisThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def append_results(self, results_io, res): if isinstance(res, bytes): res = str(res, 'utf-8', 'ignore') if res.endswith('\n'): results_io.write(res + '\n') else: results_io.write(res + '\n\n') def handle_runQuickAnalysis(self): results = '' results_io = StringIO() if not self.qlock.tryLock(): self.framework.debug_log( 'failed to acquire lock for quick analysis') else: original_stdout = sys.stdout sys.stdout = results_io try: python_code = str(self.python_code) scriptLoader = ScriptLoader() global_ns = local_ns = {} script_env = scriptLoader.load_from_string( python_code, global_ns, local_ns) begin_method = script_env.functions.get('begin') if begin_method: res = begin_method() if res: self.append_results(results_io, res) process_request_method = script_env.functions.get( 'process_request') if not process_request_method: raise Exception( 'The "process_request" method is not implemented and is required.' ) factory = RequestResponseFactory.RequestResponseFactory( self.framework, None) for row in self.Data.read_all_responses(self.read_cursor): try: rr = factory.fill_by_row(row) res = process_request_method(rr) if res: self.append_results(results_io, res) except Exception as e: results += '\nEncountered processing error: %s' % (e) end_method = script_env.functions.get('end') if end_method: res = end_method() if res: self.append_results(results_io, res) except Exception as error: self.framework.report_exception(error) results += '\nEncountered processing error: %s' % (error) finally: sys.stdout = original_stdout self.qlock.unlock() if self.callback_object: if results: results += '\n' results += results_io.getvalue() self.callback_object.emit( SIGNAL('runQuickAnalysisFinished(QString)'), results)
class QuickAnalysisThread(QThread): def __init__(self, framework, parent = None): QThread.__init__(self, parent) self.framework = framework self.callback_object = None self.qtimer = QTimer() self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.handle_quit) QObject.connect(self, SIGNAL('started()'), self.handle_started) self.Data = None self.read_cursor = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.read_cursor = self.Data.allocate_thread_cursor() def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.read_cursor: self.read_cursor.close() self.Data.release_thread_cursor(self.read_cursor) self.read_cursor = None def run(self): QObject.connect(self, SIGNAL('do_runQuickAnalysis()'), self.handle_runQuickAnalysis, Qt.DirectConnection) self.exec_() def runQuickAnalysis(self, python_code, callback_object): self.python_code = python_code self.callback_object = callback_object QTimer.singleShot(50, self, SIGNAL('do_runQuickAnalysis()')) def handle_quit(self): self.framework.debug_log('QuickAnalysisThread quit...') self.close_cursor() self.exit(0) def handle_started(self): self.framework.debug_log('QuickAnalysisThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def append_results(self, results_io, res): if isinstance(res, bytes): res = str(res, 'utf-8', 'ignore') if res.endswith('\n'): results_io.write(res + '\n') else: results_io.write(res + '\n\n') def handle_runQuickAnalysis(self): results = '' results_io = StringIO() if not self.qlock.tryLock(): self.framework.debug_log('failed to acquire lock for quick analysis') else: original_stdout = sys.stdout sys.stdout = results_io try: python_code = str(self.python_code) scriptLoader = ScriptLoader() global_ns = local_ns = {} script_env = scriptLoader.load_from_string(python_code, global_ns, local_ns) begin_method = script_env.functions.get('begin') if begin_method: res = begin_method() if res: self.append_results(results_io, res) process_request_method = script_env.functions.get('process_request') if not process_request_method: raise Exception('The "process_request" method is not implemented and is required.') factory = RequestResponseFactory.RequestResponseFactory(self.framework, None) for row in self.Data.read_all_responses(self.read_cursor): try: rr = factory.fill_by_row(row) res = process_request_method(rr) if res: self.append_results(results_io, res) except Exception as e: results += '\nEncountered processing error: %s' % (e) end_method = script_env.functions.get('end') if end_method: res = end_method() if res: self.append_results(results_io, res) except Exception as error: self.framework.report_exception(error) results += '\nEncountered processing error: %s' % (error) finally: sys.stdout = original_stdout self.qlock.unlock() if self.callback_object: if results: results += '\n' results += results_io.getvalue() self.callback_object.emit(SIGNAL('runQuickAnalysisFinished(QString)'), results)
class MainWindowController(Controller): def __init__(self, application): Controller.__init__(self, True) self.application = application self.initialiseMutex() self.equipDBSystem = None self.devicesInfo = self.getDevicesInfo() self.sparesInfo = self.getSparesInfo() self.mainWindow = MainWindow(self) self.devicesWidget = DevicesWidget(self.devicesInfo, self.mainWindow, self) self.sparesWidget = SparesWidget(self.sparesInfo, self.mainWindow, self) self.connectActions(self.mainWindow) self.mainWindow.show() def initialiseMutex(self): self.sparesMutex = QMutex() self.devicesMutex = QMutex() self.showLogMutex = QMutex() self.refreshMutex = QMutex() self.devicesActionMutex = QMutex() self.sparesActionMutex = QMutex() def getDevicesInfo(self): confDB = self.getConfDBCxOracleConnection() equipDB = self.getEquipDBCxOracleConnection() if self.equipDBSystem is None: self.equipDBSystem = equipDB.getSystem() changedSystem = confDB.getChangedSystems(self.equipDBSystem) devicesInfo = DevicesInfo() devicesInfo.setNewDevicesWithDHCPData(len(changedSystem[0].getAllDevices())) devicesInfo.setNewDevicesWithoutDHCPData(len(changedSystem[1].getAllDevices())) devicesInfo.setChangedDevices(len(changedSystem[2].getAllDevices())+len(changedSystem[3].getAllDevices())) devicesInfo.setDevicesUpToDate(len(changedSystem[4].getAllDevices())) return devicesInfo def getSparesInfo(self): confDB = self.getConfDBCxOracleConnection() equipDB = self.getEquipDBCxOracleConnection() sparesInfo = SparesInfo() self.spareDB = self.getSpareDB() sparesInfo.SparesInEquipDB = self.spareDB.getAllSparesEquipDBCount() sparesInfo.setSparesInConfDB(self.spareDB.getAllSparesConfDBCount()) sparesInfo.setChangedSpares(len(self.spareDB.getChangedSpares())) sparesInfo.setNewSpares(len(self.spareDB.getNewSpares())) return sparesInfo def close(self): print "MainWindowController.close() start" Controller.disconnectConfDBPython(self) print "MainWindowController.close() end" def connectActions(self, mainWindow): mainWindow.connect(mainWindow.aboutAction, QtCore.SIGNAL("triggered()"), mainWindow.about) mainWindow.connect(mainWindow.refreshAction, QtCore.SIGNAL("triggered()"), self.onRefresh) mainWindow.connect(mainWindow.exitAction, QtCore.SIGNAL("triggered()"), self.onExit) mainWindow.connect(mainWindow, QtCore.SIGNAL('closeEmitApp()'), QtCore.SLOT('close()') ) mainWindow.connect(mainWindow, QtCore.SIGNAL("mainWindowCloses"), self.close) ########################################################################################## mainWindow.connect(mainWindow.createSummaryLogAction, QtCore.SIGNAL("triggered()"), self.onCreateSummaryLog) mainWindow.connect(mainWindow.selectLogAction, QtCore.SIGNAL("triggered()"), self.onSelectLog) mainWindow.connect(mainWindow.showLastSessionLogAction, QtCore.SIGNAL("triggered()"), self.onShowLastSessionLog) mainWindow.connect(mainWindow.showLastSummaryLogAction, QtCore.SIGNAL("triggered()"), self.onShowLastSummaryLog) ########################################################################################## mainWindow.connect(mainWindow.insertAction, QtCore.SIGNAL("triggered()"), self.onInsert) mainWindow.connect(mainWindow.updateAction, QtCore.SIGNAL("triggered()"), self.onUpdate) mainWindow.connect(mainWindow.deleteAction, QtCore.SIGNAL("triggered()"), self.onDelete) mainWindow.connect(mainWindow.connectAction, QtCore.SIGNAL("triggered()"), self.onConnect) mainWindow.connect(mainWindow.tfcmunin01Action, QtCore.SIGNAL("triggered()"), self.onTfcmunin01) mainWindow.connect(mainWindow.swapAction, QtCore.SIGNAL("triggered()"), self.onSwap) ########################################################################################## mainWindow.connect(mainWindow.insertNewSparesAction, QtCore.SIGNAL("triggered()"), self.onInsertNewSpares) mainWindow.connect(mainWindow.updateSparesAction, QtCore.SIGNAL("triggered()"), self.onUpdateSpares) mainWindow.connect(mainWindow.deleteSparesAction, QtCore.SIGNAL("triggered()"), self.onDeleteSpares) def onExit(self): print "MainWindowController.onExit() start" self.logfile.flush() self.mainWindow.emit(QtCore.SIGNAL('closeEmitApp()')) print "MainWindowController.onExit() end" def onCreateSummaryLog(self): print "MainWindowController.onCreateSummaryLog() start" self.createSummaryLogController = CreateSummaryLogController(self) print "MainWindowController.onCreateSummaryLog() end" def onSelectLog(self): print "MainWindowController.onSelectLog() start" self.selectLogFileController = SelectLogFileController(self) print "MainWindowController.onSelectLog() end" def onShowLastSessionLog(self): print "MainWindowController.onShowLastSessionLog() start" if self.showLogMutex.tryLock(): self.logfile.flush() self.showLogWindowController = ShowLogWindowController(self, False) self.showLogMutex.unlock() print "MainWindowController.onShowLastSessionLog() end" def onShowLastSummaryLog(self): print "MainWindowController.onShowLastSummaryLog() start" if self.showLogMutex.tryLock(): self.logfile.flush() self.showLogWindowController = ShowLogWindowController(self, True) self.showLogMutex.unlock() print "MainWindowController.onShowLastSummaryLog() end" def onInsert(self): print "MainWindowController.onInsert() start" if self.devicesActionMutex.tryLock(): confDB = self.getConfDBCxOracleConnection() equipDB = self.getEquipDBCxOracleConnection() self.newDevicesWindowController = NewDevicesWindowController(self) print "MainWindowController.onInsert() end" def onRefreshDevices(self): print "MainWindowController.onRefreshDevices() start" if self.refreshMutex.tryLock(): self.logfile.flush() self.refreshWorker = RefreshWorker(self.getEquipDBCxOracleConnection(), self.getConfDBCxOracleConnection(), self.spareDB, self.devicesInfo, None, self) self.refreshWorker.start() self.refreshMutex.unlock() print "MainWindowController.onRefreshDevices() end" def onRefreshSpares(self): print "MainWindowController.onRefreshSpares() start" if self.refreshMutex.tryLock(): self.logfile.flush() self.refreshWorker = RefreshWorker(self.getEquipDBCxOracleConnection(), self.getConfDBCxOracleConnection(), self.spareDB, None, self.sparesInfo, self) self.refreshWorker.start() self.refreshMutex.unlock() print "MainWindowController.onRefreshSpares() end" def onRefresh(self): print "MainWindowController.onRefresh() start" if self.refreshMutex.tryLock(): self.logfile.flush() self.refreshWorker = RefreshWorker(self.getEquipDBCxOracleConnection(), self.getConfDBCxOracleConnection(), self.spareDB, self.devicesInfo, self.sparesInfo, self) self.refreshWorker.start() self.refreshMutex.unlock() print "MainWindowController.onRefresh() end" def onRefreshDevicesWidget(self): print "MainWindowController.onRefreshDevicesWidget() start" if self.refreshMutex.tryLock(): self.devicesWidget.setDevicesInfo(self.devicesInfo) self.refreshMutex.unlock() print "MainWindowController.onRefreshDevicesWidget() end" def onRefreshSparesWidget(self): print "MainWindowController.onRefreshSparesWidget() start" self.sparesMutex.lock() self.sparesWidget.setSparesInfo(self.sparesInfo) self.sparesMutex.unlock() print "MainWindowController.onRefreshSparesWidget() end" def onUpdate(self): print "MainWindowController.onUpdate() start" if self.devicesActionMutex.tryLock(): self.updateDevicesController = UpdateDevicesController(self) print "MainWindowController.onUpdate() end" def onDelete(self): print "MainWindowController.onDelete() start" if self.devicesActionMutex.tryLock(): self.db = self.getCn() self.deleteDevicesWindowController = DeleteDevicesWindowController(self) print "MainWindowController.onDelete() end" def onConnect(self): print "MainWindowController.onConnect() start" if self.connectMutex.tryLock(): self.db = self.getCn() self.connectMutex.unlock() print "MainWindowController.onConnect() end" def onTfcmunin01(self): print "MainWindowController.onTfcmunin01() start" if self.devicesActionMutex.tryLock(): self.tFCMunin01Controller = TFCMunin01Controller(self) print "MainWindowController.onTfcmunin01() end" def onSwap(self): print "MainWindowController.onSwap() start" if self.devicesActionMutex.tryLock() and self.sparesActionMutex.tryLock(): self.swapDevicesWindowController = SwapDevicesWindowController(self) print "MainWindowController.onSwap() end" def onInsertNewSpares(self): print "MainWindowController.onInsertNewSpares() start" if self.sparesActionMutex.tryLock(): self.insertNewSparesController = InsertNewSparesController(self) print "MainWindowController.onInsertNewSpares() end" def onUpdateSpares(self): print "MainWindowController.onUpdateSpares() start" if self.sparesActionMutex.tryLock(): self.updateNewSparesController = UpdateNewSparesController(self) print "MainWindowController.onUpdateSpares() end" def onDeleteSpares(self): print "MainWindowController.onDeleteSpares() start" if self.sparesActionMutex.tryLock(): self.deleteSparesWindowController = DeleteSparesWindowController(self) print "MainWindowController.onDeleteSpares() end"
class AnalyzerThread(QThread): def __init__(self, framework, parent = None): QThread.__init__(self, parent) self.framework = framework self.callback_object = None self.qtimer = QTimer() self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.handle_quit) QObject.connect(self, SIGNAL('started()'), self.handle_started) self.Data = None self.read_cursor = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.read_cursor = self.Data.allocate_thread_cursor() self.cursor = self.Data.allocate_thread_cursor() def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.read_cursor: self.read_cursor.close() self.Data.release_thread_cursor(self.read_cursor) self.read_cursor = None if self.cursor: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('do_runAnalysis()'), self.handle_runAnalysis, Qt.DirectConnection) self.exec_() def runAnalysis(self, callback_object): self.callback_object = callback_object QTimer.singleShot(50, self, SIGNAL('do_runAnalysis()')) def handle_quit(self): self.framework.debug_log('AnalyzerThread quit...') self.close_cursor() self.exit(0) def handle_started(self): self.framework.debug_log('AnalyzerThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def handle_runAnalysis(self): fullanalysistext = '' if not self.qlock.tryLock(): self.framework.debug_log('failed to acquire lock for analyzers') else: try: fullanalysistext = self.analyze_content() except Exception as e: self.framework.report_exception(e) finally: self.qlock.unlock() if self.callback_object: self.callback_object.emit(SIGNAL('runAnalysisFinished(QString)'), str(fullanalysistext)) def result_type_to_string(self,resulttype): """Reads type and package information from a result, and returns it as a string""" resultclass=resulttype.__class__ return "".join((resultclass.__module__,".",resultclass.__name__)) def analyze_content(self): """ Perform analysis on the captured content""" #TODO: NEW DB THREAD TO HOLD RESPONSES, ANOTHER FOR WRITING RESULTS scopeController = self.framework.getScopeController() response = self.Data.read_all_responses(self.read_cursor) response_IDs = [] for row in response: dbrow = [m or '' for m in row] Id = dbrow[ResponsesTable.ID] url = dbrow[ResponsesTable.URL] if scopeController.isUrlInScope(url, url): response_IDs.append(Id) #Instantiate all found analyzers analyzerobjects = AnalyzerList(self.framework) analyzerobjects.instantiate_analyzers() analysisrunid=self.Data.analysis_start(self.cursor) #TODO - Consider threading from here down for x in analyzerobjects: #dbconfig=self.Data.get_config_value(self.read_cursor, 'ANALYSIS', str(x.__class__)) #print "dbconfig=%s"%dbconfig print("class=%s"%x.__class__) #x.setConfiguration(dbconfig) x.preanalysis() x.initResultsData() resultinstance=x.getResults() x.analyzerinstanceid=self.Data.analysis_add_analyzer_instance(self.cursor, analysisrunid, str(x.__class__).translate('<>'), x.friendlyname,x.desc, self.result_type_to_string(resultinstance)) fullanalysistext=StringIO() for Id in response_IDs: transaction = self.framework.get_request_response(Id) for analyzer in analyzerobjects: try: analyzer.analyzeTransaction(transaction,analyzer.getResults()) tempanalysisresults=analyzer.getResults() #If there were results for this page, add them to the DB if transaction.Id in tempanalysisresults.pages: pageresultset=self.Data.analysis_add_resultset(self.cursor, analyzer.analyzerinstanceid, transaction.Id,False,transaction.responseUrl, self.result_type_to_string(tempanalysisresults.pages[transaction.Id])) for result in tempanalysisresults.pages[transaction.Id].results: self.Data.analysis_add_singleresult(self.cursor, pageresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) for key,value in list(tempanalysisresults.pages[transaction.Id].stats.items()): self.Data.analysis_add_stat(self.cursor, pageresultset, key, value) except Exception as e: # TODO: add real debugging support self.framework.debug_log('Transaction ID: ' + str(transaction.Id)) self.framework.report_exception(e) #Post Analysis for analyzer in analyzerobjects: results=analyzer.getResults() analyzer.postanalysis(results) for context in list(results.overall.keys()): overallresultset=self.Data.analysis_add_resultset(self.cursor, analyzer.analyzerinstanceid, None,True,context, self.result_type_to_string(results.overall[context])) for result in results.overall[context].results: self.Data.analysis_add_singleresult(self.cursor, overallresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) #print "WRITING:",self.result_type_to_string(result) for key,value in list(results.overall[context].stats.items()): self.Data.analysis_add_stat(self.cursor, overallresultset, key, value) self.Data.commit() #Output results to analysis tab #for analyzer in analyzerobjects: #fullanalysistext.write(analyzer.getResults().toHTML()) return ''
class ResponsesThread(QThread): def __init__(self, framework, treeViewModel, parent=None): QThread.__init__(self, parent) self.framework = framework self.treeViewModel = treeViewModel self.qlock = QMutex() self.cursor = None self.lastId = -1 self.fillAll = False self.doCallback = False self.callbackObj = None QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) self.Data = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.cursor = self.Data.allocate_thread_cursor() self.fillResponses(True) def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.cursor and self.Data: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('doFillResponses()'), self.fillResponsesHandler, Qt.DirectConnection) self.exec_() def quitHandler(self): self.framework.debug_log('ResponsesThread quit...') if self.cursor: self.cursor.close() self.exit(0) def startedHandler(self): self.framework.debug_log('ResponsesThread started...') self.framework.subscribe_response_data_added(self.fillResponsesHandler) self.framework.subscribe_database_events(self.db_attach, self.db_detach) def fillResponses(self, fillAll, callback=None): self.fillAll = fillAll if callback: self.doCallback = True self.callbackObj = callback else: self.doCallback = False QTimer.singleShot(50, self, SIGNAL('doFillResponses()')) def fillResponsesHandler(self, fillAll=False): if self.qlock.tryLock(): try: if self.fillAll: self.fillAll = False self.treeViewModel.clearModel() self.lastId = -1 rows = self.Data.read_newer_responses_info( self.cursor, self.lastId) count = 0 datarows = [] for row in rows: count += 1 if 0 == (count % 100): self.treeViewModel.append_data(datarows) datarows = [] self.yieldCurrentThread() responseItems = [m or '' for m in list(row)] Id = str(row[ResponsesTable.ID]) self.lastId = int(Id) if str(responseItems[ ResponsesTable.CONFIRMED]).lower() in ('y', '1'): confirmed = "Yes" else: confirmed = "" responseItems[ResponsesTable.CONFIRMED] = confirmed datarows.append(responseItems) self.treeViewModel.append_data(datarows) except Exception as error: print(('FIX ME! ERROR: %s' % (traceback.format_exc(error)))) finally: self.qlock.unlock() if self.doCallback: self.doCallback = False self.callbackObj.emit(SIGNAL('fillResponsesFinished()'))