def insert(self, uid, qurlstr, data): try: qurlstr = FindingsDatabase.__url_str(qurlstr) domain = extract_domain(qurlstr) findingid = FindingsDatabase.__findingid(unicode(qurlstr) + unicode(data)) if findingid not in self.db: if uid not in self.uid_findingid: self.uid_findingid[uid] = [] self.uid_findingid[uid].append(findingid) if qurlstr not in self.qurlstr_findingid: self.qurlstr_findingid[qurlstr] = [] self.qurlstr_findingid[qurlstr].append(findingid) # store information in the DB self.db[findingid] = { 'uid' : uid, 'qurlstr' : qurlstr, 'severity' : data['severity'], 'impact' : data['impact'], 'description' : data['description'], 'reference' : data['reference'], 'domain' : domain, 'trace' : data['trace'], 'type' : data['type'] } self.emit(SIGNAL('newFindingForURL'), qurlstr) self.emit(SIGNAL('refreshFindingsView')) return findingid except Exception, error: core.management.logger.exception("FindingsDatabase::insert- Error while inserting a finding (Exception: %s)" % error)
def storeUserInteractionData_Slot(self, eventOrigin, currentFrameUrl, elmtXPath, buffer, linkFollowedUrl): qurlstr = SiteInfoStore.__url_str(currentFrameUrl) request_id = self.netmanager.getNetworkRequestIDFromURL(qurlstr) domain = extract_domain(qurlstr) if domain not in self.interaction: self.interaction[domain] = {} if qurlstr not in self.interaction[domain]: self.interaction[domain][qurlstr] = {} if request_id not in self.interaction[domain][qurlstr]: self.interaction[domain][qurlstr][request_id] = [] linkFollowedUrl = linkFollowedUrl if not linkFollowedUrl.isEmpty( ) else None buffer = buffer if 0 < len(buffer) else None data = { 'url': currentFrameUrl, 'origin': eventOrigin, 'element': elmtXPath, 'buffer': buffer, 'followedLink': linkFollowedUrl } self.interaction[domain][qurlstr][request_id].append(data) self.emit(SIGNAL('newUserInteractionData'), domain, qurlstr, request_id, eventOrigin, buffer, linkFollowedUrl, elmtXPath)
def addSpideredURL(self, url, comment): qurlstr = SiteInfoStore.__url_str(url) domain = extract_domain(qurlstr) if domain not in self.db: self.db[domain] = {} if qurlstr not in self.db[domain]: self.db[domain][qurlstr] = { 'method': ['GET'], 'request_id': [], 'original': None, 'tampered': None, 'spidered': True, 'content-type': [], 'headers': {}, 'get': {}, 'post': {}, 'cookies': {}, 'fragment': [], 'redirected_from': {}, 'comment': comment } else: self.db[domain][qurlstr]['spidered'] = True if self.db[domain][qurlstr]['comment']: self.db[domain][qurlstr]['comment'] = self.db[domain][qurlstr][ 'comment'] + ", " + comment else: self.db[domain][qurlstr]['comment'] = comment if qurlstr not in self.linkSpidered: self.linkSpidered.append(qurlstr)
def storeUserInteractionData_Slot(self, eventOrigin, currentFrameUrl, elmtXPath, buffer, linkFollowedUrl): qurlstr = SiteInfoStore.__url_str(currentFrameUrl) request_id = self.netmanager.getNetworkRequestIDFromURL(qurlstr) domain = extract_domain(qurlstr) if domain not in self.interaction: self.interaction[domain] = {} if qurlstr not in self.interaction[domain]: self.interaction[domain][qurlstr] = {} if request_id not in self.interaction[domain][qurlstr]: self.interaction[domain][qurlstr][request_id] = [] linkFollowedUrl = linkFollowedUrl if not linkFollowedUrl.isEmpty() else None buffer = buffer if 0 < len(buffer) else None data = {'url' : currentFrameUrl, 'origin' : eventOrigin, 'element' : elmtXPath, 'buffer' : buffer, 'followedLink' : linkFollowedUrl} self.interaction[domain][qurlstr][request_id].append(data) self.emit(SIGNAL('newUserInteractionData'), domain, qurlstr, request_id, eventOrigin, buffer, linkFollowedUrl, elmtXPath)
def addSpideredURL(self, url, comment): qurlstr = SiteInfoStore.__url_str(url) domain = extract_domain(qurlstr) if domain not in self.db: self.db[domain] = {} if qurlstr not in self.db[domain]: self.db[domain][qurlstr] = {'method' : ['GET'], 'request_id' : [], 'original' : None, 'tampered' : None, 'spidered' : True, 'content-type' : [], 'headers': {}, 'get' : {}, 'post' : {}, 'cookies' : {}, 'fragment' : [], 'redirected_from' : {}, 'comment' : comment} else: self.db[domain][qurlstr]['spidered'] = True if self.db[domain][qurlstr]['comment']: self.db[domain][qurlstr]['comment'] = self.db[domain][qurlstr]['comment'] + ", " + comment else: self.db[domain][qurlstr]['comment'] = comment if qurlstr not in self.linkSpidered: self.linkSpidered.append(qurlstr)
def rewrite(self, qurl): if not isinstance(qurl, QUrl): qurl = QUrl(qurl) domain = extract_domain(URLRewritingStore.__url_str(qurl)) if not self.hasDomain(domain): return qurl original_path = unicode(qurl.path()) upath = None original_query = str(qurl.encodedQuery()) for urlr_id in self.store[domain]: if self.store[domain][urlr_id]['active'] and self.store[domain][urlr_id]['regexp'].match(original_path): upath = self.store[domain][urlr_id]['regexp'].sub(self.store[domain][urlr_id]['replace'], original_path) if upath != original_path: break new_path, new_query = URLRewritingStore.__modified_components(upath) qurl.setPath(new_path) qurl.setEncodedQuery(str(original_query) + str("&" + new_query) if 0 < len(new_query) else "") return qurl
def loadFinished_Slot(self): self.overlay.hide() self.loadingMessage = False self.setHistory() self.animation.stop() self.stopAction.setIcon(QIcon(self.buttons_off['stop'])) self.stopAction.setEnabled(False) self.addressLoad.setText(core.management.__release__) self.addressBar.setText(self.web.url().toString()) self.emit(SIGNAL('newHTMLSource_Signal'), self.webpage.mainFrame().toHtml()) self.emit(SIGNAL('newDOM_Signal'), self.webpage.mainFrame().documentElement()) if self.autoDiscovery: # if the URL auto discovery has been enabled, we will scan the page to get the URL (definitely flawed) temp_found_urls, found_urls = [], [] web_elmt = self.webpage.currentFrame().documentElement() for tag in ('a', 'link', 'base', 'area'): temp_found_urls += self.getElementAttribute( web_elmt, tag, 'href') for tag in ('script', 'style', 'embed', 'frame', 'iframe', 'video', 'audio', 'source', 'input'): temp_found_urls += self.getElementAttribute( web_elmt, tag, 'src') current_url = self.web.url() current_scheme = current_url.scheme() current_domain = extract_domain(current_url) current_url = unicode(current_url.toString()) for url in temp_found_urls: url = WebBrowser.correctURL(url, current_url, current_domain, current_scheme) if url and url not in found_urls: found_urls.append(url) # register everything to the appinfo, show notification? for url in found_urls: self.netmanager.appinfo.addSpideredURL( url, "URL found by BlackSheep auto-discovery") self.execute_plugins()
def loadFinished_Slot(self): self.overlay.hide() self.loadingMessage = False self.setHistory() self.animation.stop() self.stopAction.setIcon(QIcon(self.buttons_off['stop'])) self.stopAction.setEnabled(False) self.addressLoad.setText(core.management.__release__) self.addressBar.setText (self.web.url().toString()) self.emit(SIGNAL('newHTMLSource_Signal'), self.webpage.mainFrame().toHtml()) self.emit(SIGNAL('newDOM_Signal'), self.webpage.mainFrame().documentElement()) if self.autoDiscovery: # if the URL auto discovery has been enabled, we will scan the page to get the URL (definitely flawed) temp_found_urls, found_urls = [], [] web_elmt = self.webpage.currentFrame().documentElement() for tag in ('a', 'link', 'base', 'area'): temp_found_urls += self.getElementAttribute(web_elmt, tag, 'href') for tag in ('script', 'style', 'embed', 'frame', 'iframe', 'video', 'audio', 'source', 'input'): temp_found_urls += self.getElementAttribute(web_elmt, tag, 'src') current_url = self.web.url() current_scheme = current_url.scheme() current_domain = extract_domain(current_url) current_url = unicode(current_url.toString()) for url in temp_found_urls: url = WebBrowser.correctURL(url, current_url, current_domain, current_scheme) if url and url not in found_urls: found_urls.append(url) # register everything to the appinfo, show notification? for url in found_urls: self.netmanager.appinfo.addSpideredURL(url, "URL found by BlackSheep auto-discovery") self.execute_plugins()
def extractSiteInfo(self, request_id): info = self.netmanager.getNetworkHistory(request_id) if not info: core.management.logger.error( "SiteInfoStore: The request ID %s doesn't have any entry in the network history database" % str(request_id)) return # store latest request_id if self.last_request_id < request_id: self.last_request_id = request_id qurl = info['request']['url'] qurlstr = SiteInfoStore.__url_str(qurl) domain = extract_domain(qurlstr) if domain not in self.db: self.db[domain] = {} if qurlstr not in self.db[domain]: self.db[domain][qurlstr] = { 'method': [], 'request_id': [], 'original': None, 'tampered': None, 'spidered': None, 'content-type': [], 'headers': {}, 'get': {}, 'post': {}, 'cookies': {}, 'fragment': [], 'redirected_from': {}, 'comment': None } if request_id not in self.db[domain][qurlstr]['request_id']: self.db[domain][qurlstr]['request_id'].append(request_id) self.db[domain][qurlstr]['tampered'] = qurlstr in self.linkTampered self.db[domain][qurlstr]['spidered'] = qurlstr in self.linkSpidered transitive_clicked = False if info['request']['redirected_from']: req = info['request']['redirected_from'] redir_info = self.netmanager.getNetworkHistory(req) if redir_info: transitive_clicked = True redir_qurlstr = SiteInfoStore.__url_str( redir_info['request']['url']) if redir_qurlstr not in self.db[domain][qurlstr][ 'redirected_from']: self.db[domain][qurlstr]['redirected_from'][ redir_qurlstr] = req if qurlstr in self.linkClicked or transitive_clicked: self.db[domain][qurlstr]['original'] = True if domain not in self.original_requests: self.original_requests[domain] = { 'last_request_id': request_id, 'originals': [] } if request_id not in self.original_requests[domain]['originals']: self.original_requests[domain]['originals'].append(request_id) # store the latest original request id if request_id > self.original_requests[domain]['last_request_id']: self.original_requests[domain]['last_request_id'] = request_id # store the fragment fragment = qurl.fragment() if len(fragment ) and fragment not in self.db[domain][qurlstr]['fragment']: self.db[domain][qurlstr]['fragment'].append(fragment) for h in info['request']['headers']: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['headers'], h[0], h[1]) for h in info['response']['headers']: if str(h[0]).lower() == "content-type": content_type = probe_mime(str(h[1]).lower()) if content_type not in self.db[domain][qurlstr][ 'content-type']: self.db[domain][qurlstr]['content-type'].append( content_type) queryItems = qurl.queryItems() if 0 < len(queryItems): for e in queryItems: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['get'], e[0], e[1]) post = info['request']['content-QByteArray'] if post and 0 < len(post): queryItems = SiteInfoStore.__query_items_from_string(post) for e in queryItems: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['post'], e[0], e[1]) cookies = info['request']['cookies'] if cookies and 0 < len(cookies): for e in cookies: SiteInfoStore.__insert_pair( self.db[domain][qurlstr]['cookies'], e.name(), e.value()) self.emit(SIGNAL('appInfoStored'), domain, qurlstr, request_id)
def addTreeItem(self, request_id): info = self.netmanager.getNetworkHistory(request_id) if not info: core.management.logger.error("Sitempa::addTreeItem- The request_id %s is not accessible at this time" % str(request_id)) return qurl = info['request']['url'] scheme = str(qurl.scheme()) if scheme not in ('http', 'https'): return host, port, query = qurl.host(), qurl.port(), qurl.queryItems() path, domain = qurl.path(), extract_domain(Sitemap.__url_str(qurl)) domain_item, path_item = None, None # find the proper domain node or create it if domain not in self.map: self.map[domain] = {} domain_item = QTreeWidgetItem() domain_item.setText(0, domain) domain_item.setIcon(0, QIcon(probe_icon(None, None, 'domain'))) self.map[domain]['item'] = domain_item self.tree.addTopLevelItem(domain_item) else: domain_item = self.map[domain]['item'] # find the host if host not in self.map[domain]: self.map[domain][host] = {} host_item = QTreeWidgetItem() host_item.setText(0, host) host_item.setIcon(0, QIcon(probe_icon(None, None, 'host'))) # if no path this was the request page if str(path) in ('', '/'): host_item.setText(1, QString(str(request_id))) domain_item.addChild(host_item) self.map[domain][host]['item'] = host_item else: host_domain = self.map[domain][host]['item'] length = len(path) if length > 0: if path[0] == '/': path = path[1:] length -= 1 if length > 1 and path[length-1] == '/': path = path[:length-1] paths = path.split('/') d = self.map[domain][host] cur, prev = None, self.map[domain][host]['item'] length, i = len(paths), 0 for elmt in paths: i +=1 if len(elmt) < 1: continue elif elmt not in d: d[elmt] = {'item' : QTreeWidgetItem()} d[elmt]['item'].setText(0, elmt) d[elmt]['item'].setIcon(0, QIcon(probe_icon(elmt, i == length))) if i == length: d[elmt]['item'].setText(1, QString(str(request_id))) cur = d[elmt]['item'] prev.addChild(cur) prev = cur d = d[elmt] else: d[elmt]['item'].setIcon(0, QIcon(probe_icon(elmt, i == length))) if i == length: d[elmt]['item'].setText(1, QString(str(request_id))) prev = d[elmt]['item'] d = d[elmt]
def extractSiteInfo(self, request_id): info = self.netmanager.getNetworkHistory(request_id) if not info: core.management.logger.error("SiteInfoStore: The request ID %s doesn't have any entry in the network history database" % str(request_id)) return # store latest request_id if self.last_request_id < request_id: self.last_request_id = request_id qurl = info['request']['url'] qurlstr = SiteInfoStore.__url_str(qurl) domain = extract_domain(qurlstr) if domain not in self.db: self.db[domain] = {} if qurlstr not in self.db[domain]: self.db[domain][qurlstr] = {'method' : [], 'request_id' : [], 'original' : None, 'tampered' : None, 'spidered' : None, 'content-type' : [], 'headers': {}, 'get' : {}, 'post' : {}, 'cookies' : {}, 'fragment' : [], 'redirected_from' : {}, 'comment' : None} if request_id not in self.db[domain][qurlstr]['request_id']: self.db[domain][qurlstr]['request_id'].append(request_id) self.db[domain][qurlstr]['tampered'] = qurlstr in self.linkTampered self.db[domain][qurlstr]['spidered'] = qurlstr in self.linkSpidered transitive_clicked = False if info['request']['redirected_from']: req = info['request']['redirected_from'] redir_info = self.netmanager.getNetworkHistory(req) if redir_info: transitive_clicked = True redir_qurlstr = SiteInfoStore.__url_str(redir_info['request']['url']) if redir_qurlstr not in self.db[domain][qurlstr]['redirected_from']: self.db[domain][qurlstr]['redirected_from'][redir_qurlstr] = req if qurlstr in self.linkClicked or transitive_clicked: self.db[domain][qurlstr]['original'] = True if domain not in self.original_requests: self.original_requests[domain] = {'last_request_id' : request_id, 'originals' : []} if request_id not in self.original_requests[domain]['originals']: self.original_requests[domain]['originals'].append(request_id) # store the latest original request id if request_id > self.original_requests[domain]['last_request_id']: self.original_requests[domain]['last_request_id'] = request_id # store the fragment fragment = qurl.fragment() if len(fragment) and fragment not in self.db[domain][qurlstr]['fragment']: self.db[domain][qurlstr]['fragment'].append(fragment) for h in info['request']['headers']: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['headers'], h[0], h[1]) for h in info['response']['headers']: if str(h[0]).lower() == "content-type": content_type = probe_mime(str(h[1]).lower()) if content_type not in self.db[domain][qurlstr]['content-type']: self.db[domain][qurlstr]['content-type'].append(content_type) queryItems = qurl.queryItems() if 0 < len(queryItems): for e in queryItems: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['get'], e[0], e[1]) post = info['request']['content-QByteArray'] if post and 0 < len(post): queryItems = SiteInfoStore.__query_items_from_string(post) for e in queryItems: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['post'], e[0], e[1]) cookies = info['request']['cookies'] if cookies and 0 < len(cookies): for e in cookies: SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['cookies'], e.name(), e.value()) self.emit(SIGNAL('appInfoStored'), domain, qurlstr, request_id)
def addTreeItem(self, request_id): info = self.netmanager.getNetworkHistory(request_id) if not info: core.management.logger.error( "Sitempa::addTreeItem- The request_id %s is not accessible at this time" % str(request_id)) return qurl = info['request']['url'] scheme = str(qurl.scheme()) if scheme not in ('http', 'https'): return host, port, query = qurl.host(), qurl.port(), qurl.queryItems() path, domain = qurl.path(), extract_domain(Sitemap.__url_str(qurl)) domain_item, path_item = None, None # find the proper domain node or create it if domain not in self.map: self.map[domain] = {} domain_item = QTreeWidgetItem() domain_item.setText(0, domain) domain_item.setIcon(0, QIcon(probe_icon(None, None, 'domain'))) self.map[domain]['item'] = domain_item self.tree.addTopLevelItem(domain_item) else: domain_item = self.map[domain]['item'] # find the host if host not in self.map[domain]: self.map[domain][host] = {} host_item = QTreeWidgetItem() host_item.setText(0, host) host_item.setIcon(0, QIcon(probe_icon(None, None, 'host'))) # if no path this was the request page if str(path) in ('', '/'): host_item.setText(1, QString(str(request_id))) domain_item.addChild(host_item) self.map[domain][host]['item'] = host_item else: host_domain = self.map[domain][host]['item'] length = len(path) if length > 0: if path[0] == '/': path = path[1:] length -= 1 if length > 1 and path[length - 1] == '/': path = path[:length - 1] paths = path.split('/') d = self.map[domain][host] cur, prev = None, self.map[domain][host]['item'] length, i = len(paths), 0 for elmt in paths: i += 1 if len(elmt) < 1: continue elif elmt not in d: d[elmt] = {'item': QTreeWidgetItem()} d[elmt]['item'].setText(0, elmt) d[elmt]['item'].setIcon( 0, QIcon(probe_icon(elmt, i == length))) if i == length: d[elmt]['item'].setText(1, QString(str(request_id))) cur = d[elmt]['item'] prev.addChild(cur) prev = cur d = d[elmt] else: d[elmt]['item'].setIcon( 0, QIcon(probe_icon(elmt, i == length))) if i == length: d[elmt]['item'].setText(1, QString(str(request_id))) prev = d[elmt]['item'] d = d[elmt]