def insert(self, uid, qurlstr, data):
try:
qurlstr = FindingsDatabase.__url_str(qurlstr)
domain = extract_domain(qurlstr)
findingid = FindingsDatabase.__findingid(unicode(qurlstr) + unicode(data))
if findingid not in self.db:
if uid not in self.uid_findingid:
self.uid_findingid[uid] = []
self.uid_findingid[uid].append(findingid)
if qurlstr not in self.qurlstr_findingid:
self.qurlstr_findingid[qurlstr] = []
self.qurlstr_findingid[qurlstr].append(findingid)
# store information in the DB
self.db[findingid] = {
'uid' : uid,
'qurlstr' : qurlstr,
'severity' : data['severity'],
'impact' : data['impact'],
'description' : data['description'],
'reference' : data['reference'],
'domain' : domain,
'trace' : data['trace'],
'type' : data['type']
}
self.emit(SIGNAL('newFindingForURL'), qurlstr)
self.emit(SIGNAL('refreshFindingsView'))
return findingid
except Exception, error:
core.management.logger.exception("FindingsDatabase::insert- Error while inserting a finding (Exception: %s)" % error)
def storeUserInteractionData_Slot(self, eventOrigin, currentFrameUrl,
elmtXPath, buffer, linkFollowedUrl):
qurlstr = SiteInfoStore.__url_str(currentFrameUrl)
request_id = self.netmanager.getNetworkRequestIDFromURL(qurlstr)
domain = extract_domain(qurlstr)
if domain not in self.interaction:
self.interaction[domain] = {}
if qurlstr not in self.interaction[domain]:
self.interaction[domain][qurlstr] = {}
if request_id not in self.interaction[domain][qurlstr]:
self.interaction[domain][qurlstr][request_id] = []
linkFollowedUrl = linkFollowedUrl if not linkFollowedUrl.isEmpty(
) else None
buffer = buffer if 0 < len(buffer) else None
data = {
'url': currentFrameUrl,
'origin': eventOrigin,
'element': elmtXPath,
'buffer': buffer,
'followedLink': linkFollowedUrl
}
self.interaction[domain][qurlstr][request_id].append(data)
self.emit(SIGNAL('newUserInteractionData'), domain, qurlstr,
request_id, eventOrigin, buffer, linkFollowedUrl, elmtXPath)
def addSpideredURL(self, url, comment):
qurlstr = SiteInfoStore.__url_str(url)
domain = extract_domain(qurlstr)
if domain not in self.db:
self.db[domain] = {}
if qurlstr not in self.db[domain]:
self.db[domain][qurlstr] = {
'method': ['GET'],
'request_id': [],
'original': None,
'tampered': None,
'spidered': True,
'content-type': [],
'headers': {},
'get': {},
'post': {},
'cookies': {},
'fragment': [],
'redirected_from': {},
'comment': comment
}
else:
self.db[domain][qurlstr]['spidered'] = True
if self.db[domain][qurlstr]['comment']:
self.db[domain][qurlstr]['comment'] = self.db[domain][qurlstr][
'comment'] + ", " + comment
else:
self.db[domain][qurlstr]['comment'] = comment
if qurlstr not in self.linkSpidered:
self.linkSpidered.append(qurlstr)
def storeUserInteractionData_Slot(self, eventOrigin, currentFrameUrl, elmtXPath, buffer, linkFollowedUrl):
qurlstr = SiteInfoStore.__url_str(currentFrameUrl)
request_id = self.netmanager.getNetworkRequestIDFromURL(qurlstr)
domain = extract_domain(qurlstr)
if domain not in self.interaction:
self.interaction[domain] = {}
if qurlstr not in self.interaction[domain]:
self.interaction[domain][qurlstr] = {}
if request_id not in self.interaction[domain][qurlstr]:
self.interaction[domain][qurlstr][request_id] = []
linkFollowedUrl = linkFollowedUrl if not linkFollowedUrl.isEmpty() else None
buffer = buffer if 0 < len(buffer) else None
data = {'url' : currentFrameUrl, 'origin' : eventOrigin, 'element' : elmtXPath, 'buffer' : buffer, 'followedLink' : linkFollowedUrl}
self.interaction[domain][qurlstr][request_id].append(data)
self.emit(SIGNAL('newUserInteractionData'), domain, qurlstr, request_id, eventOrigin, buffer, linkFollowedUrl, elmtXPath)
def addSpideredURL(self, url, comment):
qurlstr = SiteInfoStore.__url_str(url)
domain = extract_domain(qurlstr)
if domain not in self.db:
self.db[domain] = {}
if qurlstr not in self.db[domain]:
self.db[domain][qurlstr] = {'method' : ['GET'], 'request_id' : [], 'original' : None, 'tampered' : None, 'spidered' : True,
'content-type' : [], 'headers': {}, 'get' : {}, 'post' : {},
'cookies' : {}, 'fragment' : [], 'redirected_from' : {}, 'comment' : comment}
else:
self.db[domain][qurlstr]['spidered'] = True
if self.db[domain][qurlstr]['comment']:
self.db[domain][qurlstr]['comment'] = self.db[domain][qurlstr]['comment'] + ", " + comment
else:
self.db[domain][qurlstr]['comment'] = comment
if qurlstr not in self.linkSpidered:
self.linkSpidered.append(qurlstr)
def rewrite(self, qurl):
if not isinstance(qurl, QUrl):
qurl = QUrl(qurl)
domain = extract_domain(URLRewritingStore.__url_str(qurl))
if not self.hasDomain(domain):
return qurl
original_path = unicode(qurl.path())
upath = None
original_query = str(qurl.encodedQuery())
for urlr_id in self.store[domain]:
if self.store[domain][urlr_id]['active'] and self.store[domain][urlr_id]['regexp'].match(original_path):
upath = self.store[domain][urlr_id]['regexp'].sub(self.store[domain][urlr_id]['replace'], original_path)
if upath != original_path:
break
new_path, new_query = URLRewritingStore.__modified_components(upath)
qurl.setPath(new_path)
qurl.setEncodedQuery(str(original_query) + str("&" + new_query) if 0 < len(new_query) else "")
return qurl
def rewrite(self, qurl):
if not isinstance(qurl, QUrl):
qurl = QUrl(qurl)
domain = extract_domain(URLRewritingStore.__url_str(qurl))
if not self.hasDomain(domain):
return qurl
original_path = unicode(qurl.path())
upath = None
original_query = str(qurl.encodedQuery())
for urlr_id in self.store[domain]:
if self.store[domain][urlr_id]['active'] and self.store[domain][urlr_id]['regexp'].match(original_path):
upath = self.store[domain][urlr_id]['regexp'].sub(self.store[domain][urlr_id]['replace'], original_path)
if upath != original_path:
break
new_path, new_query = URLRewritingStore.__modified_components(upath)
qurl.setPath(new_path)
qurl.setEncodedQuery(str(original_query) + str("&" + new_query) if 0 < len(new_query) else "")
return qurl
def loadFinished_Slot(self):
self.overlay.hide()
self.loadingMessage = False
self.setHistory()
self.animation.stop()
self.stopAction.setIcon(QIcon(self.buttons_off['stop']))
self.stopAction.setEnabled(False)
self.addressLoad.setText(core.management.__release__)
self.addressBar.setText(self.web.url().toString())
self.emit(SIGNAL('newHTMLSource_Signal'),
self.webpage.mainFrame().toHtml())
self.emit(SIGNAL('newDOM_Signal'),
self.webpage.mainFrame().documentElement())
if self.autoDiscovery:
# if the URL auto discovery has been enabled, we will scan the page to get the URL (definitely flawed)
temp_found_urls, found_urls = [], []
web_elmt = self.webpage.currentFrame().documentElement()
for tag in ('a', 'link', 'base', 'area'):
temp_found_urls += self.getElementAttribute(
web_elmt, tag, 'href')
for tag in ('script', 'style', 'embed', 'frame', 'iframe', 'video',
'audio', 'source', 'input'):
temp_found_urls += self.getElementAttribute(
web_elmt, tag, 'src')
current_url = self.web.url()
current_scheme = current_url.scheme()
current_domain = extract_domain(current_url)
current_url = unicode(current_url.toString())
for url in temp_found_urls:
url = WebBrowser.correctURL(url, current_url, current_domain,
current_scheme)
if url and url not in found_urls:
found_urls.append(url)
# register everything to the appinfo, show notification?
for url in found_urls:
self.netmanager.appinfo.addSpideredURL(
url, "URL found by BlackSheep auto-discovery")
self.execute_plugins()
def loadFinished_Slot(self):
self.overlay.hide()
self.loadingMessage = False
self.setHistory()
self.animation.stop()
self.stopAction.setIcon(QIcon(self.buttons_off['stop']))
self.stopAction.setEnabled(False)
self.addressLoad.setText(core.management.__release__)
self.addressBar.setText (self.web.url().toString())
self.emit(SIGNAL('newHTMLSource_Signal'), self.webpage.mainFrame().toHtml())
self.emit(SIGNAL('newDOM_Signal'), self.webpage.mainFrame().documentElement())
if self.autoDiscovery:
# if the URL auto discovery has been enabled, we will scan the page to get the URL (definitely flawed)
temp_found_urls, found_urls = [], []
web_elmt = self.webpage.currentFrame().documentElement()
for tag in ('a', 'link', 'base', 'area'):
temp_found_urls += self.getElementAttribute(web_elmt, tag, 'href')
for tag in ('script', 'style', 'embed', 'frame', 'iframe', 'video', 'audio', 'source', 'input'):
temp_found_urls += self.getElementAttribute(web_elmt, tag, 'src')
current_url = self.web.url()
current_scheme = current_url.scheme()
current_domain = extract_domain(current_url)
current_url = unicode(current_url.toString())
for url in temp_found_urls:
url = WebBrowser.correctURL(url, current_url, current_domain, current_scheme)
if url and url not in found_urls:
found_urls.append(url)
# register everything to the appinfo, show notification?
for url in found_urls:
self.netmanager.appinfo.addSpideredURL(url, "URL found by BlackSheep auto-discovery")
self.execute_plugins()
def extractSiteInfo(self, request_id):
info = self.netmanager.getNetworkHistory(request_id)
if not info:
core.management.logger.error(
"SiteInfoStore: The request ID %s doesn't have any entry in the network history database"
% str(request_id))
return
# store latest request_id
if self.last_request_id < request_id:
self.last_request_id = request_id
qurl = info['request']['url']
qurlstr = SiteInfoStore.__url_str(qurl)
domain = extract_domain(qurlstr)
if domain not in self.db:
self.db[domain] = {}
if qurlstr not in self.db[domain]:
self.db[domain][qurlstr] = {
'method': [],
'request_id': [],
'original': None,
'tampered': None,
'spidered': None,
'content-type': [],
'headers': {},
'get': {},
'post': {},
'cookies': {},
'fragment': [],
'redirected_from': {},
'comment': None
}
if request_id not in self.db[domain][qurlstr]['request_id']:
self.db[domain][qurlstr]['request_id'].append(request_id)
self.db[domain][qurlstr]['tampered'] = qurlstr in self.linkTampered
self.db[domain][qurlstr]['spidered'] = qurlstr in self.linkSpidered
transitive_clicked = False
if info['request']['redirected_from']:
req = info['request']['redirected_from']
redir_info = self.netmanager.getNetworkHistory(req)
if redir_info:
transitive_clicked = True
redir_qurlstr = SiteInfoStore.__url_str(
redir_info['request']['url'])
if redir_qurlstr not in self.db[domain][qurlstr][
'redirected_from']:
self.db[domain][qurlstr]['redirected_from'][
redir_qurlstr] = req
if qurlstr in self.linkClicked or transitive_clicked:
self.db[domain][qurlstr]['original'] = True
if domain not in self.original_requests:
self.original_requests[domain] = {
'last_request_id': request_id,
'originals': []
}
if request_id not in self.original_requests[domain]['originals']:
self.original_requests[domain]['originals'].append(request_id)
# store the latest original request id
if request_id > self.original_requests[domain]['last_request_id']:
self.original_requests[domain]['last_request_id'] = request_id
# store the fragment
fragment = qurl.fragment()
if len(fragment
) and fragment not in self.db[domain][qurlstr]['fragment']:
self.db[domain][qurlstr]['fragment'].append(fragment)
for h in info['request']['headers']:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['headers'],
h[0], h[1])
for h in info['response']['headers']:
if str(h[0]).lower() == "content-type":
content_type = probe_mime(str(h[1]).lower())
if content_type not in self.db[domain][qurlstr][
'content-type']:
self.db[domain][qurlstr]['content-type'].append(
content_type)
queryItems = qurl.queryItems()
if 0 < len(queryItems):
for e in queryItems:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['get'],
e[0], e[1])
post = info['request']['content-QByteArray']
if post and 0 < len(post):
queryItems = SiteInfoStore.__query_items_from_string(post)
for e in queryItems:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['post'],
e[0], e[1])
cookies = info['request']['cookies']
if cookies and 0 < len(cookies):
for e in cookies:
SiteInfoStore.__insert_pair(
self.db[domain][qurlstr]['cookies'], e.name(), e.value())
self.emit(SIGNAL('appInfoStored'), domain, qurlstr, request_id)
def addTreeItem(self, request_id):
info = self.netmanager.getNetworkHistory(request_id)
if not info:
core.management.logger.error("Sitempa::addTreeItem- The request_id %s is not accessible at this time" % str(request_id))
return
qurl = info['request']['url']
scheme = str(qurl.scheme())
if scheme not in ('http', 'https'):
return
host, port, query = qurl.host(), qurl.port(), qurl.queryItems()
path, domain = qurl.path(), extract_domain(Sitemap.__url_str(qurl))
domain_item, path_item = None, None
# find the proper domain node or create it
if domain not in self.map:
self.map[domain] = {}
domain_item = QTreeWidgetItem()
domain_item.setText(0, domain)
domain_item.setIcon(0, QIcon(probe_icon(None, None, 'domain')))
self.map[domain]['item'] = domain_item
self.tree.addTopLevelItem(domain_item)
else:
domain_item = self.map[domain]['item']
# find the host
if host not in self.map[domain]:
self.map[domain][host] = {}
host_item = QTreeWidgetItem()
host_item.setText(0, host)
host_item.setIcon(0, QIcon(probe_icon(None, None, 'host')))
# if no path this was the request page
if str(path) in ('', '/'):
host_item.setText(1, QString(str(request_id)))
domain_item.addChild(host_item)
self.map[domain][host]['item'] = host_item
else:
host_domain = self.map[domain][host]['item']
length = len(path)
if length > 0:
if path[0] == '/':
path = path[1:]
length -= 1
if length > 1 and path[length-1] == '/':
path = path[:length-1]
paths = path.split('/')
d = self.map[domain][host]
cur, prev = None, self.map[domain][host]['item']
length, i = len(paths), 0
for elmt in paths:
i +=1
if len(elmt) < 1:
continue
elif elmt not in d:
d[elmt] = {'item' : QTreeWidgetItem()}
d[elmt]['item'].setText(0, elmt)
d[elmt]['item'].setIcon(0, QIcon(probe_icon(elmt, i == length)))
if i == length:
d[elmt]['item'].setText(1, QString(str(request_id)))
cur = d[elmt]['item']
prev.addChild(cur)
prev = cur
d = d[elmt]
else:
d[elmt]['item'].setIcon(0, QIcon(probe_icon(elmt, i == length)))
if i == length:
d[elmt]['item'].setText(1, QString(str(request_id)))
prev = d[elmt]['item']
d = d[elmt]
def extractSiteInfo(self, request_id):
info = self.netmanager.getNetworkHistory(request_id)
if not info:
core.management.logger.error("SiteInfoStore: The request ID %s doesn't have any entry in the network history database" % str(request_id))
return
# store latest request_id
if self.last_request_id < request_id:
self.last_request_id = request_id
qurl = info['request']['url']
qurlstr = SiteInfoStore.__url_str(qurl)
domain = extract_domain(qurlstr)
if domain not in self.db:
self.db[domain] = {}
if qurlstr not in self.db[domain]:
self.db[domain][qurlstr] = {'method' : [], 'request_id' : [], 'original' : None, 'tampered' : None, 'spidered' : None,
'content-type' : [], 'headers': {}, 'get' : {}, 'post' : {},
'cookies' : {}, 'fragment' : [], 'redirected_from' : {}, 'comment' : None}
if request_id not in self.db[domain][qurlstr]['request_id']:
self.db[domain][qurlstr]['request_id'].append(request_id)
self.db[domain][qurlstr]['tampered'] = qurlstr in self.linkTampered
self.db[domain][qurlstr]['spidered'] = qurlstr in self.linkSpidered
transitive_clicked = False
if info['request']['redirected_from']:
req = info['request']['redirected_from']
redir_info = self.netmanager.getNetworkHistory(req)
if redir_info:
transitive_clicked = True
redir_qurlstr = SiteInfoStore.__url_str(redir_info['request']['url'])
if redir_qurlstr not in self.db[domain][qurlstr]['redirected_from']:
self.db[domain][qurlstr]['redirected_from'][redir_qurlstr] = req
if qurlstr in self.linkClicked or transitive_clicked:
self.db[domain][qurlstr]['original'] = True
if domain not in self.original_requests:
self.original_requests[domain] = {'last_request_id' : request_id, 'originals' : []}
if request_id not in self.original_requests[domain]['originals']:
self.original_requests[domain]['originals'].append(request_id)
# store the latest original request id
if request_id > self.original_requests[domain]['last_request_id']:
self.original_requests[domain]['last_request_id'] = request_id
# store the fragment
fragment = qurl.fragment()
if len(fragment) and fragment not in self.db[domain][qurlstr]['fragment']:
self.db[domain][qurlstr]['fragment'].append(fragment)
for h in info['request']['headers']:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['headers'], h[0], h[1])
for h in info['response']['headers']:
if str(h[0]).lower() == "content-type":
content_type = probe_mime(str(h[1]).lower())
if content_type not in self.db[domain][qurlstr]['content-type']:
self.db[domain][qurlstr]['content-type'].append(content_type)
queryItems = qurl.queryItems()
if 0 < len(queryItems):
for e in queryItems:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['get'], e[0], e[1])
post = info['request']['content-QByteArray']
if post and 0 < len(post):
queryItems = SiteInfoStore.__query_items_from_string(post)
for e in queryItems:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['post'], e[0], e[1])
cookies = info['request']['cookies']
if cookies and 0 < len(cookies):
for e in cookies:
SiteInfoStore.__insert_pair(self.db[domain][qurlstr]['cookies'], e.name(), e.value())
self.emit(SIGNAL('appInfoStored'), domain, qurlstr, request_id)
def addTreeItem(self, request_id):
info = self.netmanager.getNetworkHistory(request_id)
if not info:
core.management.logger.error(
"Sitempa::addTreeItem- The request_id %s is not accessible at this time"
% str(request_id))
return
qurl = info['request']['url']
scheme = str(qurl.scheme())
if scheme not in ('http', 'https'):
return
host, port, query = qurl.host(), qurl.port(), qurl.queryItems()
path, domain = qurl.path(), extract_domain(Sitemap.__url_str(qurl))
domain_item, path_item = None, None
# find the proper domain node or create it
if domain not in self.map:
self.map[domain] = {}
domain_item = QTreeWidgetItem()
domain_item.setText(0, domain)
domain_item.setIcon(0, QIcon(probe_icon(None, None, 'domain')))
self.map[domain]['item'] = domain_item
self.tree.addTopLevelItem(domain_item)
else:
domain_item = self.map[domain]['item']
# find the host
if host not in self.map[domain]:
self.map[domain][host] = {}
host_item = QTreeWidgetItem()
host_item.setText(0, host)
host_item.setIcon(0, QIcon(probe_icon(None, None, 'host')))
# if no path this was the request page
if str(path) in ('', '/'):
host_item.setText(1, QString(str(request_id)))
domain_item.addChild(host_item)
self.map[domain][host]['item'] = host_item
else:
host_domain = self.map[domain][host]['item']
length = len(path)
if length > 0:
if path[0] == '/':
path = path[1:]
length -= 1
if length > 1 and path[length - 1] == '/':
path = path[:length - 1]
paths = path.split('/')
d = self.map[domain][host]
cur, prev = None, self.map[domain][host]['item']
length, i = len(paths), 0
for elmt in paths:
i += 1
if len(elmt) < 1:
continue
elif elmt not in d:
d[elmt] = {'item': QTreeWidgetItem()}
d[elmt]['item'].setText(0, elmt)
d[elmt]['item'].setIcon(
0, QIcon(probe_icon(elmt, i == length)))
if i == length:
d[elmt]['item'].setText(1, QString(str(request_id)))
cur = d[elmt]['item']
prev.addChild(cur)
prev = cur
d = d[elmt]
else:
d[elmt]['item'].setIcon(
0, QIcon(probe_icon(elmt, i == length)))
if i == length:
d[elmt]['item'].setText(1, QString(str(request_id)))
prev = d[elmt]['item']
d = d[elmt]
