def walk(parent, listmode=False, level=0):
stack = []
struct = UnsortableOrderedDict()
(item, cookie) = self.tree.GetFirstChild(parent)
while item:
name = self.tree.GetItemText(item)
if name[:len(self.SPACER)] == self.SPACER:
name = name[len(self.SPACER):-1]
else:
name = name[:-1]
data = self.GetData(item)
#print level, name, type(data), data
if data != None:
if self.tree.ItemHasChildren(item):
result = walk(item,
isinstance(data, list),
level=level + 1)
if listmode:
struct[name] = data
for i in result.keys():
struct[i] = result[i]
stack += [struct]
struct = UnsortableOrderedDict()
else:
struct[name] = result
else:
if listmode:
stack += [UnsortableOrderedDict([(name, data)])]
else:
struct[name] = self.GetData(item)
(item, cookie) = self.tree.GetNextChild(item, cookie)
if listmode:
return stack
else:
return struct
def burp_items_import(xml, requests_and_responses=False):
# Burp Pro (1.6.11)
item_list = []
items = xml.xpath('/items/item')
for item in items:
host = item.xpath('./host')[0].text
method = item.xpath('./method')[0].text
port = item.xpath('./port')[0].text
location = item.xpath('./path')[0].text
scheme = item.xpath('./protocol')[0].text
#post = item.xpath('./post')[0].text
request_element = item.xpath('./request')
if 'base64' in request_element[0].attrib and request_element[0].attrib[
'base64'].lower() == 'true':
request = base64.b64decode(request_element[0].text).replace(
'\r', '')
else:
request = request_element[0].text.replace('\r', '')
post = _extract_post(request, method)
response_element = item.xpath('./response')
if response_element:
if 'base64' in response_element[0].attrib and response_element[
0].attrib['base64'].lower() == 'true':
response = base64.b64decode(response_element[0].text).replace(
'\r', '')
else:
response = response_element[0].text.replace('\r', '')
status_code = item.xpath('./status')[0].text
status_description = ' '.join(
response.split('\n')[0].split(' ')[2:])
else:
response = ''
status_code = ''
status_description = ''
#print scheme,host,port,method,location,post,status_code,status_description
#print request.split('\n')[:3]
#print '-'
#print response.split('\n')[:3]
build = UnsortableOrderedDict()
build['Scheme'] = scheme
build['Host'] = host
build['Port'] = port
build['Method'] = method
build['Location'] = location
build['Post'] = post
build['VulnParam'] = ''
build['StatusCode'] = status_code
build['StatusDescription'] = status_description
if requests_and_responses:
build['Request'] = base64.b64encode(zlib.compress(request))
build['Response'] = base64.b64encode(zlib.compress(response))
#build['Request'] = request
#build['Response'] = response
item_list += [build]
return UnsortableOrderedDict([
['Occurrences', item_list],
])
def walk(data, parent=None, level=0):
#print level, data
if isinstance(data, UnsortableOrderedDict):
for i in data:
item = self.AppendNode(i + ':', '', None, parent)
if parent != None:
self.SetData(parent, UnsortableOrderedDict())
walk(data[i], item, level=level + 1)
elif isinstance(data, list):
if len(data) == 0:
self.SetValue(parent, '')
self.SetData(parent, '')
elif isinstance(data[0], UnsortableOrderedDict):
#keys = data[0].keys()
keys = []
for i in data:
for j in i.keys():
if j not in keys:
keys += [j]
self.SetData(parent, keys)
for i in data:
#print
#print keys, i.keys()
#if i.keys() != keys:
# raise Exception('List keys differ!')
list_item = self.AppendNode(self.SPACER + keys[0] +
':',
'',
None,
parent,
bold=True)
#self.tree.SetPyData(list_item, None)
if self.T:
self.tree.SetItemImage(
list_item,
self.dotlist) #, wx.TreeItemIcon_Normal)
walk(i[keys[0]], list_item, level=level + 1)
for j in keys[1:]:
item = self.AppendNode(j + ':', '', None,
list_item)
if j in i:
walk(i[j], item, level=level + 1)
#else:
# self.SetValue(parent, '')
# self.SetData(parent, '')
else:
if parent != None:
self.SetValue(parent, data)
self.SetData(parent, data)
def burp_import(xml, requests_and_responses=False):
# initially: Burp Suite Pro (1.6beta2 / 1.6.01 used), recently: 1.6.16
issues_list = []
issues = xml.xpath('/issues/issue')
for issue in issues:
full_host = issue.xpath('./host')[0].text
scheme_split = full_host.split('://')
scheme = scheme_split[0]
full_host_parts = scheme_split[1].split(':')
host = full_host_parts[0]
if len(full_host_parts) > 1:
port = int(full_host_parts[-1])
elif scheme.lower() == 'https':
port = 443
else:
port = 80
# remove port if not needed
if scheme.lower() == 'http' and port == 80:
port = ''
if scheme.lower() == 'https' and port == 443:
port = ''
del scheme_split, full_host_parts
request_element = issue.xpath('./requestresponse/request')
if request_element:
request = base64.b64decode(request_element[0].text).replace('\r','')
method = request_element[0].attrib['method']
post = _extract_post(request, method)
else:
request = ''
method = None
post = ''
response_element = issue.xpath('./requestresponse/response')
if response_element:
response = base64.b64decode(response_element[0].text).replace('\r','')
else:
response = ''
method = None
status_parts = response.split('\n')[0].split(' ')
status_code, status_description = (None, None)
if response_element and len(status_parts) > 1:
try:
status_code, status_description = (int(status_parts[1]), ' '.join(status_parts[2:]))
except:
pass
del status_parts
location = ' '.join(request.split('\n')[0].split(' ')[1:-1])
vulnparam = issue.xpath('./location')[0].text[len(issue.xpath('./path')[0].text):]
if vulnparam:
vulnparam = vulnparam[2:-1-10]
if ' ' in vulnparam:
vulnparam = ''
severity = issue.xpath('./severity')[0].text
if severity == 'Information':
severity = 'Informational'
severity_id = ['Informational', 'Low', 'Medium', 'High'].index(severity)
#confidence = issue.xpath('./confidence')[0].text
name = issue.xpath('./name')[0].text
vuln_id = issue.xpath('./type')[0].text
issue_background_element = issue.xpath('./issueBackground')
if issue_background_element:
issue_background = issue_background_element[0].text
else:
issue_background = ''
issue_detail_element = issue.xpath('./issueDetail')
if issue_detail_element:
issue_detail = issue_detail_element[0].text
else:
issue_detail = ''
remediation_background_element = issue.xpath('./remediationBackground')
if remediation_background_element:
remediation_background = remediation_background_element[0].text
else:
remediation_background = ''
report_sections = UnsortableOrderedDict([
['issueBackground', etree.tostring(soupparser.fromstring(issue_background))],
['issueDetail', etree.tostring(soupparser.fromstring(issue_detail))],
['remediationBackground', etree.tostring(soupparser.fromstring(remediation_background))],
])
#if 'Host header poisoning' in name:
#if vuln_id == '134217728':
# print name
for i in report_sections:
report_sections[i] = fine_tune(report_sections[i], i)
issues_item = [
['Severity', severity],
['severity_id', severity_id],
['Name', name],
['vuln_id', vuln_id],
['Scheme', scheme],
['Host', host],
['Port', port],
['Method', method],
['Location', location],
['Post', post],
['VulnParam', vulnparam],
['Example', UnsortableOrderedDict([('VulnParam',vulnparam,),('Request',mangle.request_tune(request),),('Response',mangle.response_tune(response),)])],
]
if requests_and_responses:
issues_item += [
#['Request', base64.b64encode (zlib.compress (request.encode('utf-8')))],
#['Response', base64.b64encode (zlib.compress (response.encode('utf-8')))],
['Request', base64.b64encode (zlib.compress (request))],
['Response', base64.b64encode (zlib.compress (response))],
]
issues_item += [
['StatusCode', status_code],
['StatusDescription', status_description],
#['Classifications', map(lambda x: UnsortableOrderedDict([['Name', x[3]], ['URL', x[2]]]), classifications)],
['ReportSections', UnsortableOrderedDict(
map(lambda x: [x.replace(' ', ''), report_sections[x]], report_sections.keys()))],
]
issues_list += [UnsortableOrderedDict(issues_item)]
findings = []
for vuln_name in sorted(set(map(lambda x: x['Name'], issues_list))):
issue = UnsortableOrderedDict()
for i in filter(lambda x: x['Name'] == vuln_name, issues_list):
for j in ['Name', 'Severity', 'severity_id', 'ReportSections', 'Example']: #, 'Classifications'
if j not in issue:
issue[j] = i[j]
#else:
# if issue[j] != i[j]:
# print j
for j in ['Occurrences']:
if j not in issue:
issue[j] = []
v = UnsortableOrderedDict()
for k in ['Scheme', 'Host', 'Port', 'Method', 'Location', 'Post', 'VulnParam', 'StatusCode', 'StatusDescription']:
v[k] = i[k]
if requests_and_responses:
for k in ['Request','Response']:
v[k] = i[k]
issue[j] += [v]
findings += [issue]
findings.sort(key=lambda x: x['severity_id'], reverse=True)
for i in findings:
del i['severity_id']
return UnsortableOrderedDict([['Findings', findings], ])
def webinspect_import(xml, requests_and_responses=False):
# initially, HP WebInspect (10.1.177.0), recently 10.40
issues_list = []
issues = xml.xpath('/Sessions/Session/Issues/Issue')
for issue in issues:
session = issue.getparent().getparent()
scheme = session.xpath('./Scheme')[0].text
host = session.xpath('./Host')[0].text
port = int(session.xpath('./Port')[0].text)
# remove port if not needed
if scheme.lower() == 'http' and port == 80:
port = ''
if scheme.lower() == 'https' and port == 443:
port = ''
#print scheme, host, port
request = session.xpath('./RawRequest')[0].text
response = session.xpath('./RawResponse')[0].text
method = session.xpath('./Request/Method')[0].text
response_element = session.xpath('./Response')
if response_element:
status_code = int(response_element[0].xpath('./StatusCode')[0].text)
status_description = response_element[0].xpath('./StatusDescription')[0].text
else:
status_code, status_description = (None, None)
#print status_code, status_description
location = ' '.join(request.split('\n')[0].split(' ')[1:-1])
fullurl = scheme+'://'+host+['', ':'+str(port)][bool(port)]+location
#print method, location
if method == 'POST':
# fix tested only for Burp reports:
#post = request.split('\n')[-1]
request_temp = request.replace('\r','')
loc = request_temp.find('\n\n')
if loc != -1:
post = request_temp[loc:].strip()
del request_temp
else:
post = ''
vulnparam = session.xpath('./AttackParamDescriptor')[0].text
if vulnparam == None:
vulnparam = ''
severity_id = int(issue.xpath('./Severity')[0].text)
severity = ['Informational', 'Low', 'Medium', 'High', 'Critical'][severity_id]
name = issue.xpath('./Name')[0].text
if issue.xpath('./CheckTypeID')[0].text == 'Best Practices':
severity = 'Best Practices'
vuln_id = issue.xpath('./VulnerabilityID')[0].text
#print severity,'\t',name
classifications = map(lambda x: [x.attrib['kind'], x.attrib['identifier'], x.attrib['href'], x.text],
issue.xpath('./Classifications/Classification'))
report_sections = map(lambda x: [x.xpath('./Name')[0].text, x.xpath('./SectionText')[0].text],
issue.xpath('./ReportSection'))
for i in range(len(report_sections)):
if report_sections[i][1]:
report_sections[i][1] = fine_tune(etree.tostring(soupparser.fromstring(report_sections[i][1])), fullurl)
#print issue.xpath ('./DetectionSelection/*')
issues_item = [
['Severity', severity],
['severity_id', severity_id],
['Name', name],
['vuln_id', vuln_id],
['Scheme', scheme],
['Host', host],
['Port', port],
['Method', method],
['Location', location],
['Post', post],
['VulnParam', vulnparam],
['Example', UnsortableOrderedDict([('VulnParam',vulnparam,),('Request',mangle.request_tune(request),),('Response',mangle.response_tune(response),)])],
#['Request', request],
]
if requests_and_responses:
issues_item += [
['Request', base64.b64encode (zlib.compress (request.encode('utf-8')))],
['Response', base64.b64encode (zlib.compress (response.encode('utf-8')))],
]
issues_item += [
['StatusCode', status_code],
['StatusDescription', status_description],
['Classifications', map(lambda x: UnsortableOrderedDict([['Name', x[3]], ['URL', '<ihtml><a href="'+x[2]+'">'+x[2]+'</a></ihtml>']]), classifications)],
['ReportSections', UnsortableOrderedDict(map(lambda x: [x[0].replace(' ', ''), x[1]], report_sections))],
]
issues_list += [UnsortableOrderedDict(issues_item)]
findings = []
for vuln_id in sorted(set(map(lambda x: str(x['vuln_id']), issues_list))):
issue = UnsortableOrderedDict()
for i in filter(lambda x: str(x['vuln_id']) == vuln_id, issues_list):
for j in ['Name', 'Severity', 'severity_id', 'ReportSections', 'Classifications', 'Example']:
if j not in issue:
issue[j] = i[j]
#else:
# if issue[j] != i[j]:
# print j
for j in ['Occurrences']:
if j not in issue:
issue[j] = []
v = UnsortableOrderedDict()
for k in ['Scheme', 'Host', 'Port', 'Method', 'Location', 'Post', 'VulnParam', 'StatusCode', 'StatusDescription']:
v[k] = i[k]
if requests_and_responses:
for k in ['Request','Response']:
v[k] = i[k]
issue[j] += [v]
findings += [issue]
findings.sort(key=lambda x: x['severity_id'], reverse=True)
for i in findings:
del i['severity_id']
return UnsortableOrderedDict([['Findings', findings], ])
def burp_import(xml, requests_and_responses=False):
# initially: Burp Suite Pro (1.6beta2 / 1.6.01 used), recently: 1.6.16
issues_list = []
issues = xml.xpath('/issues/issue')
for issue in issues:
full_host = issue.xpath('./host')[0].text
scheme_split = full_host.split('://')
scheme = scheme_split[0]
full_host_parts = scheme_split[1].split(':')
host = full_host_parts[0]
if len(full_host_parts) > 1:
port = int(full_host_parts[-1])
elif scheme.lower() == 'https':
port = 443
else:
port = 80
# remove port if not needed
if scheme.lower() == 'http' and port == 80:
port = ''
if scheme.lower() == 'https' and port == 443:
port = ''
del scheme_split, full_host_parts
request_element = issue.xpath('./requestresponse/request')
if request_element:
request = base64.b64decode(request_element[0].text).replace('\r','')
method = request_element[0].attrib['method']
post = _extract_post(request, method)
else:
request = ''
method = None
post = ''
response_element = issue.xpath('./requestresponse/response')
if response_element:
response = base64.b64decode(response_element[0].text).replace('\r','')
else:
response = ''
method = None
status_parts = response.split('\n')[0].split(' ')
status_code, status_description = (None, None)
if response_element and len(status_parts) > 1:
try:
status_code, status_description = (int(status_parts[1]), ' '.join(status_parts[2:]))
except:
pass
del status_parts
location = ' '.join(request.split('\n')[0].split(' ')[1:-1])
vulnparam = issue.xpath('./location')[0].text[len(issue.xpath('./path')[0].text):]
if vulnparam:
vulnparam = vulnparam[2:-1-10]
if ' ' in vulnparam:
vulnparam = ''
severity = issue.xpath('./severity')[0].text
if severity == 'Information':
severity = 'Informational'
severity_id = ['Informational', 'Low', 'Medium', 'High'].index(severity)
confidence = issue.xpath('./confidence')[0].text
name = issue.xpath('./name')[0].text
vuln_id = issue.xpath('./type')[0].text
issue_background_element = issue.xpath('./issueBackground')
if issue_background_element:
issue_background = issue_background_element[0].text
else:
issue_background = ''
issue_detail_element = issue.xpath('./issueDetail')
if issue_detail_element:
issue_detail = issue_detail_element[0].text
else:
issue_detail = ''
remediation_background_element = issue.xpath('./remediationBackground')
if remediation_background_element:
remediation_background = remediation_background_element[0].text
else:
remediation_background = ''
report_sections = UnsortableOrderedDict([
['issueBackground', mangle.soap_flatten(issue_background)],
['issueDetail', mangle.soap_flatten(issue_detail)],
['remediationBackground', mangle.soap_flatten(remediation_background)],
])
#if 'Host header poisoning' in name:
#if vuln_id == '134217728':
# print name
for i in report_sections:
report_sections[i] = fine_tune(report_sections[i], i)
issues_item = [
['Severity', severity],
['severity_id', severity_id],
['Name', name],
['Confidence', confidence],
['vuln_id', vuln_id],
['Scheme', scheme],
['Host', host],
['Port', port],
['Method', method],
['Location', location],
['Post', post],
['VulnParam', vulnparam],
['Example', UnsortableOrderedDict([('VulnParam',vulnparam,),('Request',mangle.request_tune(request),),('Response',mangle.response_tune(response),)])],
]
if requests_and_responses:
issues_item += [
#['Request', base64.b64encode (zlib.compress (request.encode('utf-8')))],
#['Response', base64.b64encode (zlib.compress (response.encode('utf-8')))],
['Request', base64.b64encode (zlib.compress (request))],
['Response', base64.b64encode (zlib.compress (response))],
]
issues_item += [
['StatusCode', status_code],
['StatusDescription', status_description],
#['Classifications', map(lambda x: UnsortableOrderedDict([['Name', x[3]], ['URL', x[2]]]), classifications)],
['ReportSections', UnsortableOrderedDict(
map(lambda x: [x.replace(' ', ''), report_sections[x]], report_sections.keys()))],
]
issues_list += [UnsortableOrderedDict(issues_item)]
findings = []
for vuln_name in sorted(set(map(lambda x: x['Name'], issues_list))):
issue = UnsortableOrderedDict()
for i in filter(lambda x: x['Name'] == vuln_name, issues_list):
for j in ['Name', 'Severity', 'severity_id', 'Confidence']: #, 'Classifications'
if j not in issue:
issue[j] = i[j]
issue['Summary'] = UnsortableOrderedDict()
issue['Summary']['Description'] = ''
issue['Summary']['Recommendation'] = ''
issue['Description'] = mangle.soap_flatten(issue_detail)
issue['Recommendation'] = mangle.soap_flatten(issue_background)
for j in ['ReportSections', 'Example']:
if j not in issue:
issue[j] = i[j]
for j in ['Occurrences']:
if j not in issue:
issue[j] = []
v = UnsortableOrderedDict()
for k in ['Scheme', 'Host', 'Port', 'Method', 'Location', 'Post', 'VulnParam', 'StatusCode', 'StatusDescription']:
v[k] = i[k]
if requests_and_responses:
for k in ['Request','Response']:
v[k] = i[k]
issue[j] += [v]
findings += [issue]
for j in ['Occurrences']:
if j not in issue:
issue[j] = []
v = UnsortableOrderedDict()
for k in ['Scheme', 'Host', 'Port', 'Method', 'Location', 'Post', 'VulnParam', 'StatusCode', 'StatusDescription']:
v[k] = i[k]
if requests_and_responses:
for k in ['Request','Response']:
v[k] = i[k]
issue[j] += [v]
findings += [issue]
findings.sort(key=lambda x: x['severity_id'], reverse=True)
for i in findings:
del i['severity_id']
#add_extra_fields(findings)
return UnsortableOrderedDict([['Findings', findings], ])
def burp_items_import(xml, requests_and_responses=False):
# Burp Pro (1.6.11)
item_list = []
items = xml.xpath('/items/item')
for item in items:
host = item.xpath('./host')[0].text
method = item.xpath('./method')[0].text
port = item.xpath('./port')[0].text
location = item.xpath('./path')[0].text
scheme = item.xpath('./protocol')[0].text
#post = item.xpath('./post')[0].text
request_element = item.xpath('./request')
if 'base64' in request_element[0].attrib and request_element[0].attrib['base64'].lower() == 'true':
request = base64.b64decode(request_element[0].text).replace('\r','')