def readWebPage( webpageURL ):
webpageURL = webpageURL.strip()
log(VERBOSE_, "readWebpage webpageURL=" + webpageURL)
url = URL(webpageURL)
conn = url.openConnection()
conn.setConnectTimeout(30000)
conn.setReadTimeout(10000)
conn.connect()
responseCode = conn.getResponseCode()
cookie = conn.getHeaderField("Set-Cookie")
cookiePath = None
pathDiscr = " Path="
if cookie and cookie.find(pathDiscr) > 0:
cookiePath = cookie[cookie.index(pathDiscr) + len(pathDiscr):]
respLines = []
if responseCode >= 400:
log(ERROR_, "HTTP ERROR " + `responseCode` + ": " + `conn.getResponseMessage()`)
else:
log(VERBOSE_, "WebPageResponse status=" + `responseCode` + " reason=" + `conn.getResponseMessage()`)
#log(DEBUG_,"WebPageResponse resp="+`resp` )
reader = BufferedReader(InputStreamReader(conn.getInputStream()))
inputLine = reader.readLine()
while inputLine is not None:
respLines.append(inputLine)
inputLine = reader.readLine()
reader.close()
return respLines, cookiePath
def post(targetURL, params, contentType="text/xml", username=None):
if(type(params) is dict):
paramStr = ""
for aKey in params.keys():
paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
paramStr=paramStr[:-1]
else:
paramStr = params
url = URL(targetURL)
print targetURL
print paramStr
print contentType
connection = url.openConnection()
if username!=None:
userpass = username
basicAuth = "Basic " + base64.b64encode(userpass);
connection.setRequestProperty ("Authorization", basicAuth);
connection.setRequestMethod("POST")
if contentType != None:
connection.setRequestProperty("Content-Type", contentType)
connection.setRequestProperty("Content-Length", str(len(paramStr)))
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoInput(1)
connection.setDoOutput(2)
wr= DataOutputStream(connection.getOutputStream())
wr.writeBytes(paramStr)
wr.flush()
wr.close()
return getResponse(connection);
def googleSiteIndex(self, url, mCallBacks, startIndex):
#print 'Starting Google Site: Index for URL: ' + str(url)
data = 'Starting Google Site: Index for URL: ' + str(url) + '\n'
self.parent.printLogTab(str(data))
googleRequest = self.buildGoogleRequest(url, startIndex)
googleResponse = mCallBacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest)
googleStringResponse = googleResponse.tostring()
for urlInSearch in re.findall(r'''<a href="([^<]+)" class=l''', googleStringResponse):
uUrl = URL(urlInSearch)
port = 80
if str(uUrl.getProtocol()) == "https":
port = 443
if mCallBacks.isInScope(uUrl):
newRequest = self.buildGenericRequest(uUrl)
newResponse = mCallBacks.makeHttpRequest(str(uUrl.getHost()), port, (str(uUrl.getProtocol()) == "https"), newRequest)
newResponseString = newResponse.tostring()
firstWord, statusCode, restResponse = newResponseString.split(" ", 2)
requestResponse = HttpRequestResponse.HttpRequestResponse(None, uUrl.getHost(), port, uUrl.getProtocol(), newRequest, newResponse, int(statusCode), uUrl)
mCallBacks.addToSiteMap(requestResponse)
#print "Adding: " + urlInSearch
data = "Adding: " + urlInSearch + "\n"
self.parent.printLogTab(str(data))
else:
data = "\n" + urlInSearch + " was not in scope.\n\n"
self.parent.printLogTab(str(data))
data = "End of Google Site Indexing for URL " + str(url) + "\n"
self.parent.printLogTab(str(data))
def get(targetURL, params, username=None):
paramStr = ""
for aKey in params.keys():
paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
paramStr=paramStr[:-1]
url = URL(targetURL+"?"+paramStr)
print url
connection = url.openConnection()
if username!=None:
userpass = username
basicAuth = "Basic " + base64.b64encode(userpass);
#print basicAuth
connection.setRequestProperty ("Authorization", basicAuth);
connection.setRequestMethod("GET")
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoOutput(2)
inStream= connection.getInputStream()
rd= BufferedReader(InputStreamReader(inStream))
response = ""
line = rd.readLine()
while line != None:
response +=line+"\r"
line = rd.readLine()
rd.close()
return response
def crawl(site, trm , depth, linksfile):
from java.net import URL
from org.w3c.tidy import Tidy
pattern = re.compile('href="/wiki/(.*?)"')
f = open(linksfile, 'a+')
#try:
if depth < MAX_DEPTH:
print 'crawling [%s]...' % trm,
print >> f, '[%s]' % trm
td = Tidy()
td.setXmlOut(1)
u = URL(site + trm)
input = BufferedInputStream(u.openStream())
output = ByteArrayOutputStream()
#tidy.setInputEncoding("UTF8")
#tidy.setOutputEncoding("UTF8")
td.parse(input, output)
content = output.toString()
hits = pattern.findall(content)
for hit in hits:
if hit.find(":") == -1:
print >> f, hit
print 'done.'
print >> f, ''
for hit in hits:
if hit.find(":") == -1:
crawl(site, hit, depth + 1, linksfile)
#except:
# print "wrong"
f.close()
def getUrl(WSDLUrl, containerOSH):
res = ObjectStateHolderVector()
urlIP = None
try:
url = URL(WSDLUrl)
hostName = url.getHost()
urlIP = netutils.getHostAddress(hostName, None)
if (not netutils.isValidIp(urlIP)) or netutils.isLocalIp(urlIP):
urlIP = None
except:
urlIP = None
urlOSH = modeling.createUrlOsh(containerOSH, WSDLUrl, 'wsdl')
urlIpOSH = None
if urlIP != None:
try:
urlIpOSH = modeling.createIpOSH(urlIP)
except:
urlIpOSH = None
res.add(urlOSH)
if urlIpOSH:
res.add(urlIpOSH)
urlToIpOSH = modeling.createLinkOSH('depend', urlOSH, urlIpOSH)
res.add(urlToIpOSH)
return res
def nukeAndPave(islandID, wipeMasters=False, wipeSlaves=False):
if not wipeMasters and not wipeSlaves:
return
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
slaves = []
masters = []
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
slaveHost = slaveURL.getHost()
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
slaves.append(slaveHost)
masters.append(masterHost)
rc = 0
if wipeSlaves:
print "Removing Solr from slaves to prepare for upgrade", slaves
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-install"], slaves)
if failures.isEmpty():
print "INFO: nuke and pave of slaves succeeded"
else:
print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures
rc = 1
return rc
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
urlString = Framework.getParameter(PARAM_URL)
reportPoweredOffVms = 0
reportPoweredOffVmsValue = Framework.getParameter(PARAM_REPORT_POWERED_OFF_VMS)
if reportPoweredOffVmsValue and reportPoweredOffVmsValue.lower() =='true':
reportPoweredOffVms = 1
ipAddress = None
try:
urlObject = URL(urlString)
hostname = urlObject.getHost()
if not hostname:
logger.debug("Hostname is not defined in URL '%s'" % urlString)
raise MalformedURLException()
ipAddress = vcloud_discover.getIpFromUrlObject(urlObject)
if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress):
msg = "Failed to resolve the IP address of server from specified URL"
errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
return OSHVResult
except MalformedURLException:
msg = "Specified URL '%s' is malformed" % urlString
errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
except:
msg = logger.prepareJythonStackTrace("")
errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
else:
#configure how connections should be discovered/established
connectionDiscoverer = vcloud_discover.ConnectionDiscoverer(Framework)
urlGenerator = vcloud_discover.ConstantUrlGenerator(urlString)
connectionDiscoverer.setUrlGenerator(urlGenerator)
connectionDiscoverer.addIp(ipAddress)
#configure how established/failed connection should be used
connectionHandler = vcloud_discover.BaseDiscoveryConnectionHandler(Framework)
topologyDiscoverer = vcloud_discover.createVcloudDiscoverer(Framework)
topologyReporter = vcloud_report.createVcloudReporter(Framework, None, reportPoweredOffVms)
connectionHandler.setDiscoverer(topologyDiscoverer)
connectionHandler.setReporter(topologyReporter)
connectionDiscoverer.setConnectionHandler(connectionHandler)
connectionDiscoverer.initConnectionConfigurations()
connectionDiscoverer.discover(firstSuccessful=0)
if not connectionHandler.connected:
for errorMsg in connectionHandler.connectionErrors:
Framework.reportError(errorMsg)
for warningMsg in connectionHandler.connectionWarnings:
Framework.reportWarning(warningMsg)
return OSHVResult
def isUrlAvailable(url, acceptedStatusCodeRange, timeout=10000):
'''
Checks whether url is available
str, list(str), int -> bool
'''
from com.hp.ucmdb.discovery.library.clients import SSLContextManager
from com.hp.ucmdb.discovery.library.clients.http import ApacheHttpClientWrapper as HttpClientWrapper
if not url or not acceptedStatusCodeRange:
return 0
with _create_http_client_wrapper() as client:
client.setSocketTimeout(timeout)
try:
jurl = URL(url)
if jurl.getProtocol() == 'https':
port = jurl.getPort() or HttpClientWrapper.DEFAULT_HTTPS_PORT
context = SSLContextManager.getAutoAcceptSSLContext()
client.registerProtocol(context, port)
except:
logger.warn('Failed parsing url % ' % url)
try:
httpResult = client.get(url)
return httpResult.statusCode in acceptedStatusCodeRange
except:
logger.warn('Get Failed: %s' % logger.prepareJavaStackTrace())
return 0
def __init__(self, queryString):
self.args = queryString
# We would like content_type to be a class property but this is
# not supported in Python 2.1
self.content_type = "text/plain"
reqURL = URL("http://myhost/mywms/")
self.server = FakeModPythonServerObject("%s:%d" % (reqURL.getHost(), reqURL.getPort()))
self.unparsed_uri = str(reqURL.getPath())
def delete(targetURL, params):
url = URL(targetURL)
connection = url.openConnection()
connection.setRequestMethod("DELETE")
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoOutput(2)
return getResponse(connection);
def createURLOSHV(urlString, framework = None):
OSHVResult = ObjectStateHolderVector()
#urlOSH2 = modeling.createOshByCmdbIdString('uri_endpoint', urlId)
logger.debug("Starting URL discovery on '%s'" % urlString)
#urlString = urlString[1:len(urlString)-1]
if not urlString:
return OSHVResult
try:
urlString = str(urlString).replace("\\", "//")
urlObject = URL(urlString)
hostname = urlObject.getHost()
if not hostname:
raise MalformedURLException("Hostname is not defined in URL '%s'" % urlString)
urlObjectResolver = URLObjectResolver(urlObject)
protocol = urlObjectResolver.getProtocolFromUrlObject()
if not protocol:
raise Exception("Failed to resolve the http/https protocol from specified URL")
port = urlObjectResolver.getPortFromUrlObject()
if not port:
raise Exception("Failed to resolve the port number from specified URL")
# get topology
# create business element CI and attach the url as configuration document CI to it
ips = urlObjectResolver.getIpFromUrlObject()
for ipAddress in ips:
logger.debug('%s: Reporting ip address: %s' % (urlString, ipAddress))
if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress):
raise Exception("Failed to resolve the IP address of server from specified URL")
hostOSH, ipOSH, OSHVResult2 = createIPEndpointOSHV(framework, ipAddress, port, protocol, hostname)
OSHVResult.addAll(OSHVResult2)
# create UriEndpoint and relations between business element and UriEndpoint
urlOSH = modeling.createUrlOsh(hostOSH, urlString, None)
#urlOSH.setCmdbObjectId(urlOSH2.getCmdbObjectId())
OSHVResult.add(urlOSH)
OSHVResult.add(modeling.createLinkOSH('dependency', urlOSH, ipOSH))
#create Web Server
except:
logger.warnException("Error creating URL OSH for %s" % urlString)
return OSHVResult
def _get_map_file(self):
map_param = self.getParameter('map')
map_url = URL(self.getDocumentBase(), map_param)
self.showStatus('Loading ' + map_url.toString())
map_is = map_url.openStream()
map_buf = ""
i = map_is.read()
while i != -1:
map_buf = map_buf + chr(i)
i = map_is.read()
return StringIO(map_buf)
def doService(httpMethod, url, credential, requestBody=None):
Security.addProvider(MySSLProvider())
Security.setProperty("ssl.TrustManagerFactory.algorithm", "TrustAllCertificates")
HttpsURLConnection.setDefaultHostnameVerifier(MyHostnameVerifier())
urlObj = URL(url)
con = urlObj.openConnection()
con.setRequestProperty("Accept", "application/xml")
con.setRequestProperty("Content-Type", "application/xml")
con.setRequestProperty("Authorization", credential)
con.setDoInput(True);
if httpMethod == 'POST':
con.setDoOutput(True)
con.setRequestMethod(httpMethod)
output = DataOutputStream(con.getOutputStream());
if requestBody:
output.writeBytes(requestBody);
output.close();
responseCode = con.getResponseCode()
logger.info('response code: ' + str(responseCode))
responseMessage = con.getResponseMessage()
logger.info('response message: ' + str(responseMessage))
contentLength = con.getHeaderField('Content-Length')
logger.info('content length: ' + str(contentLength))
stream = None
if responseCode == 200 or responseCode == 201 or responseCode == 202:
stream = con.getInputStream()
elif contentLength:
stream = con.getErrorStream()
if stream:
dataString = getStreamData(stream)
logger.info(httpMethod + ' url: ' + url)
if not url.endswith('.xsd') and len(dataString) < 4096:
xmlStr = Util.prettfyXmlByString(dataString)
logger.info(httpMethod + ' result: \n\n' + xmlStr)
else:
logger.info('response body too big, no print out')
if responseCode == 200 or responseCode == 201 or responseCode == 202:
return dataString
else:
''' to mark the case failed if response code is not 200-202 '''
return None
else:
logger.error('')
logger.error('---------------------------------------------------------------------------------------------------')
logger.error('-------->>> Input or Error stream is None, it may be a defect if it is positive test case')
logger.error('---------------------------------------------------------------------------------------------------')
logger.error('')
return None
def new_urlopen(req):
if isinstance(req, str):
full_url = req
else:
full_url = req.get_full_url()
if full_url.startswith('https'):
u = URL(full_url)
conn = u.openConnection()
return ConnReader(conn)
else:
org_urlopen(req)
def addAuthentication(self, httpClient, method):
BasicAuthStrategy.addAuthentication(self, httpClient, method)
url = URL(self.__baseUrl)
proxy = ProxySelector.getDefault().select(url.toURI()).get(0)
httpClient.getParams().setAuthenticationPreemptive(False);
if not proxy.type().equals(Proxy.Type.DIRECT):
address = proxy.address()
proxyHost = address.getHostName()
proxyPort = address.getPort()
httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort)
print "Using proxy '%s:%s'" % (proxyHost, proxyPort)
def __init__(self, request, response):
self._response = response
self.args = request.getQueryString()
# We would like content_type to be a class property but this is
# not supported in Python 2.1
self.content_type = "text/plain"
reqURL = URL(request.getRequestURL().toString())
self.server = FakeModPythonServerObject("%s:%d" % (reqURL.getHost(), reqURL.getPort()))
self.unparsed_uri = str(reqURL.getPath())
self.headers_out = {} # Dictionary of HTTP headers
self.headers_set = 0
def reportToToolServer(self, url):
"""Tell to the tool server when an error is fixed (user clicked
on correctedBtn) or false positive (falsePositiveBtn).
Tool instance must have either:
self.fixedFeedbackMode = "url"
self.falseFeedbackMode = "url"
"""
try:
url = URL(url)
uc = url.openConnection()
uc.getInputStream()
uc.disconnect()
except (UnknownHostException, IOException, SocketException):
print url
print "* I can't connect to the tool server."
def getIpFromUrlString(urlString):
try:
urlObject = URL(urlString)
hostname = urlObject.getHost()
if not hostname:
logger.debug("Hostname is not defined in URL '%s'" % urlString)
raise MalformedURLException()
ipAddress = getIpFromUrlObject(urlObject)
if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress):
raise ValueError("Failed determining the IP address of server")
return ipAddress
except:
logger.warnException("Invalid URL")
def processCLI(self): cli = self._callbacks.getCommandLineArguments() if len(cli) < 0: print "Incomplete target information provided." return False elif not cli: print "Integris Security Carbonator is now loaded." print "If Carbonator was loaded through the BApp store then you can run in headless mode simply adding the `-Djava.awt.headless=true` flag from within your shell. Note: If burp doesn't close at the conclusion of a scan then disable Automatic Backup on Exit." print "For questions or feature requests contact us at carbonator at integris security dot com." print "Visit carbonator at https://www.integrissecurity.com/Carbonator" return False else: self.url = URL(cli[0]) self.rtype = cli[1] self.output = cli[2] self.scheme = self.url.getProtocol() self.fqdn = self.url.getHost() self.port1 = self.url.getPort() if self.port1 == -1 and self.scheme == 'http': self.port = 80 elif self.port1 == -1 and self.scheme == 'https': self.port = 443 else: self.port = self.port1 self.path = self.url.getFile() print "self.url: " + str(self.url) + "\n" print "Scheme: " + self.scheme + "\n" print "FQDN: " + self.fqdn + "\n" print "Port: " + str(self.port1) + "\n" print "Path: " + self.path + "\n" return True
def send_to_josm(self, url):
"""Remote control command to make JOSM download the area nearby
the error
"""
try:
url = URL(url)
uc = url.openConnection()
uc.getInputStream()
return True
except UnknownHostException:
return False
except FileNotFoundException:
return False
except SocketException:
print "\n* Please, enable JOSM remote from Preferences"
return False
def connect(self, url):
if re.findall('(?:\:\d{2,5})', url) and self.urlRegex.match(url):
try:
javaURL = URL(url)
newRequest = self._helpers.buildHttpRequest(javaURL)
requestResponse = self._callbacks.makeHttpRequest(self._helpers.buildHttpService(str(javaURL.getHost()), javaURL.getPort(), str(javaURL.getProtocol())), newRequest)
# Follow redirects if a 301 or 302 response is received. As of Oct 9 2014 the API is not capable of this: http://forum.portswigger.net/thread/1504/ask-burp-follow-redirections-extension
response = requestResponse.getResponse()
if response:
requestInfo = self._helpers.analyzeResponse(response)
responseHeader = requestInfo.getHeaders()
if ('301' in responseHeader[0] or '302' in responseHeader[0]) and self.redirectsCheckbox.isSelected():
self.redirectCounter += 1
for headerLine in responseHeader:
if 'Location: ' in headerLine or 'location: ' in headerLine:
url = self.locationHeaderConvert(str(headerLine.split(' ')[1]), str(javaURL.getPort()), str(javaURL.getHost()), '')
self.connect(url)
self.goodUrlList.append(url)
self._callbacks.addToSiteMap(requestResponse)
else:
self.badUrlList.append(url)
except:
self.badUrlList.append(url)
else:
if 'http://' in url:
fixedUrl = self.addPort(url, '80')
self.connect(fixedUrl)
elif 'https://' in url:
fixedUrl = self.addPort(url, '443')
self.connect(fixedUrl)
else:
self.badUrlList.append(url)
def delete(targetURL, params):
url = URL(targetURL)
connection = url.openConnection()
connection.setRequestMethod("DELETE")
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoOutput(2)
inStream= connection.getInputStream()
rd= BufferedReader(InputStreamReader(inStream))
response = ""
line = rd.readLine()
while line != None:
response +=line+"\r"
line = rd.readLine()
rd.close()
return response
def make_jar_classloader(jar):
import os
from java.net import URL, URLClassLoader
url = URL('jar:file:%s!/' % jar)
if os._name == 'nt':
# URLJarFiles keep a cached open file handle to the jar even
# after this ClassLoader is GC'ed, disallowing Windows tests
# from removing the jar file from disk when finished with it
conn = url.openConnection()
if conn.getDefaultUseCaches():
# XXX: Globally turn off jar caching: this stupid
# instance method actually toggles a static flag. Need a
# better fix
conn.setDefaultUseCaches(False)
return URLClassLoader([url])
def upgradeMaster(islandID):
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
hosts = []
for loc in locs:
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
hosts.append(masterHost)
service = Service("ems-solr")
service.setThreadCount(1)
service.invoke("stop", hosts)
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "save-index"], hosts)
if not failures.isEmpty():
print "WARN: Failed to backup the index on all hosts. Check logs.", failures
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-index"], hosts)
if not failures.isEmpty():
print "Failed to start the backup job running on all hosts. Check logs.", failures
return 1
failures = runJob(
["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-status"],
hosts,
sleepTime=120,
maxWaits=43200,
interimResult=3,
)
if not failures.isEmpty():
print "ERROR: upgrade job failed. Upgrade aborted", failures
return 1
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts)
if not failures.isEmpty():
print "ERROR: unable to remove solr service from all hosts", failures
return 1
print "Please check for any errors in the output before proceeding"
print "If it appears the Solr index has been upgraded successfully. Please proceed to install the new version of Solr and then run the validate-master step"
return 0
def googleSiteIndex_run(querys, maxIndex):
for i in querys:
try:
maxIndex = int(maxIndex)
except:
self.appendToResults("Index Value was not a valid Integer\n")
return
# setup counts to stay within provided index range
resultsCount = 0
currentIndex = 0
previousIndex = -1
self.appendToResults("Starting Google Hack for " + i.strip() + "\n")
while (currentIndex < maxIndex) and (previousIndex != currentIndex):
previousIndex = currentIndex
googleRequest = self.buildGoogleRequest(i, currentIndex)
try:
googleResponse = self._callbacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest).tostring()
except:
self.appendToResults("Call to google was not made: (Could not make a connection)\n")
return
if re.findall(r'<a href="([^<]+)" class=l', googleResponse) and resultsCount < maxIndex:
currentIndex += 100
for urlInSearch in re.findall(r'<a href="([^<]+)" class=l', googleResponse):
if resultsCount < maxIndex:
uUrl = URL(urlInSearch)
port = 80
if str(uUrl.getProtocol()) == "https":
port = 443
if self._callbacks.isInScope(uUrl):
newRequest = self.buildGenericRequest(uUrl)
try:
requestResponse = self._callbacks.makeHttpRequest(self._helpers.buildHttpService(str(uUrl.getHost()), port, str(uUrl.getProtocol()) == "https"), newRequest)
self._callbacks.addToSiteMap(requestResponse)
resultsCount += 1
self.appendToResults("Adding " + urlInSearch + " to Target:\n")
except:
self.appendToResults("Call to URL found was not made: (Could not make a connection)\n")
else:
self.appendToResults(urlInSearch + " was found but not in Scope (Not Adding to Target)\n")
else:
previousIndex = currentIndex
self.appendToResults("Reached end of query " + i.strip() + " with " + str(resultsCount) + " results found\n\n")
def post(targetURL, params, contentType="text/xml", username=None):
#paramStr = params["data"]
paramStr = ""
for aKey in params.keys():
paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
paramStr=paramStr[:-1]
url = URL(targetURL)
print targetURL
print paramStr
print contentType
url = URL(targetURL+"?"+paramStr)
print url
connection = url.openConnection()
if username!=None:
userpass = username
basicAuth = "Basic " + base64.b64encode(userpass);
connection.setRequestProperty ("Authorization", basicAuth);
connection.setRequestMethod("POST")
if contentType != None:
connection.setRequestProperty("Content-Type", contentType)
connection.setRequestProperty("Content-Length", str(len(paramStr)))
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoInput(1)
connection.setDoOutput(2)
#wr= DataOutputStream(connection.getOutputStream())
#wr.writeBytes(paramStr)
#wr.flush()
#wr.close()
inStream= connection.getInputStream()
rd= BufferedReader(InputStreamReader(inStream))
response = ""
line = rd.readLine()
while line != None:
response +=line+"\r"
line = rd.readLine()
rd.close()
return response
def __init__(self):
sessionDuration = grinder.properties.getInt("player.sessionDurationInSeconds", 0)
self.password = grinder.properties.getProperty("player.password")
players_url = grinder.properties.getProperty("player.source-url")
passwordHash = URLEncoder.encode(grinder.properties.getProperty("player.passwordHash"), "UTF-8")
workers = grinder.properties.getInt("grinder.processes", 0)
threads = grinder.properties.getInt("grinder.threads", 0)
workerIndex = grinder.processNumber - grinder.firstProcessNumber
threadsPerAgent = workers * threads
agentIndex = grinder.agentNumber
start = agentIndex * threadsPerAgent + workerIndex * threads
if isLogEnabled:
log("Worker will handle %s players starting from %d" % (threads, start))
playerRange = (start, threads)
self.workerPlayers = []
params = "passwordHash=%s&minimumBalance=%s&startRow=%s&limit=%s" % (
passwordHash,
str(sessionDuration / 5),
str(playerRange[0]),
str(threads),
)
urlStr = players_url + "?" + params
try:
data = StringBuffer()
url = URL(urlStr)
conn = url.openConnection()
rd = BufferedReader(InputStreamReader(conn.getInputStream()))
line = rd.readLine()
while line is not None:
data.append(line)
line = rd.readLine()
rd.close()
if isLogEnabled:
log(data.toString())
message = JSONValue.parse(str(String(data.toString(), "UTF-8")))
for entry in message:
self.workerPlayers.append((entry.get(0), entry.get(1)))
except Exception:
raise Exception("Couldn't fetch players from %s: %s" % (urlStr, traceback.format_exc()))
if isLogEnabled:
log(str(self.workerPlayers))
def get(targetURL, params, username=None):
paramStr = ""
for aKey in params.keys():
paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
paramStr=paramStr[:-1]
url = URL(targetURL+"?"+paramStr)
print url
connection = url.openConnection()
if username!=None:
userpass = username
basicAuth = "Basic " + base64.b64encode(userpass);
print basicAuth
connection.setRequestProperty ("Authorization", basicAuth);
connection.setRequestMethod("GET")
connection.setRequestProperty("Content-Language", "en-GB")
connection.setUseCaches(0)
connection.setDoOutput(2)
return getResponse(connection);
def call(self):
consumer = DefaultOAuthConsumer(CONSUMER_KEY, CONSUMER_SECRET);
provider = DefaultOAuthProvider(
REQUEST_TOKEN_URL, ACCESS_TOKEN_URL, AUTHORIZE_URL
);
print "Fetching request token from PassaporteWeb..."
authUrl = provider.retrieveRequestToken(consumer, CALLBACK_URL);
authUrl = OAuth.addQueryParameters(authUrl, OAuth.OAUTH_CONSUMER_KEY, CONSUMER_KEY);
print "Request token: ", consumer.getToken()
print "Token secret: ", consumer.getTokenSecret()
print "Now visit:\n", authUrl, "\n... and grant this app authorization"
print "After granting authorization, you should be redirected to the callback url"
pin = raw_input("Enter the PIN code and hit ENTER when you're done:")
print "Fetching access token from PassaporteWeb..."
provider.retrieveAccessToken(consumer, pin);
print "Access token: ", consumer.getToken()
print "Token secret: ", consumer.getTokenSecret()
url = URL(USER_DATA);
request = url.openConnection();
consumer.sign(request);
print "Sending request..."
request.connect();
print "Response: %d %s" % (request.getResponseCode(), request.getResponseMessage())
def bing_query(self, bing_query_string):
'''query bing'''
print 'Performing Bing search: %s' % bing_query_string
# encode our query
quoted_query = urllib.quote(bing_query_string)
my_key = base64.b64encode(':%s' % self.get_my_key())
host = 'api.datamarket.azure.com'
host = 'api.cognitive.microsoft.com'
base_url = 'https://%s/Bing/Search/Web' % host
base_url = 'https://%s/bing/v5.0/search?q='
agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
http_request = 'GET %s?$format=json&$top=20&Query=%s' % (base_url, quoted_query)
http_request += ' HTTP/1.1\r\n'
http_request += 'Host: %s\r\n' % host
http_request += 'Connection: close\r\n'
http_request += 'Authorization: Basic %s\r\n' % my_key
http_request += 'User-Agent: %s' % agent
print 'Making http_request...',
json_body = self._callbacks.makeHttpRequest(host, 443, True, http_request).tostring()
json_body = json_body.split('\r\n\r\n', 1)[1]
print 'done.'
try:
r = json.loads(json_body)
print str(r)
if len(r['d']['results']):
for site in r['d']['results']:
print '*' * 100
print site['Title']
print site['Url']
print site['Description']
print '*' * 100
j_url = URL(site['Url'])
if not self._callbacks.isInScope(j_url):
print 'Adding to Burp Scope'
self._callbacks.includeInScope(j_url)
except Exception as err:
print str(err)
#print 'No results from Bing'
return
def fuzzUrlsAPI(self):
print 'Fuzzing %s viable URLs with a wsdl extension\n' % len(
self.detectedUrlList)
for messageUrl in self.detectedUrlList:
wsdlSuffixUrl = messageUrl + '?wsdl'
if wsdlSuffixUrl not in self.fuzzedWsdlList and wsdlSuffixUrl not in self.foundWsdlList:
# Build full HTTP URL with port
fullUrl = self.originalMsgUrl + ':' + str(self.originalMsgPort)
# Build the URL path without domain to be used in the GET request
requestWsdlUrl = wsdlSuffixUrl[len(fullUrl):]
# Build HTTP request string to be used with Burp API
requestString = """GET %s HTTP/1.1\nHost: %s\r\n\r\n""" % (
requestWsdlUrl, self.originalMsgHost)
# Make request of the generated wsdl URL using Burp API
response = self._callbacks.makeHttpRequest(
self.originalMsgHost, self.originalMsgPort,
self.originalMsgProtocol == "https",
self._helpers.stringToBytes(requestString))
wsdlSuffixUrlResponseData = self._helpers.bytesToString(
response)
# Check if response is valid
if self._helpers.analyzeResponse(
response).getStatusCode() == 200:
for wsdlKeyword in self.wsdlKeywordList:
# Loop through the keywords and check if one of them are in the response (very good indicator of wsdl or similar file)
if wsdlKeyword in wsdlSuffixUrlResponseData:
# wsdl file found from created URL, add to fuzzed list
self.fuzzedWsdlList.append(wsdlSuffixUrl)
# Build a request and response based on the found wsdl URL and add them to the Burp site map
wsdlJavaURL = URL(wsdlSuffixUrl)
newRequest = self._helpers.buildHttpRequest(
wsdlJavaURL)
requestResponse = self._callbacks.makeHttpRequest(
self._helpers.buildHttpService(
str(self.originalMsgHost),
int(self.originalMsgPort),
self.originalMsgProtocol == "https"),
newRequest)
self._callbacks.addToSiteMap(requestResponse)
break
def bing_query(self, bing_query_string):
print "Performing Bing search: %s" % bing_query_string
# encode our query
quoted_query = urllib.quote(bing_query_string)
http_request = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query
http_request += "Host: api.datamarket.azure.com\r\n"
http_request += "Connection: close\r\n"
http_request += "Authorization: Basic %s\r\n" % base64.b64encode(
":%s" % bing_api_key)
http_request += "User-Agent: Blackhat Python\r\n\r\n"
json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com",
443, True,
http_request).tostring()
json_body = json_body.split("\r\n\r\n", 1)[1]
try:
r = json.loads(json_body)
if len(r["d"]["results"]):
for site in r["d"]["results"]:
print "*" * 100
print site['Title']
print site['Url']
print site['Description']
print "*" * 100
j_url = URL(site['Url'])
if not self._callbacks.isInScope(j_url):
print "Adding to Burp scope"
self._callbacks.includeInScope(j_url)
except:
print "No results from Bing"
pass
return
def post(self, url):
if re.findall('(?:\:\d{2,5})', url) and self.urlRegex.match(url):
try:
javaURL = URL(url)
newRequest = self._helpers.buildHttpRequest(javaURL)
newRequest = self._helpers.toggleRequestMethod(
newRequest) # for post requests
requestResponse = self._callbacks.makeHttpRequest(
self._helpers.buildHttpService(str(javaURL.getHost()),
javaURL.getPort(),
str(javaURL.getProtocol())),
newRequest)
# Follow redirects if a 301 or 302 response is received. As of Oct 9 2014 the API is not capable of this: http://forum.portswigger.net/thread/1504/ask-burp-follow-redirections-extension
response = requestResponse.getResponse()
if response:
requestInfo = self._helpers.analyzeResponse(response)
responseHeader = requestInfo.getHeaders()
if ('301' in responseHeader[0]
or '302' in responseHeader[0]
) and self.redirectsCheckbox.isSelected():
self.redirectCounter += 1
for headerLine in responseHeader:
if 'Location: ' in headerLine or 'location: ' in headerLine:
url = self.locationHeaderConvert(
str(headerLine.split(' ')[1]),
str(javaURL.getPort()),
str(javaURL.getHost()), '')
self.post(url)
self.goodUrlList.append(url)
self._callbacks.addToSiteMap(requestResponse)
else:
self.badUrlList.append(url)
except:
self.badUrlList.append(url)
else:
if 'http://' in url:
fixedUrl = self.addPort(url, '80')
self.post(fixedUrl)
elif 'https://' in url:
fixedUrl = self.addPort(url, '443')
self.post(fixedUrl)
else:
self.badUrlList.append(url)
def processProxyMessage(self, messageReference, messageIsRequest,
remoteHost, remotePort, serviceIsHttps, httpMethod,
path, resourceType, statusCode,
responseContentType, message, interceptAction):
self.loader.reloadIfChanged()
url = URL("HTTPS" if serviceIsHttps else "HTTP", remoteHost,
remotePort, path)
if self.mCallBacks.isInScope(url):
if messageIsRequest:
self.dispatcher.processProxyRequest(
messageReference, messageIsRequest, remoteHost, remotePort,
serviceIsHttps, httpMethod, path, resourceType, statusCode,
responseContentType, message, interceptAction)
else:
self.dispatcher.processProxyResponse(
messageReference, messageIsRequest, remoteHost, remotePort,
serviceIsHttps, httpMethod, path, resourceType, statusCode,
responseContentType, message, interceptAction)
return message
def doScan(self, baseReqRes) :
httpService = baseReqRes.getHttpService()
host, port, protocol = httpService.getHost(), httpService.getPort(), httpService.getProtocol()
# Check if already scanned
for scannedHost, scannedPort in self.scannerInstance.scannedHost :
if scannedHost == host and scannedPort == port :
print "[SSL Scanner] The host has already been scanned, aborting"
self.scannerInstance._callbacks.issueAlert("The host %s:%d has already been scanned (Use [Clear scanned host] button to enable rescanning)" % (host, port))
break
else :
self.scannerInstance.addHostToScannedList(host, port)
self.scannerInstance._callbacks.issueAlert("Scanning %s:%d" % (host, port))
issues = self.scannerInstance.scan(URL(protocol, host, port, "/"), True)
self.scannerInstance._callbacks.issueAlert("Scan finished for %s:%d" % (host, port))
return issues
return None
def buildHTML(node):
for child in node.findAllDepthFirst():
flink = None
if child.link.text is not None:
flink = child.link.text
if child.text != SEPERATOR:
child.setDetailsText("")
child.setHideDetails(True)
if flink is not None:
ext = [".PNG", ".png", ".JPG", ".jpg", ".jpeg"]
if flink.endswith(tuple(ext)):
img = ImageIO.read(URL("file:" + flink))
mywidth = 400
wpercent = (mywidth / float(img.getWidth()))
hsize = int((float(img.getHeight()) * float(wpercent)))
child.setDetailsText("<html><body><p><img src=" + flink +
" width=" + str(mywidth) +
" height=" + str(hsize) +
"></p></body></html>")
child.parent.folded = True
def _up_check(self, url):
parse_object = urlparse.urlparse(url)
hostname = parse_object.netloc
if parse_object.scheme == 'https':
port = 443
SSL = True
else:
port = 80
SSL = False
try:
req_bytes = self._helpers.buildHttpRequest(URL(str(url)))
res_bytes = self._callbacks.makeHttpRequest(
hostname, port, SSL, req_bytes)
res = self._helpers.analyzeResponse(res_bytes)
if res.getStatusCode() not in WHITELIST_CODES:
return url
except Exception as e:
print(e)
print('SKIPPING : ', url)
return None
def testAddEndpointTableModelMax100(self):
state = GenericMock()
callbacks = GenericMock()
etm = EndpointTableModel(state, callbacks)
ret = callbacks.helpers.analyzeRequest.return_value
ret.method = "GET"
ret.url = URL("http://www.example.org/users")
for a in range(200):
etm.add(GenericMock())
self.assertEqual(len(etm.endpoints), 1)
self.assertEqual(etm.endpoints["GET|http://www.example.org/users"].url,
"http://www.example.org/users")
self.assertEqual(
etm.endpoints["GET|http://www.example.org/users"].method, "GET")
self.assertEqual(
len(etm.endpoints["GET|http://www.example.org/users"].requests),
etm.MAX_REQUESTS_PER_ENDPOINT)
def lfiDir(self, request, protocol, host, port, ishttps, url,html404_status, html404_content):
for paraNewValue in self.fuzzLFI.dir2:
newRequest = self._helpers.buildHttpRequest(URL(url+paraNewValue))
#print(url+paraNewValue)
newResponse = self._callbacks.makeHttpRequest(host, port, ishttps, newRequest)
newAnalyzedRequest, newReqHeaders, newReqBodys, newReqMethod, newReqParameters = self.get_request_info(
newRequest)
newResHeaders, newResBodys, newResStatusCode, resLength = self.get_response_info(
newResponse)
errorInject = self.fuzzLFI.errorFlag.findall(newResBodys)
if errorInject or (newResStatusCode == 206) or ((newResStatusCode == 200 or newResStatusCode == 302 or newResStatusCode == 301) and abs(resLength - len(html404_content))>50):
newReqUrl = self.get_request_url(protocol, newReqHeaders,host,port)
content = '[+]{} ->{} {}\n[Headers] -> {}\n[Bodys] -> {}'.format('[DIR GET]',errorInject,newReqUrl, newReqHeaders, newReqBodys)
print(content)
self.save(content + '\t\n')
print('-' * 50)
break
def POL_Download(self):
setupWindowId = self.command[2]
url = self.command[3]
currentDirectory = self.command[4]
try:
checkSum = self.command[5]
except IndexError:
checkSum = ""
setupWindow = self.setupWindowManager.getWindow(setupWindowId)
localFile = os.path.join(
currentDirectory,
Downloader.wizard(setupWindow).findFileNameFromURL(URL(url)))
downloader = Downloader.wizard(setupWindow).get(url, localFile)
if (checkSum != ""):
downloader.check(checkSum)
def processCLI(self):
cli = self._callbacks.getCommandLineArguments()
if len(cli) < 5:
logging.error("Invalid CLI arguments")
return False
if not cli:
return False
else:
logging.error("Initiating carbonator")
self.scheme = cli[0]
self.fqdn = cli[1]
self.port = int(cli[2])
self.path = cli[3]
self.export = cli[4]
self.url = URL(self.scheme, self.fqdn, self.port, self.path)
return True
def testIssueCheckerOnlyOnce(self):
ic, state, callbacks = self._ic()
state.scope_urls = ["http://example.org/"]
ic.reportIssue = GenericMock()
issue = GenericMock()
issue.url = URL("http://www.example.org/users")
issue.issueName = "SQL Injection"
callbacks.getScanIssues.return_value = [issue]
ic.run()
self.assertEquals(ic.reportIssue.call_count, 1)
ic.run()
self.assertEquals(
ic.reportIssue.call_count, 1,
"Should still be one because it's the same issue and it has already been reported."
)
def generateUrls(self, url, requestResponse):
urls = []
path = url.getPath()
parts = filter(None, path.split("/"))
for part in parts:
if "." in part:
continue
# Checks if /part../ results in 403
if not self.quickCheck(url, part, requestResponse):
continue
self._stdout.println("Potentially vulnerable: %s" % url)
replacement = "/%s../%s/" % (part, part)
urls.append(URL(url.toString().replace("/%s/" % part, replacement)))
if self.enableDirectoryGuessing:
urls = urls + self.guessDirectories(url, part)
return urls
def __trigger_bing_query(self, bing_query_string):
# encode our query
quoted_query = urllib.quote(bing_query_string)
# OMG, :facepalm: burp http api requires that we build up the entire HTTP request as a string before sending it off,
# and in particular you can see that we need to base64-encode
http_request = "GET https://api.datamarket.azure.com/Bing/SEarch/WEb?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query
http_request += "Host: api.datamarket.azure.com\r\n"
http_request += "Connection: close \r\n"
http_request += "Authorization: Basic %s\n\n" % base64.b64encode(
":%s" % bing_api_key)
http_request += "User-Agent: Blackhat Python\r\n\r\n"
json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com",
443, True,
http_request).tostring()
# split the headers off
json_body = json_body.split("\r\n\r\n", 1)[1]
try:
r = json.loads(json_body)
if len(r["d"]["results"]):
for site in r["d"]["results"]:
print "*" * 100
print site['Title']
print site['Url']
print site['Description']
print "*" * 100
j_url = URL(site['Url'])
# not sure about this part. How is Burp Target Scope?
if not self._callbacks.isInScope(j_url):
print "Adding to Burp scope"
self._callbacks.includeInScope(j_url)
except:
print "No results from Bing"
def bing_query(self, bing_query_string):
print('Performing Bing search: %s' % bing_query_string)
http_request = 'GET https://%s/bing/v7.0/search?' % API_HOST
http_request += 'q=%s HTTP/1.1\r\n' % urllib.quote(bing_query_string)
http_request += 'Host: %s' % API_HOST
http_request += 'Connection:close\r\n'
http_request += 'Ocp-Apim-Subscription-Key: %s\r\n' % API_KEY
http_request += 'User-Agent: Black Hat Python\r\n\r\n'
json_body = self._callbacks.makeHttpRequest(API_HOST, 443, True,
http_request).tostring()
json_body = json_body.split('\r\n\r\n', 1)[1]
try:
response = json.loads(json_body)
except (TypeError, ValueError) as err:
print('No results from Bing: %s' % err)
else:
sites = list()
if response.get('webPages'):
sites = response['webPages']['value']
if len(sites):
for site in sites:
print('*' * 100)
print('Name: %s ' % site['name'])
print('URL: %s ' % site['url'])
print('Description: %r' % site['snippet'])
print('*' * 100)
java_url = URL(site['url'])
if not self._callbacks.isInScope(java_url):
print('Adding %s to Burp scope' % site['url'])
self._callbacks.includeInScope(java_url)
else:
print('Empty response from Bing.: %s' % bing_query_string)
return
# if __name__ == '__main__':
# p = BurpExtender()
# p.bing_query('wikipedia.com')
def crawl(self, event):
print("Starting")
host = self.hostField.text
if host.find("://") == -1:
host = "http://" + host
try:
self._callbacks.includeInScope(URL(host))
except:
JOptionPane.showMessageDialog(self._splitpane,
"Can't add host to scope", "Error",
JOptionPane.ERROR_MESSAGE)
return
self.resultTableModel.clearAllRow()
self.crawlingEvent.set()
self.crawlerThread = Thread(target=self.crawl_thread, args=(host, ))
self.crawlerThread.start()
print("Started")
def bing_query(self, bing_query_string):
print('Performing Bing search: %s' % bing_query_string)
http_request = 'GET https://%s/bing/v7.0/search?' % api_host
# encode our query
http_request += 'q=%s HTTP/1.1\r\n' % urllib.quote(bing_query_string)
http_request += 'Host: %s\r\n' % api_host
http_request += 'Connection:close\r\n'
http_request += 'Ocp-Apim-Subscription-Key: %s\r\n' % api_key
http_request += 'User-Agent: %s\r\n\r\n' % agent
json_body = self._callbacks.makeHttpRequest(
api_host, 443, True, http_request).tostring()
json_body = json_body.split('\r\n\r\n', 1)[1]
try:
# Deseralize the response
response = json.loads(json_body)
except (TypeError, ValueError) as err:
print('No results from Bing: %s' % err)
else:
sites = list()
if response.get('webPages'):
sites = response['webPages']['value']
if len(sites):
for site in sites:
print('*' * 100)
print('Name: %s ' % site['name'])
print('URL : %s ' % site['url'])
print('Description: %s ' % site['snippet'])
print('*' * 100)
java_url = URL(site['url'])
if not self._callbacks.isInScope(java_url):
print('Adding %s to Burp scope' % site['url'])
self._callbacks.includeInScope(java_url)
else:
print('Empty response from Being.: %s' % bing_query_string)
return
def doActiveScan(self, basePair, insertionPoint):
global check
if check == 0:
return None
if self._helpers.analyzeRequest(
basePair.getRequest()).getMethod() == "GET":
method = IParameter.PARAM_URL
else:
method = IParameter.PARAM_BODY
path = self._helpers.analyzeRequest(basePair).getUrl().getPath()
port = self._helpers.analyzeRequest(basePair).getUrl().getPort()
proto = self._helpers.analyzeRequest(basePair).getUrl().getProtocol()
hostname = self._helpers.analyzeRequest(basePair).getUrl().getHost()
if not '.' in path:
return None
for temps in self._payloads:
url = URL(proto, hostname, port, path + temps)
newRequest = self._helpers.buildHttpRequest(url)
attack = callbacks.makeHttpRequest(basePair.getHttpService(),
newRequest)
resp_code = self._helpers.analyzeResponse(
attack.getResponse()).getStatusCode()
if resp_code == 200:
url = self._helpers.analyzeRequest(attack).getUrl()
print "Possible Source code disclosure!"
if (url not in self._done):
self._done.append(url)
return [
CustomScanIssue(
attack.getHttpService(), url, [attack],
'Source code disclosure',
"The server contains temporary files.<p>",
'Tentative', 'Low')
]
def connect(self):
self.client = HttpClient.create(URL(self.url), "POST")
self.client.setRequestBody(self.params)
self.client.setHeader("Content-Type",
"application/x-www-form-urlencoded")
self.client.setHeader(
"User-Agent",
"Polyglot's JOSM Bot (runs in Java Openstreetmap Editor, developed from scratch in Jython)"
)
self.setCookies()
self.httpConnection = self.client.connect()
self.getCookies()
pprint.pprint(self.cookies)
self.content = json.loads(self.httpConnection.fetchContent())
# print self.httpConnection.getHeaderFields()
# print 'content: ' + self.content['login']['result']
pprint.pprint(self.content)
if 'login' in self.content:
self.login = self.content['login']
if 'result' in self.login:
result = self.login['result']
print 'result: ', result
return self.content
def processProxyMessage(self, messageIsRequest, message):
# message have to be in scope first
if self._callbacks.isInScope(
URL(message.getMessageInfo().getHttpService().toString())
) and self._httpVerbFlag is False:
request = message.getMessageInfo().getRequest() # return in Byte[]
host = message.getMessageInfo().getHttpService().getHost()
port = message.getMessageInfo().getHttpService().getPort()
request_string = self._helpers.bytesToString(request)
for method in self._httpVerbList:
request_string2 = request_string.replace("GET", method)
newRequest = self._helpers.stringToBytes(request_string2)
newResponse = self._callbacks.makeHttpRequest(
host, port, False, newRequest)
newRsponseInfo = self._helpers.analyzeResponse(newResponse)
newRespondHeaderList = newRsponseInfo.getHeaders()
newStatusCode = newRsponseInfo.getStatusCode()
print method + " returns " + str(newStatusCode)
self._httpVerbFlag = True
def bing_query(self, bing_query_string):
print("Realizando busqueda en Bing: %s" % bing_query_string)
# codificamos nuestro query
quoted_query = urllib.parse.quote(bing_query_string)
http_request = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query
http_request += "Host: api.datamarket.azure.com\r\n"
http_request += "Connection: close\r\n"
http_request += "Authorization: Basic %s\r\n" % base64.b64encode(
":%s" % bing_api_key)
http_request += "User-Agent: Blackhat Python\r\n\r\n"
json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com",
443, True,
http_request).tostring()
json_body = json_body.split("\r\n\r\n", 1)[1]
try:
r = json.loads(json_body)
if len(r["d"]["results"]):
for site in r["d"]["results"]:
print("*" * 100)
print(site['Title'])
print(site['Url'])
print(site['Description'])
print("*" * 100)
j_url = URL(site['Url'])
if not self._callbacks.isInScope(j_url):
print("Adding to Burp scope")
self._callbacks.includeInScope(j_url)
except:
print("No se obtuvieron resultados de Bing")
pass
return
def queryBing(self, query):
print("Perfoming bing search: %s" % query)
quotedQuery = urllib.parse.quote(query)
httpReq = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&QUery=%s HTTP/1.1\r\n" % quotedQuery
httpReq += "Host: api.datamarket.azure.com\r\n"
httpReq += "Connection: close\r\n"
httpReq += "Authorization: Basic %s\r\n" % base64.b64encode(":%s" % bingKey)
httpReq += "User_Agent: Blackhat Python\r\n\r\n"
# send http request to microsoft servers
jsonBody = self._callbacks.makeHttpRequest("api.datamarket.azure.com", 443, True, httpReq).tostring()
# split off headers
jsonBody = jsonBody.split("\r\n\r\n", 1)[1]
try:
r = json.loads(jsonBody)
if len(r["d"]["results"]):
for site in r["d"]["results"]:
print("*" * 100)
print(site["Title"])
print(site["Url"])
print(site["Description"])
print("*" * 100)
j_url = URL(site["Url"])
if not self._callbacks.includeInScope(j_url):
print("Adding to Burp Scope")
self._callbacks.includeInScope(j_url)
except LookupError as e:
print("No results from bing")
pass
return
def doActiveScan(self, baseRequestResponse, insertionPoint):
reqinfo = helpers.analyzeRequest(baseRequestResponse)
url = str(reqinfo.getUrl())
url = urlparse(url)
issues = []
burl = url.scheme + '://' + url.netloc + '/' + url.path
if burl in self.hs or url.path[-1] == '/':
return None
self.hs.add(burl)
for regex in self.patterns.iterkeys():
subs = self.patterns[regex]
for sub in subs:
baktest = helpers.buildHttpRequest(
URL(url.scheme, url.hostname, url.port,
regex.sub(sub, url.path)))
attack = cbs.makeHttpRequest(
baseRequestResponse.getHttpService(), baktest)
reqinfo = helpers.analyzeResponse(attack.getResponse())
if reqinfo.getStatusCode() == 200:
issues.append(
CustomScanIssue(
attack.getHttpService(),
helpers.analyzeRequest(attack).getUrl(), [attack],
"Backup file leaked",
"Suspecious backup file likely leaked", "High"))
return issues
def bing_query(self, bing_query_string):
print(f"Performing Bing search: {bing_query_string}")
http_request = f'GET https://{API_HOST}/bing/v7.0/search?'
http_request += f'q={urllib.quote(bing_query_string)} HTTP/1.1\r\n'
http_request += f"Host: {API_HOST}\r\n"
http_request += f"Connection:close\r\n"
http_request += f"Ocp-Apim-Subscription-Key: {API_KEY}\r\n"
http_request += f"User-Agent: PyBing-Go\r\n\r\n"
json_body = self._callbacks.makeHttpRequest(API_HOST, 443, True, http_request).tostring()
json_body = json_body.split("\r\n\r\n", 1)[1]
try:
response = json.loads(json_body)
except (TypeError, ValueError) as err:
print(f"No results from Bing: {err}")
else:
site = list()
if response.get('webPages'):
sites = response['webPage']['value']
if len(sites):
for site in sites:
print('*'*100)
print(f"Name: {site['name']}")
print(f"URL: {site['url']}")
print(f"Description: {site['snippet']}")
print('*'*100)
java_url = URL(site['url'])
if not self._callbacks.isInScope(java_url):
print(f"Adding {site['url']} to Burp Scope")
else:
print(f"Empty response from Bing: {bing_query_string}")
return
def __init__(self, callbacks):
self.config = {}
# Burp specific configs
self.config['callbacks'] = callbacks
self.config['callbacks'].setProxyInterceptionEnabled(False)
conf_to_reload = self.config['callbacks'].saveConfig()
conf_to_reload['proxy.interceptresponses'] = True
self.config['callbacks'].loadConfig(conf_to_reload)
# Scope
url = URL("HTTPS", "isecpartners.com", 443, "/")
self.config['callbacks'].includeInScope(url)
# Hiccupy specific configs
self.config['path_2_enabled_plugins'] = 'lib/plugins/plugins-enabled/'
self.config['debug'] = True
print "[%s] Completed configuration load" % self.__class__.__name__
if self.config['debug']:
tmp_cnf = self.config['callbacks'].saveConfig()
for option in tmp_cnf.entrySet():
print "%s => %s" % (option.key, option.value)
def googleSiteIndex(self, url, mCallBacks, startIndex):
#print 'Starting Google Site: Index for URL: ' + str(url)
data = 'Starting Google Site: Index for URL: ' + str(url) + '\n'
self.parent.printLogTab(str(data))
googleRequest = self.buildGoogleRequest(url, startIndex)
googleResponse = mCallBacks.makeHttpRequest('www.google.com',
int('80'), False,
googleRequest)
googleStringResponse = googleResponse.tostring()
for urlInSearch in re.findall(r'''<a href="([^<]+)" class=l''',
googleStringResponse):
uUrl = URL(urlInSearch)
port = 80
if str(uUrl.getProtocol()) == "https":
port = 443
if mCallBacks.isInScope(uUrl):
newRequest = self.buildGenericRequest(uUrl)
newResponse = mCallBacks.makeHttpRequest(
str(uUrl.getHost()), port,
(str(uUrl.getProtocol()) == "https"), newRequest)
newResponseString = newResponse.tostring()
firstWord, statusCode, restResponse = newResponseString.split(
" ", 2)
requestResponse = HttpRequestResponse.HttpRequestResponse(
None, uUrl.getHost(), port, uUrl.getProtocol(), newRequest,
newResponse, int(statusCode), uUrl)
mCallBacks.addToSiteMap(requestResponse)
#print "Adding: " + urlInSearch
data = "Adding: " + urlInSearch + "\n"
self.parent.printLogTab(str(data))
else:
data = "\n" + urlInSearch + " was not in scope.\n\n"
self.parent.printLogTab(str(data))
data = "End of Google Site Indexing for URL " + str(url) + "\n"
self.parent.printLogTab(str(data))
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
businessElementId = Framework.getDestinationAttribute('id')
urlString = Framework.getDestinationAttribute('url')
jobId = Framework.getDiscoveryJobId()
dnsServers = Framework.getParameter('dnsServers') or None
localShell = None
if dnsServers:
dnsServers = [
dnsServer for dnsServer in dnsServers.split(',')
if dnsServer and dnsServer.strip()
] or None
if dnsServers:
logger.debug('Using dns servers: ', dnsServers)
localShell = shellutils.ShellUtils(
Framework.createClient(ClientsConsts.LOCAL_SHELL_PROTOCOL_NAME))
if not urlString:
msg = "There is no specified URL in the input BusinessElement CI"
errormessages.resolveAndReport(msg, jobId, Framework)
return OSHVResult
try:
urlString = urlString[1:len(urlString) - 1]
if netutils.isValidIp(urlString):
createHostOSHwithIP(OSHVResult, urlString)
return OSHVResult
urlObject = URL(urlString)
hostname = urlObject.getHost()
if not hostname:
logger.debug("Hostname is not defined in URL '%s'" % urlString)
raise MalformedURLException()
urlObjectResolver = URLObjectResolver(urlObject)
protocol = urlObjectResolver.getProtocolFromUrlObject()
if not protocol:
msg = "Failed to resolve the http/https protocol from specified URL"
errormessages.resolveAndReport(msg, jobId, Framework)
return OSHVResult
port = urlObjectResolver.getPortFromUrlObject()
if not port:
msg = "Failed to resolve the port number from specified URL"
errormessages.resolveAndReport(msg, jobId, Framework)
return OSHVResult
# get topology
# create business element CI and attach the url as configuration document CI to it
bizOSH = modeling.createOshByCmdbIdString('business_element',
businessElementId)
urlConfigOSH = modeling.createConfigurationDocumentOSH(
'url.txt', '', urlString, bizOSH)
linkBizUrlConifg = modeling.createLinkOSH('composition', bizOSH,
urlConfigOSH)
OSHVResult.add(bizOSH)
OSHVResult.add(urlConfigOSH)
OSHVResult.add(linkBizUrlConifg)
hostDNSName = None
if not netutils.isValidIp(hostname):
# Treat the host name as its DNS name if it is not a valid ip address
hostDNSName = hostname
ipAddresses = []
if dnsServers:
ipAddresses = urlObjectResolver.getIpFromUrlObjectWithDnsList(
dnsServers, localShell)
else:
ipAddresses = urlObjectResolver.getIpFromUrlObject()
for ipAddress in ipAddresses:
logger.debug('Reporting ip address: ', ipAddresses)
if not ipAddress or not netutils.isValidIp(
ipAddress) or netutils.isLocalIp(ipAddress):
msg = "Failed to resolve the IP address of server from specified URL"
errormessages.resolveAndReport(msg, jobId, Framework)
return OSHVResult
hostOSH, ipOSH = createHostOSHwithIP(OSHVResult, ipAddress)
if hostDNSName:
ipOSH.setAttribute('authoritative_dns_name', hostDNSName)
# create UriEndpoint and relations between business element and UriEndpoint
urlOSH = modeling.createServiceURLAddressOsh(hostOSH, urlString)
linkBizUrl = modeling.createLinkOSH('usage', bizOSH, urlOSH)
OSHVResult.add(urlOSH)
OSHVResult.add(linkBizUrl)
# create ipServiceEndpoint and relations between UriEndpoint and ipServiceEndpoint
ipPort = modeling.createServiceAddressOsh(
hostOSH, ipAddress, port, modeling.SERVICEADDRESS_TYPE_TCP)
linkUrlIP = modeling.createLinkOSH('dependency', urlOSH, ipOSH)
OSHVResult.add(ipPort)
OSHVResult.add(linkUrlIP)
except MalformedURLException:
msg = "Specified URL '%s' is malformed" % urlString
errormessages.resolveAndReport(msg, jobId, Framework)
except:
msg = logger.prepareJythonStackTrace("")
errormessages.resolveAndReport(msg, jobId, Framework)
return OSHVResult
def doInBackground(self):
#print "\n- Checking for the latest version..."
try:
url = URL(self.app.scriptVersionUrl)
uc = url.openConnection()
ins = uc.getInputStream()
p = Properties()
p.load(ins)
latestScriptVersion = p.getProperty("script")
self.app.latestToolsVersion = p.getProperty("tools")
except (UnknownHostException, SocketException):
print "I can't connect to:\n%s" % url
ins.close()
return
if latestScriptVersion == self.app.SCRIPTVERSION:
#using latest script
print " already using the latest script version:", self.app.SCRIPTVERSION
if self.app.latestToolsVersion == self.app.TOOLSVERSION:
#using latest tools
print " already using the latest tools version:", self.app.TOOLSVERSION
if self.mode != "auto":
JOptionPane.showMessageDialog(
self.app.preferencesFrame,
self.app.strings.getString("using_latest"))
return
else:
#not using latest tools
print " tools can be updated: %s -> %s" % (
self.app.TOOLSVERSION, self.app.latestToolsVersion)
if self.app.mode == "stable":
infoString = self.app.strings.getString(
"update_tools_question")
else:
infoString = self.app.strings.getString(
"dev_update_tools_question")
answer = JOptionPane.showConfirmDialog(
Main.parent, infoString,
self.app.strings.getString("updates_available"),
JOptionPane.YES_NO_OPTION)
if answer == 0:
#Download the updated tools data
print "\n- Update tools data"
try:
self.app.toolsProgressDialog
except AttributeError:
from java.awt import Dialog
self.app.toolsProgressDialog = ToolsProgressDialog(
Main.parent,
self.app.strings.getString(
"download_tools_updates"),
Dialog.ModalityType.APPLICATION_MODAL, self.app)
self.app.toolsProgressDialog.show()
else:
#not using latest script
print "a new script version is available:\n%s -> %s" % (
self.app.SCRIPTVERSION, latestScriptVersion)
messageArguments = array(
[self.app.SCRIPTVERSION, latestScriptVersion], String)
formatter = MessageFormat("")
if self.app.mode == "stable":
formatter.applyPattern(
self.app.strings.getString("updates_warning"))
infoBtnString = self.app.strings.getString("Visit_Wiki")
else:
formatter.applyPattern(
self.app.strings.getString("dev_updates_warning"))
infoBtnString = self.app.strings.getString("Visit_git")
msg = formatter.format(messageArguments)
options = [
self.app.strings.getString("Do_not_check_for_updates"),
infoBtnString,
self.app.strings.getString("cancel")
]
answer = JOptionPane.showOptionDialog(
Main.parent, msg,
self.app.strings.getString("updates_available"),
JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE,
None, options, options[1])
if answer == 0:
self.app.properties.setProperty("check_for_update", "off")
self.app.save_config(self)
elif answer == 1:
if self.app.mode == "stable":
url = self.app.SCRIPTWEBSITE
else:
url = self.app.GITWEBSITE
OpenBrowser.displayUrl(url)
while ((line = in.readLine()) != null) {
response.append(line);
}
in.close();
return response.toString();
}
public static String Translate (String text, String from, String to) throws Exception {
String queryPath = "";
if(!from.equals("detect")) {
queryPath += "&from=" + from;
}
queryPath += "&to=" + to;
URL url = new URL (host + path + queryPath);
List<Util.RequestBody> objList = new ArrayList<>();
objList.add(new Util.RequestBody(text));
String content = new Gson().toJson(objList);
return Post(url, content);
}
public static String prettify(String json_text) {
json_text = json_text.substring(1, json_text.length() - 1);
JsonParser parser = new JsonParser();
JsonElement json = parser.parse(json_text);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
return gson.toJson(json);
}
