def readWebPage( webpageURL ): webpageURL = webpageURL.strip() log(VERBOSE_, "readWebpage webpageURL=" + webpageURL) url = URL(webpageURL) conn = url.openConnection() conn.setConnectTimeout(30000) conn.setReadTimeout(10000) conn.connect() responseCode = conn.getResponseCode() cookie = conn.getHeaderField("Set-Cookie") cookiePath = None pathDiscr = " Path=" if cookie and cookie.find(pathDiscr) > 0: cookiePath = cookie[cookie.index(pathDiscr) + len(pathDiscr):] respLines = [] if responseCode >= 400: log(ERROR_, "HTTP ERROR " + `responseCode` + ": " + `conn.getResponseMessage()`) else: log(VERBOSE_, "WebPageResponse status=" + `responseCode` + " reason=" + `conn.getResponseMessage()`) #log(DEBUG_,"WebPageResponse resp="+`resp` ) reader = BufferedReader(InputStreamReader(conn.getInputStream())) inputLine = reader.readLine() while inputLine is not None: respLines.append(inputLine) inputLine = reader.readLine() reader.close() return respLines, cookiePath
def post(targetURL, params, contentType="text/xml", username=None): if(type(params) is dict): paramStr = "" for aKey in params.keys(): paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&" paramStr=paramStr[:-1] else: paramStr = params url = URL(targetURL) print targetURL print paramStr print contentType connection = url.openConnection() if username!=None: userpass = username basicAuth = "Basic " + base64.b64encode(userpass); connection.setRequestProperty ("Authorization", basicAuth); connection.setRequestMethod("POST") if contentType != None: connection.setRequestProperty("Content-Type", contentType) connection.setRequestProperty("Content-Length", str(len(paramStr))) connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoInput(1) connection.setDoOutput(2) wr= DataOutputStream(connection.getOutputStream()) wr.writeBytes(paramStr) wr.flush() wr.close() return getResponse(connection);
def googleSiteIndex(self, url, mCallBacks, startIndex): #print 'Starting Google Site: Index for URL: ' + str(url) data = 'Starting Google Site: Index for URL: ' + str(url) + '\n' self.parent.printLogTab(str(data)) googleRequest = self.buildGoogleRequest(url, startIndex) googleResponse = mCallBacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest) googleStringResponse = googleResponse.tostring() for urlInSearch in re.findall(r'''<a href="([^<]+)" class=l''', googleStringResponse): uUrl = URL(urlInSearch) port = 80 if str(uUrl.getProtocol()) == "https": port = 443 if mCallBacks.isInScope(uUrl): newRequest = self.buildGenericRequest(uUrl) newResponse = mCallBacks.makeHttpRequest(str(uUrl.getHost()), port, (str(uUrl.getProtocol()) == "https"), newRequest) newResponseString = newResponse.tostring() firstWord, statusCode, restResponse = newResponseString.split(" ", 2) requestResponse = HttpRequestResponse.HttpRequestResponse(None, uUrl.getHost(), port, uUrl.getProtocol(), newRequest, newResponse, int(statusCode), uUrl) mCallBacks.addToSiteMap(requestResponse) #print "Adding: " + urlInSearch data = "Adding: " + urlInSearch + "\n" self.parent.printLogTab(str(data)) else: data = "\n" + urlInSearch + " was not in scope.\n\n" self.parent.printLogTab(str(data)) data = "End of Google Site Indexing for URL " + str(url) + "\n" self.parent.printLogTab(str(data))
def get(targetURL, params, username=None): paramStr = "" for aKey in params.keys(): paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&" paramStr=paramStr[:-1] url = URL(targetURL+"?"+paramStr) print url connection = url.openConnection() if username!=None: userpass = username basicAuth = "Basic " + base64.b64encode(userpass); #print basicAuth connection.setRequestProperty ("Authorization", basicAuth); connection.setRequestMethod("GET") connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoOutput(2) inStream= connection.getInputStream() rd= BufferedReader(InputStreamReader(inStream)) response = "" line = rd.readLine() while line != None: response +=line+"\r" line = rd.readLine() rd.close() return response
def crawl(site, trm , depth, linksfile): from java.net import URL from org.w3c.tidy import Tidy pattern = re.compile('href="/wiki/(.*?)"') f = open(linksfile, 'a+') #try: if depth < MAX_DEPTH: print 'crawling [%s]...' % trm, print >> f, '[%s]' % trm td = Tidy() td.setXmlOut(1) u = URL(site + trm) input = BufferedInputStream(u.openStream()) output = ByteArrayOutputStream() #tidy.setInputEncoding("UTF8") #tidy.setOutputEncoding("UTF8") td.parse(input, output) content = output.toString() hits = pattern.findall(content) for hit in hits: if hit.find(":") == -1: print >> f, hit print 'done.' print >> f, '' for hit in hits: if hit.find(":") == -1: crawl(site, hit, depth + 1, linksfile) #except: # print "wrong" f.close()
def getUrl(WSDLUrl, containerOSH): res = ObjectStateHolderVector() urlIP = None try: url = URL(WSDLUrl) hostName = url.getHost() urlIP = netutils.getHostAddress(hostName, None) if (not netutils.isValidIp(urlIP)) or netutils.isLocalIp(urlIP): urlIP = None except: urlIP = None urlOSH = modeling.createUrlOsh(containerOSH, WSDLUrl, 'wsdl') urlIpOSH = None if urlIP != None: try: urlIpOSH = modeling.createIpOSH(urlIP) except: urlIpOSH = None res.add(urlOSH) if urlIpOSH: res.add(urlIpOSH) urlToIpOSH = modeling.createLinkOSH('depend', urlOSH, urlIpOSH) res.add(urlToIpOSH) return res
def nukeAndPave(islandID, wipeMasters=False, wipeSlaves=False): if not wipeMasters and not wipeSlaves: return clusterMgr = mc.getClusterManager() clusters = clusterMgr.enumClusters(islandID) slaves = [] masters = [] for cluster in clusters: locs = clusterMgr.enumClusterLocations(cluster.getID()) for loc in locs: slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)) slaveHost = slaveURL.getHost() masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP)) masterHost = masterURL.getHost() slaves.append(slaveHost) masters.append(masterHost) rc = 0 if wipeSlaves: print "Removing Solr from slaves to prepare for upgrade", slaves failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-install"], slaves) if failures.isEmpty(): print "INFO: nuke and pave of slaves succeeded" else: print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures rc = 1 return rc
def DiscoveryMain(Framework): OSHVResult = ObjectStateHolderVector() urlString = Framework.getParameter(PARAM_URL) reportPoweredOffVms = 0 reportPoweredOffVmsValue = Framework.getParameter(PARAM_REPORT_POWERED_OFF_VMS) if reportPoweredOffVmsValue and reportPoweredOffVmsValue.lower() =='true': reportPoweredOffVms = 1 ipAddress = None try: urlObject = URL(urlString) hostname = urlObject.getHost() if not hostname: logger.debug("Hostname is not defined in URL '%s'" % urlString) raise MalformedURLException() ipAddress = vcloud_discover.getIpFromUrlObject(urlObject) if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress): msg = "Failed to resolve the IP address of server from specified URL" errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework) return OSHVResult except MalformedURLException: msg = "Specified URL '%s' is malformed" % urlString errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework) except: msg = logger.prepareJythonStackTrace("") errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework) else: #configure how connections should be discovered/established connectionDiscoverer = vcloud_discover.ConnectionDiscoverer(Framework) urlGenerator = vcloud_discover.ConstantUrlGenerator(urlString) connectionDiscoverer.setUrlGenerator(urlGenerator) connectionDiscoverer.addIp(ipAddress) #configure how established/failed connection should be used connectionHandler = vcloud_discover.BaseDiscoveryConnectionHandler(Framework) topologyDiscoverer = vcloud_discover.createVcloudDiscoverer(Framework) topologyReporter = vcloud_report.createVcloudReporter(Framework, None, reportPoweredOffVms) connectionHandler.setDiscoverer(topologyDiscoverer) connectionHandler.setReporter(topologyReporter) connectionDiscoverer.setConnectionHandler(connectionHandler) connectionDiscoverer.initConnectionConfigurations() connectionDiscoverer.discover(firstSuccessful=0) if not connectionHandler.connected: for errorMsg in connectionHandler.connectionErrors: Framework.reportError(errorMsg) for warningMsg in connectionHandler.connectionWarnings: Framework.reportWarning(warningMsg) return OSHVResult
def isUrlAvailable(url, acceptedStatusCodeRange, timeout=10000): ''' Checks whether url is available str, list(str), int -> bool ''' from com.hp.ucmdb.discovery.library.clients import SSLContextManager from com.hp.ucmdb.discovery.library.clients.http import ApacheHttpClientWrapper as HttpClientWrapper if not url or not acceptedStatusCodeRange: return 0 with _create_http_client_wrapper() as client: client.setSocketTimeout(timeout) try: jurl = URL(url) if jurl.getProtocol() == 'https': port = jurl.getPort() or HttpClientWrapper.DEFAULT_HTTPS_PORT context = SSLContextManager.getAutoAcceptSSLContext() client.registerProtocol(context, port) except: logger.warn('Failed parsing url % ' % url) try: httpResult = client.get(url) return httpResult.statusCode in acceptedStatusCodeRange except: logger.warn('Get Failed: %s' % logger.prepareJavaStackTrace()) return 0
def __init__(self, queryString): self.args = queryString # We would like content_type to be a class property but this is # not supported in Python 2.1 self.content_type = "text/plain" reqURL = URL("http://myhost/mywms/") self.server = FakeModPythonServerObject("%s:%d" % (reqURL.getHost(), reqURL.getPort())) self.unparsed_uri = str(reqURL.getPath())
def delete(targetURL, params): url = URL(targetURL) connection = url.openConnection() connection.setRequestMethod("DELETE") connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoOutput(2) return getResponse(connection);
def createURLOSHV(urlString, framework = None): OSHVResult = ObjectStateHolderVector() #urlOSH2 = modeling.createOshByCmdbIdString('uri_endpoint', urlId) logger.debug("Starting URL discovery on '%s'" % urlString) #urlString = urlString[1:len(urlString)-1] if not urlString: return OSHVResult try: urlString = str(urlString).replace("\\", "//") urlObject = URL(urlString) hostname = urlObject.getHost() if not hostname: raise MalformedURLException("Hostname is not defined in URL '%s'" % urlString) urlObjectResolver = URLObjectResolver(urlObject) protocol = urlObjectResolver.getProtocolFromUrlObject() if not protocol: raise Exception("Failed to resolve the http/https protocol from specified URL") port = urlObjectResolver.getPortFromUrlObject() if not port: raise Exception("Failed to resolve the port number from specified URL") # get topology # create business element CI and attach the url as configuration document CI to it ips = urlObjectResolver.getIpFromUrlObject() for ipAddress in ips: logger.debug('%s: Reporting ip address: %s' % (urlString, ipAddress)) if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress): raise Exception("Failed to resolve the IP address of server from specified URL") hostOSH, ipOSH, OSHVResult2 = createIPEndpointOSHV(framework, ipAddress, port, protocol, hostname) OSHVResult.addAll(OSHVResult2) # create UriEndpoint and relations between business element and UriEndpoint urlOSH = modeling.createUrlOsh(hostOSH, urlString, None) #urlOSH.setCmdbObjectId(urlOSH2.getCmdbObjectId()) OSHVResult.add(urlOSH) OSHVResult.add(modeling.createLinkOSH('dependency', urlOSH, ipOSH)) #create Web Server except: logger.warnException("Error creating URL OSH for %s" % urlString) return OSHVResult
def _get_map_file(self): map_param = self.getParameter('map') map_url = URL(self.getDocumentBase(), map_param) self.showStatus('Loading ' + map_url.toString()) map_is = map_url.openStream() map_buf = "" i = map_is.read() while i != -1: map_buf = map_buf + chr(i) i = map_is.read() return StringIO(map_buf)
def doService(httpMethod, url, credential, requestBody=None): Security.addProvider(MySSLProvider()) Security.setProperty("ssl.TrustManagerFactory.algorithm", "TrustAllCertificates") HttpsURLConnection.setDefaultHostnameVerifier(MyHostnameVerifier()) urlObj = URL(url) con = urlObj.openConnection() con.setRequestProperty("Accept", "application/xml") con.setRequestProperty("Content-Type", "application/xml") con.setRequestProperty("Authorization", credential) con.setDoInput(True); if httpMethod == 'POST': con.setDoOutput(True) con.setRequestMethod(httpMethod) output = DataOutputStream(con.getOutputStream()); if requestBody: output.writeBytes(requestBody); output.close(); responseCode = con.getResponseCode() logger.info('response code: ' + str(responseCode)) responseMessage = con.getResponseMessage() logger.info('response message: ' + str(responseMessage)) contentLength = con.getHeaderField('Content-Length') logger.info('content length: ' + str(contentLength)) stream = None if responseCode == 200 or responseCode == 201 or responseCode == 202: stream = con.getInputStream() elif contentLength: stream = con.getErrorStream() if stream: dataString = getStreamData(stream) logger.info(httpMethod + ' url: ' + url) if not url.endswith('.xsd') and len(dataString) < 4096: xmlStr = Util.prettfyXmlByString(dataString) logger.info(httpMethod + ' result: \n\n' + xmlStr) else: logger.info('response body too big, no print out') if responseCode == 200 or responseCode == 201 or responseCode == 202: return dataString else: ''' to mark the case failed if response code is not 200-202 ''' return None else: logger.error('') logger.error('---------------------------------------------------------------------------------------------------') logger.error('-------->>> Input or Error stream is None, it may be a defect if it is positive test case') logger.error('---------------------------------------------------------------------------------------------------') logger.error('') return None
def new_urlopen(req): if isinstance(req, str): full_url = req else: full_url = req.get_full_url() if full_url.startswith('https'): u = URL(full_url) conn = u.openConnection() return ConnReader(conn) else: org_urlopen(req)
def addAuthentication(self, httpClient, method): BasicAuthStrategy.addAuthentication(self, httpClient, method) url = URL(self.__baseUrl) proxy = ProxySelector.getDefault().select(url.toURI()).get(0) httpClient.getParams().setAuthenticationPreemptive(False); if not proxy.type().equals(Proxy.Type.DIRECT): address = proxy.address() proxyHost = address.getHostName() proxyPort = address.getPort() httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort) print "Using proxy '%s:%s'" % (proxyHost, proxyPort)
def __init__(self, request, response): self._response = response self.args = request.getQueryString() # We would like content_type to be a class property but this is # not supported in Python 2.1 self.content_type = "text/plain" reqURL = URL(request.getRequestURL().toString()) self.server = FakeModPythonServerObject("%s:%d" % (reqURL.getHost(), reqURL.getPort())) self.unparsed_uri = str(reqURL.getPath()) self.headers_out = {} # Dictionary of HTTP headers self.headers_set = 0
def reportToToolServer(self, url): """Tell to the tool server when an error is fixed (user clicked on correctedBtn) or false positive (falsePositiveBtn). Tool instance must have either: self.fixedFeedbackMode = "url" self.falseFeedbackMode = "url" """ try: url = URL(url) uc = url.openConnection() uc.getInputStream() uc.disconnect() except (UnknownHostException, IOException, SocketException): print url print "* I can't connect to the tool server."
def getIpFromUrlString(urlString): try: urlObject = URL(urlString) hostname = urlObject.getHost() if not hostname: logger.debug("Hostname is not defined in URL '%s'" % urlString) raise MalformedURLException() ipAddress = getIpFromUrlObject(urlObject) if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress): raise ValueError("Failed determining the IP address of server") return ipAddress except: logger.warnException("Invalid URL")
def processCLI(self): cli = self._callbacks.getCommandLineArguments() if len(cli) < 0: print "Incomplete target information provided." return False elif not cli: print "Integris Security Carbonator is now loaded." print "If Carbonator was loaded through the BApp store then you can run in headless mode simply adding the `-Djava.awt.headless=true` flag from within your shell. Note: If burp doesn't close at the conclusion of a scan then disable Automatic Backup on Exit." print "For questions or feature requests contact us at carbonator at integris security dot com." print "Visit carbonator at https://www.integrissecurity.com/Carbonator" return False else: self.url = URL(cli[0]) self.rtype = cli[1] self.output = cli[2] self.scheme = self.url.getProtocol() self.fqdn = self.url.getHost() self.port1 = self.url.getPort() if self.port1 == -1 and self.scheme == 'http': self.port = 80 elif self.port1 == -1 and self.scheme == 'https': self.port = 443 else: self.port = self.port1 self.path = self.url.getFile() print "self.url: " + str(self.url) + "\n" print "Scheme: " + self.scheme + "\n" print "FQDN: " + self.fqdn + "\n" print "Port: " + str(self.port1) + "\n" print "Path: " + self.path + "\n" return True
def send_to_josm(self, url): """Remote control command to make JOSM download the area nearby the error """ try: url = URL(url) uc = url.openConnection() uc.getInputStream() return True except UnknownHostException: return False except FileNotFoundException: return False except SocketException: print "\n* Please, enable JOSM remote from Preferences" return False
def connect(self, url): if re.findall('(?:\:\d{2,5})', url) and self.urlRegex.match(url): try: javaURL = URL(url) newRequest = self._helpers.buildHttpRequest(javaURL) requestResponse = self._callbacks.makeHttpRequest(self._helpers.buildHttpService(str(javaURL.getHost()), javaURL.getPort(), str(javaURL.getProtocol())), newRequest) # Follow redirects if a 301 or 302 response is received. As of Oct 9 2014 the API is not capable of this: http://forum.portswigger.net/thread/1504/ask-burp-follow-redirections-extension response = requestResponse.getResponse() if response: requestInfo = self._helpers.analyzeResponse(response) responseHeader = requestInfo.getHeaders() if ('301' in responseHeader[0] or '302' in responseHeader[0]) and self.redirectsCheckbox.isSelected(): self.redirectCounter += 1 for headerLine in responseHeader: if 'Location: ' in headerLine or 'location: ' in headerLine: url = self.locationHeaderConvert(str(headerLine.split(' ')[1]), str(javaURL.getPort()), str(javaURL.getHost()), '') self.connect(url) self.goodUrlList.append(url) self._callbacks.addToSiteMap(requestResponse) else: self.badUrlList.append(url) except: self.badUrlList.append(url) else: if 'http://' in url: fixedUrl = self.addPort(url, '80') self.connect(fixedUrl) elif 'https://' in url: fixedUrl = self.addPort(url, '443') self.connect(fixedUrl) else: self.badUrlList.append(url)
def delete(targetURL, params): url = URL(targetURL) connection = url.openConnection() connection.setRequestMethod("DELETE") connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoOutput(2) inStream= connection.getInputStream() rd= BufferedReader(InputStreamReader(inStream)) response = "" line = rd.readLine() while line != None: response +=line+"\r" line = rd.readLine() rd.close() return response
def make_jar_classloader(jar): import os from java.net import URL, URLClassLoader url = URL('jar:file:%s!/' % jar) if os._name == 'nt': # URLJarFiles keep a cached open file handle to the jar even # after this ClassLoader is GC'ed, disallowing Windows tests # from removing the jar file from disk when finished with it conn = url.openConnection() if conn.getDefaultUseCaches(): # XXX: Globally turn off jar caching: this stupid # instance method actually toggles a static flag. Need a # better fix conn.setDefaultUseCaches(False) return URLClassLoader([url])
def upgradeMaster(islandID): clusterMgr = mc.getClusterManager() clusters = clusterMgr.enumClusters(islandID) for cluster in clusters: locs = clusterMgr.enumClusterLocations(cluster.getID()) hosts = [] for loc in locs: masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP)) masterHost = masterURL.getHost() hosts.append(masterHost) service = Service("ems-solr") service.setThreadCount(1) service.invoke("stop", hosts) failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "save-index"], hosts) if not failures.isEmpty(): print "WARN: Failed to backup the index on all hosts. Check logs.", failures failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-index"], hosts) if not failures.isEmpty(): print "Failed to start the backup job running on all hosts. Check logs.", failures return 1 failures = runJob( ["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-status"], hosts, sleepTime=120, maxWaits=43200, interimResult=3, ) if not failures.isEmpty(): print "ERROR: upgrade job failed. Upgrade aborted", failures return 1 failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts) if not failures.isEmpty(): print "ERROR: unable to remove solr service from all hosts", failures return 1 print "Please check for any errors in the output before proceeding" print "If it appears the Solr index has been upgraded successfully. Please proceed to install the new version of Solr and then run the validate-master step" return 0
def googleSiteIndex_run(querys, maxIndex): for i in querys: try: maxIndex = int(maxIndex) except: self.appendToResults("Index Value was not a valid Integer\n") return # setup counts to stay within provided index range resultsCount = 0 currentIndex = 0 previousIndex = -1 self.appendToResults("Starting Google Hack for " + i.strip() + "\n") while (currentIndex < maxIndex) and (previousIndex != currentIndex): previousIndex = currentIndex googleRequest = self.buildGoogleRequest(i, currentIndex) try: googleResponse = self._callbacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest).tostring() except: self.appendToResults("Call to google was not made: (Could not make a connection)\n") return if re.findall(r'<a href="([^<]+)" class=l', googleResponse) and resultsCount < maxIndex: currentIndex += 100 for urlInSearch in re.findall(r'<a href="([^<]+)" class=l', googleResponse): if resultsCount < maxIndex: uUrl = URL(urlInSearch) port = 80 if str(uUrl.getProtocol()) == "https": port = 443 if self._callbacks.isInScope(uUrl): newRequest = self.buildGenericRequest(uUrl) try: requestResponse = self._callbacks.makeHttpRequest(self._helpers.buildHttpService(str(uUrl.getHost()), port, str(uUrl.getProtocol()) == "https"), newRequest) self._callbacks.addToSiteMap(requestResponse) resultsCount += 1 self.appendToResults("Adding " + urlInSearch + " to Target:\n") except: self.appendToResults("Call to URL found was not made: (Could not make a connection)\n") else: self.appendToResults(urlInSearch + " was found but not in Scope (Not Adding to Target)\n") else: previousIndex = currentIndex self.appendToResults("Reached end of query " + i.strip() + " with " + str(resultsCount) + " results found\n\n")
def post(targetURL, params, contentType="text/xml", username=None): #paramStr = params["data"] paramStr = "" for aKey in params.keys(): paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&" paramStr=paramStr[:-1] url = URL(targetURL) print targetURL print paramStr print contentType url = URL(targetURL+"?"+paramStr) print url connection = url.openConnection() if username!=None: userpass = username basicAuth = "Basic " + base64.b64encode(userpass); connection.setRequestProperty ("Authorization", basicAuth); connection.setRequestMethod("POST") if contentType != None: connection.setRequestProperty("Content-Type", contentType) connection.setRequestProperty("Content-Length", str(len(paramStr))) connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoInput(1) connection.setDoOutput(2) #wr= DataOutputStream(connection.getOutputStream()) #wr.writeBytes(paramStr) #wr.flush() #wr.close() inStream= connection.getInputStream() rd= BufferedReader(InputStreamReader(inStream)) response = "" line = rd.readLine() while line != None: response +=line+"\r" line = rd.readLine() rd.close() return response
def __init__(self): sessionDuration = grinder.properties.getInt("player.sessionDurationInSeconds", 0) self.password = grinder.properties.getProperty("player.password") players_url = grinder.properties.getProperty("player.source-url") passwordHash = URLEncoder.encode(grinder.properties.getProperty("player.passwordHash"), "UTF-8") workers = grinder.properties.getInt("grinder.processes", 0) threads = grinder.properties.getInt("grinder.threads", 0) workerIndex = grinder.processNumber - grinder.firstProcessNumber threadsPerAgent = workers * threads agentIndex = grinder.agentNumber start = agentIndex * threadsPerAgent + workerIndex * threads if isLogEnabled: log("Worker will handle %s players starting from %d" % (threads, start)) playerRange = (start, threads) self.workerPlayers = [] params = "passwordHash=%s&minimumBalance=%s&startRow=%s&limit=%s" % ( passwordHash, str(sessionDuration / 5), str(playerRange[0]), str(threads), ) urlStr = players_url + "?" + params try: data = StringBuffer() url = URL(urlStr) conn = url.openConnection() rd = BufferedReader(InputStreamReader(conn.getInputStream())) line = rd.readLine() while line is not None: data.append(line) line = rd.readLine() rd.close() if isLogEnabled: log(data.toString()) message = JSONValue.parse(str(String(data.toString(), "UTF-8"))) for entry in message: self.workerPlayers.append((entry.get(0), entry.get(1))) except Exception: raise Exception("Couldn't fetch players from %s: %s" % (urlStr, traceback.format_exc())) if isLogEnabled: log(str(self.workerPlayers))
def get(targetURL, params, username=None): paramStr = "" for aKey in params.keys(): paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&" paramStr=paramStr[:-1] url = URL(targetURL+"?"+paramStr) print url connection = url.openConnection() if username!=None: userpass = username basicAuth = "Basic " + base64.b64encode(userpass); print basicAuth connection.setRequestProperty ("Authorization", basicAuth); connection.setRequestMethod("GET") connection.setRequestProperty("Content-Language", "en-GB") connection.setUseCaches(0) connection.setDoOutput(2) return getResponse(connection);
def call(self): consumer = DefaultOAuthConsumer(CONSUMER_KEY, CONSUMER_SECRET); provider = DefaultOAuthProvider( REQUEST_TOKEN_URL, ACCESS_TOKEN_URL, AUTHORIZE_URL ); print "Fetching request token from PassaporteWeb..." authUrl = provider.retrieveRequestToken(consumer, CALLBACK_URL); authUrl = OAuth.addQueryParameters(authUrl, OAuth.OAUTH_CONSUMER_KEY, CONSUMER_KEY); print "Request token: ", consumer.getToken() print "Token secret: ", consumer.getTokenSecret() print "Now visit:\n", authUrl, "\n... and grant this app authorization" print "After granting authorization, you should be redirected to the callback url" pin = raw_input("Enter the PIN code and hit ENTER when you're done:") print "Fetching access token from PassaporteWeb..." provider.retrieveAccessToken(consumer, pin); print "Access token: ", consumer.getToken() print "Token secret: ", consumer.getTokenSecret() url = URL(USER_DATA); request = url.openConnection(); consumer.sign(request); print "Sending request..." request.connect(); print "Response: %d %s" % (request.getResponseCode(), request.getResponseMessage())
def bing_query(self, bing_query_string): '''query bing''' print 'Performing Bing search: %s' % bing_query_string # encode our query quoted_query = urllib.quote(bing_query_string) my_key = base64.b64encode(':%s' % self.get_my_key()) host = 'api.datamarket.azure.com' host = 'api.cognitive.microsoft.com' base_url = 'https://%s/Bing/Search/Web' % host base_url = 'https://%s/bing/v5.0/search?q=' agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0' http_request = 'GET %s?$format=json&$top=20&Query=%s' % (base_url, quoted_query) http_request += ' HTTP/1.1\r\n' http_request += 'Host: %s\r\n' % host http_request += 'Connection: close\r\n' http_request += 'Authorization: Basic %s\r\n' % my_key http_request += 'User-Agent: %s' % agent print 'Making http_request...', json_body = self._callbacks.makeHttpRequest(host, 443, True, http_request).tostring() json_body = json_body.split('\r\n\r\n', 1)[1] print 'done.' try: r = json.loads(json_body) print str(r) if len(r['d']['results']): for site in r['d']['results']: print '*' * 100 print site['Title'] print site['Url'] print site['Description'] print '*' * 100 j_url = URL(site['Url']) if not self._callbacks.isInScope(j_url): print 'Adding to Burp Scope' self._callbacks.includeInScope(j_url) except Exception as err: print str(err) #print 'No results from Bing' return
def fuzzUrlsAPI(self): print 'Fuzzing %s viable URLs with a wsdl extension\n' % len( self.detectedUrlList) for messageUrl in self.detectedUrlList: wsdlSuffixUrl = messageUrl + '?wsdl' if wsdlSuffixUrl not in self.fuzzedWsdlList and wsdlSuffixUrl not in self.foundWsdlList: # Build full HTTP URL with port fullUrl = self.originalMsgUrl + ':' + str(self.originalMsgPort) # Build the URL path without domain to be used in the GET request requestWsdlUrl = wsdlSuffixUrl[len(fullUrl):] # Build HTTP request string to be used with Burp API requestString = """GET %s HTTP/1.1\nHost: %s\r\n\r\n""" % ( requestWsdlUrl, self.originalMsgHost) # Make request of the generated wsdl URL using Burp API response = self._callbacks.makeHttpRequest( self.originalMsgHost, self.originalMsgPort, self.originalMsgProtocol == "https", self._helpers.stringToBytes(requestString)) wsdlSuffixUrlResponseData = self._helpers.bytesToString( response) # Check if response is valid if self._helpers.analyzeResponse( response).getStatusCode() == 200: for wsdlKeyword in self.wsdlKeywordList: # Loop through the keywords and check if one of them are in the response (very good indicator of wsdl or similar file) if wsdlKeyword in wsdlSuffixUrlResponseData: # wsdl file found from created URL, add to fuzzed list self.fuzzedWsdlList.append(wsdlSuffixUrl) # Build a request and response based on the found wsdl URL and add them to the Burp site map wsdlJavaURL = URL(wsdlSuffixUrl) newRequest = self._helpers.buildHttpRequest( wsdlJavaURL) requestResponse = self._callbacks.makeHttpRequest( self._helpers.buildHttpService( str(self.originalMsgHost), int(self.originalMsgPort), self.originalMsgProtocol == "https"), newRequest) self._callbacks.addToSiteMap(requestResponse) break
def bing_query(self, bing_query_string): print "Performing Bing search: %s" % bing_query_string # encode our query quoted_query = urllib.quote(bing_query_string) http_request = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query http_request += "Host: api.datamarket.azure.com\r\n" http_request += "Connection: close\r\n" http_request += "Authorization: Basic %s\r\n" % base64.b64encode( ":%s" % bing_api_key) http_request += "User-Agent: Blackhat Python\r\n\r\n" json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com", 443, True, http_request).tostring() json_body = json_body.split("\r\n\r\n", 1)[1] try: r = json.loads(json_body) if len(r["d"]["results"]): for site in r["d"]["results"]: print "*" * 100 print site['Title'] print site['Url'] print site['Description'] print "*" * 100 j_url = URL(site['Url']) if not self._callbacks.isInScope(j_url): print "Adding to Burp scope" self._callbacks.includeInScope(j_url) except: print "No results from Bing" pass return
def post(self, url): if re.findall('(?:\:\d{2,5})', url) and self.urlRegex.match(url): try: javaURL = URL(url) newRequest = self._helpers.buildHttpRequest(javaURL) newRequest = self._helpers.toggleRequestMethod( newRequest) # for post requests requestResponse = self._callbacks.makeHttpRequest( self._helpers.buildHttpService(str(javaURL.getHost()), javaURL.getPort(), str(javaURL.getProtocol())), newRequest) # Follow redirects if a 301 or 302 response is received. As of Oct 9 2014 the API is not capable of this: http://forum.portswigger.net/thread/1504/ask-burp-follow-redirections-extension response = requestResponse.getResponse() if response: requestInfo = self._helpers.analyzeResponse(response) responseHeader = requestInfo.getHeaders() if ('301' in responseHeader[0] or '302' in responseHeader[0] ) and self.redirectsCheckbox.isSelected(): self.redirectCounter += 1 for headerLine in responseHeader: if 'Location: ' in headerLine or 'location: ' in headerLine: url = self.locationHeaderConvert( str(headerLine.split(' ')[1]), str(javaURL.getPort()), str(javaURL.getHost()), '') self.post(url) self.goodUrlList.append(url) self._callbacks.addToSiteMap(requestResponse) else: self.badUrlList.append(url) except: self.badUrlList.append(url) else: if 'http://' in url: fixedUrl = self.addPort(url, '80') self.post(fixedUrl) elif 'https://' in url: fixedUrl = self.addPort(url, '443') self.post(fixedUrl) else: self.badUrlList.append(url)
def processProxyMessage(self, messageReference, messageIsRequest, remoteHost, remotePort, serviceIsHttps, httpMethod, path, resourceType, statusCode, responseContentType, message, interceptAction): self.loader.reloadIfChanged() url = URL("HTTPS" if serviceIsHttps else "HTTP", remoteHost, remotePort, path) if self.mCallBacks.isInScope(url): if messageIsRequest: self.dispatcher.processProxyRequest( messageReference, messageIsRequest, remoteHost, remotePort, serviceIsHttps, httpMethod, path, resourceType, statusCode, responseContentType, message, interceptAction) else: self.dispatcher.processProxyResponse( messageReference, messageIsRequest, remoteHost, remotePort, serviceIsHttps, httpMethod, path, resourceType, statusCode, responseContentType, message, interceptAction) return message
def doScan(self, baseReqRes) : httpService = baseReqRes.getHttpService() host, port, protocol = httpService.getHost(), httpService.getPort(), httpService.getProtocol() # Check if already scanned for scannedHost, scannedPort in self.scannerInstance.scannedHost : if scannedHost == host and scannedPort == port : print "[SSL Scanner] The host has already been scanned, aborting" self.scannerInstance._callbacks.issueAlert("The host %s:%d has already been scanned (Use [Clear scanned host] button to enable rescanning)" % (host, port)) break else : self.scannerInstance.addHostToScannedList(host, port) self.scannerInstance._callbacks.issueAlert("Scanning %s:%d" % (host, port)) issues = self.scannerInstance.scan(URL(protocol, host, port, "/"), True) self.scannerInstance._callbacks.issueAlert("Scan finished for %s:%d" % (host, port)) return issues return None
def buildHTML(node): for child in node.findAllDepthFirst(): flink = None if child.link.text is not None: flink = child.link.text if child.text != SEPERATOR: child.setDetailsText("") child.setHideDetails(True) if flink is not None: ext = [".PNG", ".png", ".JPG", ".jpg", ".jpeg"] if flink.endswith(tuple(ext)): img = ImageIO.read(URL("file:" + flink)) mywidth = 400 wpercent = (mywidth / float(img.getWidth())) hsize = int((float(img.getHeight()) * float(wpercent))) child.setDetailsText("<html><body><p><img src=" + flink + " width=" + str(mywidth) + " height=" + str(hsize) + "></p></body></html>") child.parent.folded = True
def _up_check(self, url): parse_object = urlparse.urlparse(url) hostname = parse_object.netloc if parse_object.scheme == 'https': port = 443 SSL = True else: port = 80 SSL = False try: req_bytes = self._helpers.buildHttpRequest(URL(str(url))) res_bytes = self._callbacks.makeHttpRequest( hostname, port, SSL, req_bytes) res = self._helpers.analyzeResponse(res_bytes) if res.getStatusCode() not in WHITELIST_CODES: return url except Exception as e: print(e) print('SKIPPING : ', url) return None
def testAddEndpointTableModelMax100(self): state = GenericMock() callbacks = GenericMock() etm = EndpointTableModel(state, callbacks) ret = callbacks.helpers.analyzeRequest.return_value ret.method = "GET" ret.url = URL("http://www.example.org/users") for a in range(200): etm.add(GenericMock()) self.assertEqual(len(etm.endpoints), 1) self.assertEqual(etm.endpoints["GET|http://www.example.org/users"].url, "http://www.example.org/users") self.assertEqual( etm.endpoints["GET|http://www.example.org/users"].method, "GET") self.assertEqual( len(etm.endpoints["GET|http://www.example.org/users"].requests), etm.MAX_REQUESTS_PER_ENDPOINT)
def lfiDir(self, request, protocol, host, port, ishttps, url,html404_status, html404_content): for paraNewValue in self.fuzzLFI.dir2: newRequest = self._helpers.buildHttpRequest(URL(url+paraNewValue)) #print(url+paraNewValue) newResponse = self._callbacks.makeHttpRequest(host, port, ishttps, newRequest) newAnalyzedRequest, newReqHeaders, newReqBodys, newReqMethod, newReqParameters = self.get_request_info( newRequest) newResHeaders, newResBodys, newResStatusCode, resLength = self.get_response_info( newResponse) errorInject = self.fuzzLFI.errorFlag.findall(newResBodys) if errorInject or (newResStatusCode == 206) or ((newResStatusCode == 200 or newResStatusCode == 302 or newResStatusCode == 301) and abs(resLength - len(html404_content))>50): newReqUrl = self.get_request_url(protocol, newReqHeaders,host,port) content = '[+]{} ->{} {}\n[Headers] -> {}\n[Bodys] -> {}'.format('[DIR GET]',errorInject,newReqUrl, newReqHeaders, newReqBodys) print(content) self.save(content + '\t\n') print('-' * 50) break
def POL_Download(self): setupWindowId = self.command[2] url = self.command[3] currentDirectory = self.command[4] try: checkSum = self.command[5] except IndexError: checkSum = "" setupWindow = self.setupWindowManager.getWindow(setupWindowId) localFile = os.path.join( currentDirectory, Downloader.wizard(setupWindow).findFileNameFromURL(URL(url))) downloader = Downloader.wizard(setupWindow).get(url, localFile) if (checkSum != ""): downloader.check(checkSum)
def processCLI(self): cli = self._callbacks.getCommandLineArguments() if len(cli) < 5: logging.error("Invalid CLI arguments") return False if not cli: return False else: logging.error("Initiating carbonator") self.scheme = cli[0] self.fqdn = cli[1] self.port = int(cli[2]) self.path = cli[3] self.export = cli[4] self.url = URL(self.scheme, self.fqdn, self.port, self.path) return True
def testIssueCheckerOnlyOnce(self): ic, state, callbacks = self._ic() state.scope_urls = ["http://example.org/"] ic.reportIssue = GenericMock() issue = GenericMock() issue.url = URL("http://www.example.org/users") issue.issueName = "SQL Injection" callbacks.getScanIssues.return_value = [issue] ic.run() self.assertEquals(ic.reportIssue.call_count, 1) ic.run() self.assertEquals( ic.reportIssue.call_count, 1, "Should still be one because it's the same issue and it has already been reported." )
def generateUrls(self, url, requestResponse): urls = [] path = url.getPath() parts = filter(None, path.split("/")) for part in parts: if "." in part: continue # Checks if /part../ results in 403 if not self.quickCheck(url, part, requestResponse): continue self._stdout.println("Potentially vulnerable: %s" % url) replacement = "/%s../%s/" % (part, part) urls.append(URL(url.toString().replace("/%s/" % part, replacement))) if self.enableDirectoryGuessing: urls = urls + self.guessDirectories(url, part) return urls
def __trigger_bing_query(self, bing_query_string): # encode our query quoted_query = urllib.quote(bing_query_string) # OMG, :facepalm: burp http api requires that we build up the entire HTTP request as a string before sending it off, # and in particular you can see that we need to base64-encode http_request = "GET https://api.datamarket.azure.com/Bing/SEarch/WEb?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query http_request += "Host: api.datamarket.azure.com\r\n" http_request += "Connection: close \r\n" http_request += "Authorization: Basic %s\n\n" % base64.b64encode( ":%s" % bing_api_key) http_request += "User-Agent: Blackhat Python\r\n\r\n" json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com", 443, True, http_request).tostring() # split the headers off json_body = json_body.split("\r\n\r\n", 1)[1] try: r = json.loads(json_body) if len(r["d"]["results"]): for site in r["d"]["results"]: print "*" * 100 print site['Title'] print site['Url'] print site['Description'] print "*" * 100 j_url = URL(site['Url']) # not sure about this part. How is Burp Target Scope? if not self._callbacks.isInScope(j_url): print "Adding to Burp scope" self._callbacks.includeInScope(j_url) except: print "No results from Bing"
def bing_query(self, bing_query_string): print('Performing Bing search: %s' % bing_query_string) http_request = 'GET https://%s/bing/v7.0/search?' % API_HOST http_request += 'q=%s HTTP/1.1\r\n' % urllib.quote(bing_query_string) http_request += 'Host: %s' % API_HOST http_request += 'Connection:close\r\n' http_request += 'Ocp-Apim-Subscription-Key: %s\r\n' % API_KEY http_request += 'User-Agent: Black Hat Python\r\n\r\n' json_body = self._callbacks.makeHttpRequest(API_HOST, 443, True, http_request).tostring() json_body = json_body.split('\r\n\r\n', 1)[1] try: response = json.loads(json_body) except (TypeError, ValueError) as err: print('No results from Bing: %s' % err) else: sites = list() if response.get('webPages'): sites = response['webPages']['value'] if len(sites): for site in sites: print('*' * 100) print('Name: %s ' % site['name']) print('URL: %s ' % site['url']) print('Description: %r' % site['snippet']) print('*' * 100) java_url = URL(site['url']) if not self._callbacks.isInScope(java_url): print('Adding %s to Burp scope' % site['url']) self._callbacks.includeInScope(java_url) else: print('Empty response from Bing.: %s' % bing_query_string) return # if __name__ == '__main__': # p = BurpExtender() # p.bing_query('wikipedia.com')
def crawl(self, event): print("Starting") host = self.hostField.text if host.find("://") == -1: host = "http://" + host try: self._callbacks.includeInScope(URL(host)) except: JOptionPane.showMessageDialog(self._splitpane, "Can't add host to scope", "Error", JOptionPane.ERROR_MESSAGE) return self.resultTableModel.clearAllRow() self.crawlingEvent.set() self.crawlerThread = Thread(target=self.crawl_thread, args=(host, )) self.crawlerThread.start() print("Started")
def bing_query(self, bing_query_string): print('Performing Bing search: %s' % bing_query_string) http_request = 'GET https://%s/bing/v7.0/search?' % api_host # encode our query http_request += 'q=%s HTTP/1.1\r\n' % urllib.quote(bing_query_string) http_request += 'Host: %s\r\n' % api_host http_request += 'Connection:close\r\n' http_request += 'Ocp-Apim-Subscription-Key: %s\r\n' % api_key http_request += 'User-Agent: %s\r\n\r\n' % agent json_body = self._callbacks.makeHttpRequest( api_host, 443, True, http_request).tostring() json_body = json_body.split('\r\n\r\n', 1)[1] try: # Deseralize the response response = json.loads(json_body) except (TypeError, ValueError) as err: print('No results from Bing: %s' % err) else: sites = list() if response.get('webPages'): sites = response['webPages']['value'] if len(sites): for site in sites: print('*' * 100) print('Name: %s ' % site['name']) print('URL : %s ' % site['url']) print('Description: %s ' % site['snippet']) print('*' * 100) java_url = URL(site['url']) if not self._callbacks.isInScope(java_url): print('Adding %s to Burp scope' % site['url']) self._callbacks.includeInScope(java_url) else: print('Empty response from Being.: %s' % bing_query_string) return
def doActiveScan(self, basePair, insertionPoint): global check if check == 0: return None if self._helpers.analyzeRequest( basePair.getRequest()).getMethod() == "GET": method = IParameter.PARAM_URL else: method = IParameter.PARAM_BODY path = self._helpers.analyzeRequest(basePair).getUrl().getPath() port = self._helpers.analyzeRequest(basePair).getUrl().getPort() proto = self._helpers.analyzeRequest(basePair).getUrl().getProtocol() hostname = self._helpers.analyzeRequest(basePair).getUrl().getHost() if not '.' in path: return None for temps in self._payloads: url = URL(proto, hostname, port, path + temps) newRequest = self._helpers.buildHttpRequest(url) attack = callbacks.makeHttpRequest(basePair.getHttpService(), newRequest) resp_code = self._helpers.analyzeResponse( attack.getResponse()).getStatusCode() if resp_code == 200: url = self._helpers.analyzeRequest(attack).getUrl() print "Possible Source code disclosure!" if (url not in self._done): self._done.append(url) return [ CustomScanIssue( attack.getHttpService(), url, [attack], 'Source code disclosure', "The server contains temporary files.<p>", 'Tentative', 'Low') ]
def connect(self): self.client = HttpClient.create(URL(self.url), "POST") self.client.setRequestBody(self.params) self.client.setHeader("Content-Type", "application/x-www-form-urlencoded") self.client.setHeader( "User-Agent", "Polyglot's JOSM Bot (runs in Java Openstreetmap Editor, developed from scratch in Jython)" ) self.setCookies() self.httpConnection = self.client.connect() self.getCookies() pprint.pprint(self.cookies) self.content = json.loads(self.httpConnection.fetchContent()) # print self.httpConnection.getHeaderFields() # print 'content: ' + self.content['login']['result'] pprint.pprint(self.content) if 'login' in self.content: self.login = self.content['login'] if 'result' in self.login: result = self.login['result'] print 'result: ', result return self.content
def processProxyMessage(self, messageIsRequest, message): # message have to be in scope first if self._callbacks.isInScope( URL(message.getMessageInfo().getHttpService().toString()) ) and self._httpVerbFlag is False: request = message.getMessageInfo().getRequest() # return in Byte[] host = message.getMessageInfo().getHttpService().getHost() port = message.getMessageInfo().getHttpService().getPort() request_string = self._helpers.bytesToString(request) for method in self._httpVerbList: request_string2 = request_string.replace("GET", method) newRequest = self._helpers.stringToBytes(request_string2) newResponse = self._callbacks.makeHttpRequest( host, port, False, newRequest) newRsponseInfo = self._helpers.analyzeResponse(newResponse) newRespondHeaderList = newRsponseInfo.getHeaders() newStatusCode = newRsponseInfo.getStatusCode() print method + " returns " + str(newStatusCode) self._httpVerbFlag = True
def bing_query(self, bing_query_string): print("Realizando busqueda en Bing: %s" % bing_query_string) # codificamos nuestro query quoted_query = urllib.parse.quote(bing_query_string) http_request = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&Query=%s HTTP/1.1\r\n" % quoted_query http_request += "Host: api.datamarket.azure.com\r\n" http_request += "Connection: close\r\n" http_request += "Authorization: Basic %s\r\n" % base64.b64encode( ":%s" % bing_api_key) http_request += "User-Agent: Blackhat Python\r\n\r\n" json_body = self._callbacks.makeHttpRequest("api.datamarket.azure.com", 443, True, http_request).tostring() json_body = json_body.split("\r\n\r\n", 1)[1] try: r = json.loads(json_body) if len(r["d"]["results"]): for site in r["d"]["results"]: print("*" * 100) print(site['Title']) print(site['Url']) print(site['Description']) print("*" * 100) j_url = URL(site['Url']) if not self._callbacks.isInScope(j_url): print("Adding to Burp scope") self._callbacks.includeInScope(j_url) except: print("No se obtuvieron resultados de Bing") pass return
def queryBing(self, query): print("Perfoming bing search: %s" % query) quotedQuery = urllib.parse.quote(query) httpReq = "GET https://api.datamarket.azure.com/Bing/Search/Web?$format=json&$top=20&QUery=%s HTTP/1.1\r\n" % quotedQuery httpReq += "Host: api.datamarket.azure.com\r\n" httpReq += "Connection: close\r\n" httpReq += "Authorization: Basic %s\r\n" % base64.b64encode(":%s" % bingKey) httpReq += "User_Agent: Blackhat Python\r\n\r\n" # send http request to microsoft servers jsonBody = self._callbacks.makeHttpRequest("api.datamarket.azure.com", 443, True, httpReq).tostring() # split off headers jsonBody = jsonBody.split("\r\n\r\n", 1)[1] try: r = json.loads(jsonBody) if len(r["d"]["results"]): for site in r["d"]["results"]: print("*" * 100) print(site["Title"]) print(site["Url"]) print(site["Description"]) print("*" * 100) j_url = URL(site["Url"]) if not self._callbacks.includeInScope(j_url): print("Adding to Burp Scope") self._callbacks.includeInScope(j_url) except LookupError as e: print("No results from bing") pass return
def doActiveScan(self, baseRequestResponse, insertionPoint): reqinfo = helpers.analyzeRequest(baseRequestResponse) url = str(reqinfo.getUrl()) url = urlparse(url) issues = [] burl = url.scheme + '://' + url.netloc + '/' + url.path if burl in self.hs or url.path[-1] == '/': return None self.hs.add(burl) for regex in self.patterns.iterkeys(): subs = self.patterns[regex] for sub in subs: baktest = helpers.buildHttpRequest( URL(url.scheme, url.hostname, url.port, regex.sub(sub, url.path))) attack = cbs.makeHttpRequest( baseRequestResponse.getHttpService(), baktest) reqinfo = helpers.analyzeResponse(attack.getResponse()) if reqinfo.getStatusCode() == 200: issues.append( CustomScanIssue( attack.getHttpService(), helpers.analyzeRequest(attack).getUrl(), [attack], "Backup file leaked", "Suspecious backup file likely leaked", "High")) return issues
def bing_query(self, bing_query_string): print(f"Performing Bing search: {bing_query_string}") http_request = f'GET https://{API_HOST}/bing/v7.0/search?' http_request += f'q={urllib.quote(bing_query_string)} HTTP/1.1\r\n' http_request += f"Host: {API_HOST}\r\n" http_request += f"Connection:close\r\n" http_request += f"Ocp-Apim-Subscription-Key: {API_KEY}\r\n" http_request += f"User-Agent: PyBing-Go\r\n\r\n" json_body = self._callbacks.makeHttpRequest(API_HOST, 443, True, http_request).tostring() json_body = json_body.split("\r\n\r\n", 1)[1] try: response = json.loads(json_body) except (TypeError, ValueError) as err: print(f"No results from Bing: {err}") else: site = list() if response.get('webPages'): sites = response['webPage']['value'] if len(sites): for site in sites: print('*'*100) print(f"Name: {site['name']}") print(f"URL: {site['url']}") print(f"Description: {site['snippet']}") print('*'*100) java_url = URL(site['url']) if not self._callbacks.isInScope(java_url): print(f"Adding {site['url']} to Burp Scope") else: print(f"Empty response from Bing: {bing_query_string}") return
def __init__(self, callbacks): self.config = {} # Burp specific configs self.config['callbacks'] = callbacks self.config['callbacks'].setProxyInterceptionEnabled(False) conf_to_reload = self.config['callbacks'].saveConfig() conf_to_reload['proxy.interceptresponses'] = True self.config['callbacks'].loadConfig(conf_to_reload) # Scope url = URL("HTTPS", "isecpartners.com", 443, "/") self.config['callbacks'].includeInScope(url) # Hiccupy specific configs self.config['path_2_enabled_plugins'] = 'lib/plugins/plugins-enabled/' self.config['debug'] = True print "[%s] Completed configuration load" % self.__class__.__name__ if self.config['debug']: tmp_cnf = self.config['callbacks'].saveConfig() for option in tmp_cnf.entrySet(): print "%s => %s" % (option.key, option.value)
def googleSiteIndex(self, url, mCallBacks, startIndex): #print 'Starting Google Site: Index for URL: ' + str(url) data = 'Starting Google Site: Index for URL: ' + str(url) + '\n' self.parent.printLogTab(str(data)) googleRequest = self.buildGoogleRequest(url, startIndex) googleResponse = mCallBacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest) googleStringResponse = googleResponse.tostring() for urlInSearch in re.findall(r'''<a href="([^<]+)" class=l''', googleStringResponse): uUrl = URL(urlInSearch) port = 80 if str(uUrl.getProtocol()) == "https": port = 443 if mCallBacks.isInScope(uUrl): newRequest = self.buildGenericRequest(uUrl) newResponse = mCallBacks.makeHttpRequest( str(uUrl.getHost()), port, (str(uUrl.getProtocol()) == "https"), newRequest) newResponseString = newResponse.tostring() firstWord, statusCode, restResponse = newResponseString.split( " ", 2) requestResponse = HttpRequestResponse.HttpRequestResponse( None, uUrl.getHost(), port, uUrl.getProtocol(), newRequest, newResponse, int(statusCode), uUrl) mCallBacks.addToSiteMap(requestResponse) #print "Adding: " + urlInSearch data = "Adding: " + urlInSearch + "\n" self.parent.printLogTab(str(data)) else: data = "\n" + urlInSearch + " was not in scope.\n\n" self.parent.printLogTab(str(data)) data = "End of Google Site Indexing for URL " + str(url) + "\n" self.parent.printLogTab(str(data))
def DiscoveryMain(Framework): OSHVResult = ObjectStateHolderVector() businessElementId = Framework.getDestinationAttribute('id') urlString = Framework.getDestinationAttribute('url') jobId = Framework.getDiscoveryJobId() dnsServers = Framework.getParameter('dnsServers') or None localShell = None if dnsServers: dnsServers = [ dnsServer for dnsServer in dnsServers.split(',') if dnsServer and dnsServer.strip() ] or None if dnsServers: logger.debug('Using dns servers: ', dnsServers) localShell = shellutils.ShellUtils( Framework.createClient(ClientsConsts.LOCAL_SHELL_PROTOCOL_NAME)) if not urlString: msg = "There is no specified URL in the input BusinessElement CI" errormessages.resolveAndReport(msg, jobId, Framework) return OSHVResult try: urlString = urlString[1:len(urlString) - 1] if netutils.isValidIp(urlString): createHostOSHwithIP(OSHVResult, urlString) return OSHVResult urlObject = URL(urlString) hostname = urlObject.getHost() if not hostname: logger.debug("Hostname is not defined in URL '%s'" % urlString) raise MalformedURLException() urlObjectResolver = URLObjectResolver(urlObject) protocol = urlObjectResolver.getProtocolFromUrlObject() if not protocol: msg = "Failed to resolve the http/https protocol from specified URL" errormessages.resolveAndReport(msg, jobId, Framework) return OSHVResult port = urlObjectResolver.getPortFromUrlObject() if not port: msg = "Failed to resolve the port number from specified URL" errormessages.resolveAndReport(msg, jobId, Framework) return OSHVResult # get topology # create business element CI and attach the url as configuration document CI to it bizOSH = modeling.createOshByCmdbIdString('business_element', businessElementId) urlConfigOSH = modeling.createConfigurationDocumentOSH( 'url.txt', '', urlString, bizOSH) linkBizUrlConifg = modeling.createLinkOSH('composition', bizOSH, urlConfigOSH) OSHVResult.add(bizOSH) OSHVResult.add(urlConfigOSH) OSHVResult.add(linkBizUrlConifg) hostDNSName = None if not netutils.isValidIp(hostname): # Treat the host name as its DNS name if it is not a valid ip address hostDNSName = hostname ipAddresses = [] if dnsServers: ipAddresses = urlObjectResolver.getIpFromUrlObjectWithDnsList( dnsServers, localShell) else: ipAddresses = urlObjectResolver.getIpFromUrlObject() for ipAddress in ipAddresses: logger.debug('Reporting ip address: ', ipAddresses) if not ipAddress or not netutils.isValidIp( ipAddress) or netutils.isLocalIp(ipAddress): msg = "Failed to resolve the IP address of server from specified URL" errormessages.resolveAndReport(msg, jobId, Framework) return OSHVResult hostOSH, ipOSH = createHostOSHwithIP(OSHVResult, ipAddress) if hostDNSName: ipOSH.setAttribute('authoritative_dns_name', hostDNSName) # create UriEndpoint and relations between business element and UriEndpoint urlOSH = modeling.createServiceURLAddressOsh(hostOSH, urlString) linkBizUrl = modeling.createLinkOSH('usage', bizOSH, urlOSH) OSHVResult.add(urlOSH) OSHVResult.add(linkBizUrl) # create ipServiceEndpoint and relations between UriEndpoint and ipServiceEndpoint ipPort = modeling.createServiceAddressOsh( hostOSH, ipAddress, port, modeling.SERVICEADDRESS_TYPE_TCP) linkUrlIP = modeling.createLinkOSH('dependency', urlOSH, ipOSH) OSHVResult.add(ipPort) OSHVResult.add(linkUrlIP) except MalformedURLException: msg = "Specified URL '%s' is malformed" % urlString errormessages.resolveAndReport(msg, jobId, Framework) except: msg = logger.prepareJythonStackTrace("") errormessages.resolveAndReport(msg, jobId, Framework) return OSHVResult
def doInBackground(self): #print "\n- Checking for the latest version..." try: url = URL(self.app.scriptVersionUrl) uc = url.openConnection() ins = uc.getInputStream() p = Properties() p.load(ins) latestScriptVersion = p.getProperty("script") self.app.latestToolsVersion = p.getProperty("tools") except (UnknownHostException, SocketException): print "I can't connect to:\n%s" % url ins.close() return if latestScriptVersion == self.app.SCRIPTVERSION: #using latest script print " already using the latest script version:", self.app.SCRIPTVERSION if self.app.latestToolsVersion == self.app.TOOLSVERSION: #using latest tools print " already using the latest tools version:", self.app.TOOLSVERSION if self.mode != "auto": JOptionPane.showMessageDialog( self.app.preferencesFrame, self.app.strings.getString("using_latest")) return else: #not using latest tools print " tools can be updated: %s -> %s" % ( self.app.TOOLSVERSION, self.app.latestToolsVersion) if self.app.mode == "stable": infoString = self.app.strings.getString( "update_tools_question") else: infoString = self.app.strings.getString( "dev_update_tools_question") answer = JOptionPane.showConfirmDialog( Main.parent, infoString, self.app.strings.getString("updates_available"), JOptionPane.YES_NO_OPTION) if answer == 0: #Download the updated tools data print "\n- Update tools data" try: self.app.toolsProgressDialog except AttributeError: from java.awt import Dialog self.app.toolsProgressDialog = ToolsProgressDialog( Main.parent, self.app.strings.getString( "download_tools_updates"), Dialog.ModalityType.APPLICATION_MODAL, self.app) self.app.toolsProgressDialog.show() else: #not using latest script print "a new script version is available:\n%s -> %s" % ( self.app.SCRIPTVERSION, latestScriptVersion) messageArguments = array( [self.app.SCRIPTVERSION, latestScriptVersion], String) formatter = MessageFormat("") if self.app.mode == "stable": formatter.applyPattern( self.app.strings.getString("updates_warning")) infoBtnString = self.app.strings.getString("Visit_Wiki") else: formatter.applyPattern( self.app.strings.getString("dev_updates_warning")) infoBtnString = self.app.strings.getString("Visit_git") msg = formatter.format(messageArguments) options = [ self.app.strings.getString("Do_not_check_for_updates"), infoBtnString, self.app.strings.getString("cancel") ] answer = JOptionPane.showOptionDialog( Main.parent, msg, self.app.strings.getString("updates_available"), JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE, None, options, options[1]) if answer == 0: self.app.properties.setProperty("check_for_update", "off") self.app.save_config(self) elif answer == 1: if self.app.mode == "stable": url = self.app.SCRIPTWEBSITE else: url = self.app.GITWEBSITE OpenBrowser.displayUrl(url)
while ((line = in.readLine()) != null) { response.append(line); } in.close(); return response.toString(); } public static String Translate (String text, String from, String to) throws Exception { String queryPath = ""; if(!from.equals("detect")) { queryPath += "&from=" + from; } queryPath += "&to=" + to; URL url = new URL (host + path + queryPath); List<Util.RequestBody> objList = new ArrayList<>(); objList.add(new Util.RequestBody(text)); String content = new Gson().toJson(objList); return Post(url, content); } public static String prettify(String json_text) { json_text = json_text.substring(1, json_text.length() - 1); JsonParser parser = new JsonParser(); JsonElement json = parser.parse(json_text); Gson gson = new GsonBuilder().setPrettyPrinting().create(); return gson.toJson(json); }