def ignoreJavaSSL():
"""
Creates a dummy socket factory that doesn't verify connections.
HttpsURLConnection.setDefaultSSLSocketFactory(...)
This code was taken from multiple sources.
Only makes since in jython (java). otherwise, just use verify=False!
"""
import sys
if not 'java' in sys.platform:
raise RuntimeError('only use if platform (sys.platform) is java!')
else:
#===================================================================
# set default SSL socket to ignore verification
#===================================================================
import javax.net.ssl.X509TrustManager as X509TrustManager # @UnresolvedImport
class MyTrustManager(X509TrustManager):
def getAcceptedIssuers(self,*args,**keys):
return None
def checkServerTrusted(self,*args,**keys):
pass
def checkClientTrusted(self,*args,**keys):
pass
import com.sun.net.ssl.internal.ssl.Provider # @UnresolvedImport
from java.security import Security # @UnresolvedImport
Security.addProvider(com.sun.net.ssl.internal.ssl.Provider())
trustAllCerts = [MyTrustManager()]
import javax.net.ssl.SSLContext as SSLContext # @UnresolvedImport
sc = SSLContext.getInstance("SSL");
import java.security.SecureRandom as SecureRandom # @UnresolvedImport
sc.init(None, trustAllCerts,SecureRandom())
import javax.net.ssl.HttpsURLConnection as HttpsURLConnection # @UnresolvedImport
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory())
#===================================================================
# Do a test!
#===================================================================
'''
# setup proxy
import java.net.Proxy as Proxy
import java.net.InetSocketAddress as InetSocketAddress
p = Proxy(Proxy.Type.HTTP,InetSocketAddress("host",port))
import java.net.URL as URL
u = URL("https://www.google.com/")
conn = u.openConnection(p)
print 'server response: %r',conn.getResponseCode()
'''
#===================================================================
# ignore requests's error logging - this is for dev
#===================================================================
try:
import requests.packages.urllib3 as urllib3
urllib3.disable_warnings()
except: pass
return 'SSL verification in Java is disabled!'
def _initializeMXPI(serverName, serverPort, protocol,
MxpiMain5_1SoapBindingStubClass,
VerifyAllHostnameVerifierClass):
serverPortName = 'MxpiMain5_1'
namespaceURI = "urn:client.v5_1.soap.mx.hp.com"
serviceName = "MxpiMainService"
wsdlURL = "%s://%s:%s/mxsoap/services/%s?wsdl" % (protocol, serverName,
serverPort,
serverPortName)
# Set trust manager
if protocol == 'https':
verifyAllHostnameVerifier = VerifyAllHostnameVerifierClass()
sslContext = SSLContextManager.getAutoAcceptSSLContext()
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory())
HttpsURLConnection.setDefaultHostnameVerifier(verifyAllHostnameVerifier)
## Set trust all SSL Socket to accept all certificates
System.setProperty("ssl.SocketFactory.provider",
"TrustAllSSLSocketFactory")
Security.setProperty("ssl.SocketFactory.provider",
"TrustAllSSLSocketFactory")
# Try and initialize connection
simBindingStub = MxpiMain5_1SoapBindingStubClass()
simServiceFactory = ServiceFactory.newInstance()
simService = simServiceFactory.createService(URL(wsdlURL),
QName(namespaceURI,
serviceName))
theMxpiMain = simService.getPort(QName(namespaceURI, serverPortName),
simBindingStub.getClass())
return theMxpiMain
def doService(httpMethod, url, credential, requestBody=None):
Security.addProvider(MySSLProvider())
Security.setProperty("ssl.TrustManagerFactory.algorithm", "TrustAllCertificates")
HttpsURLConnection.setDefaultHostnameVerifier(MyHostnameVerifier())
urlObj = URL(url)
con = urlObj.openConnection()
con.setRequestProperty("Accept", "application/xml")
con.setRequestProperty("Content-Type", "application/xml")
con.setRequestProperty("Authorization", credential)
con.setDoInput(True);
if httpMethod == 'POST':
con.setDoOutput(True)
con.setRequestMethod(httpMethod)
output = DataOutputStream(con.getOutputStream());
if requestBody:
output.writeBytes(requestBody);
output.close();
responseCode = con.getResponseCode()
logger.info('response code: ' + str(responseCode))
responseMessage = con.getResponseMessage()
logger.info('response message: ' + str(responseMessage))
contentLength = con.getHeaderField('Content-Length')
logger.info('content length: ' + str(contentLength))
stream = None
if responseCode == 200 or responseCode == 201 or responseCode == 202:
stream = con.getInputStream()
elif contentLength:
stream = con.getErrorStream()
if stream:
dataString = getStreamData(stream)
logger.info(httpMethod + ' url: ' + url)
if not url.endswith('.xsd') and len(dataString) < 4096:
xmlStr = Util.prettfyXmlByString(dataString)
logger.info(httpMethod + ' result: \n\n' + xmlStr)
else:
logger.info('response body too big, no print out')
if responseCode == 200 or responseCode == 201 or responseCode == 202:
return dataString
else:
''' to mark the case failed if response code is not 200-202 '''
return None
else:
logger.error('')
logger.error('---------------------------------------------------------------------------------------------------')
logger.error('-------->>> Input or Error stream is None, it may be a defect if it is positive test case')
logger.error('---------------------------------------------------------------------------------------------------')
logger.error('')
return None
def crawl(self, currentURL):
#uses the python module urlparse to parse the URL into segments
scheme, domain, filePath, params, query, fragment = urlparse(
currentURL)
ipAddr = socket.getaddrinfo(domain, 443)
#RETRIEVE SSL CERTIFICATES (IF APPLICABLE)
try:
#uses the javax.net.ssl.* and java.security.cert.* libraries to obtain certificate
factory = HttpsURLConnection.getDefaultSSLSocketFactory()
tmpSocket = factory.createSocket(domain, 443)
tmpSocket.startHandshake()
session = tmpSocket.getSession()
domainCerts = session.getPeerCertificateChain()
except SSLHandshakeException: #except thrown if the domain does not support SSL
print 'javax.net.ssl.SSLHandshakeException with domain: ' + domain
except ConnectException:
print 'java.net.ConnectException with domain: ' + domain
#RETRIEVE PAGE SOURCE
pageSource = urllib.urlopen(currentURL).read()
#PARSE PAGE SOURCE FOR LINKS
myparser = MyParser(scheme, domain)
myparser.parse(pageSource)
myparser.sort_hyperlinks(depth)
childLinks = myparser.get_hyperlinks()
return ipAddr, domainCerts, pageSource, childLinks
def crawlDomain(self, seed):
currentURL, depth = seed
scheme, domain, filePath, params, query, fragment = urlparse(currentURL)
#######check for regular traffic on port 80
#######think of a way to grab information on servers that host on non-standard ports.
ipAddr = socket.getaddrinfo(domain, 443)
#RETRIEVE SSL CERTIFICATES (IF APPLICABLE)
try:
#uses the javax.net.ssl.* and java.security.cert.* libraries to obtain certificate
factory = HttpsURLConnection.getDefaultSSLSocketFactory()
tmpSocket = factory.createSocket(domain, 443)
tmpSocket.startHandshake()
session = tmpSocket.getSession()
domainCerts = session.getPeerCertificateChain()
except SSLHandshakeException: #except thrown if the domain does not support SSL
print 'javax.net.ssl.SSLHandshakeException with domain: ' + domain
domainCerts = " "
except ConnectException:
print 'java.net.ConnectException with domain: ' + domain
domainCerts = None
tempdd = DomainDetail() #create a temporary DomainDetail object
tempdd.setDomainName(domain) #set the domainName value in the DomainDetail object
#stores the IP addresses obtain from the crawlDomain() function
ipAddresses = []
ipAddrLen = len(ipAddr)
for i in range (0, ipAddrLen):
a, b, c, d, e = ipAddr[i]
a, b = e
oct1, sep, leftover = a.partition('.')
oct2, sep, leftover = leftover.partition('.')
oct3, sep, leftover = leftover.partition('.')
oct4, sep, leftover = leftover.partition('.')
ipAddresses.append(IPAddress(int(oct1), int(oct2), int(oct3), int(oct4)))
tempdd.setIPAddresses(ipAddresses)
print "IP Addresses for '" + domain + "' stored"
#parses and stores certificate Information
certArray = []
if(domainCerts != None):
chainLength = len(domainCerts)
for i in range(0, chainLength):
certArray.append(Certificate())
certArray[i].setFullCertificate(pprint.pformat(domainCerts[i]))
certArray[i].setIssuer(domainCerts[i].getIssuerDN().getName())
certArray[i].setHierarchy(i)
certArray[i].setSubject(domainCerts[i].getSubjectDN().getName())
certArray[i].setValidFrom(domainCerts[i].getNotBefore())
certArray[i].setValidTill(domainCerts[i].getNotAfter())
certArray[i].setBasicConstraint(0) ###DONT HOW HOW TO DO###
tempdd.setCertificates(certArray)
print "certs for '" + domain + "' stored"
return tempdd
def testPage(self, page):
class MyTrustManager(X509TrustManager):
def getAcceptedIssuers(self):
return None
def checkClientTrusted(self, certs, auth):
pass
def checkServerTrusted(self, certs, auth):
pass
trustAllCerts = [MyTrustManager()]
sc = SSLContext.getInstance("SSL")
sc.init(None, trustAllCerts, SecureRandom())
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory())
class MyHostnameVerifier(HostnameVerifier):
def verify(self, host, sess):
return True
HttpsURLConnection.setDefaultHostnameVerifier(MyHostnameVerifier())
try:
httpsURL = 'https://%s:%s/%s' % (self._host, self._port, page)
url = URL(httpsURL)
conn = url.openConnection()
conn.setConnectTimeout(5000)
conn.setRequestProperty("Accept-encoding", 'gzip,deflate,compress')
conn.setRequestProperty(
"User-agent",
'https://google.com/' if 'google' not in self._host else
'https://yandex.ru/') # Use foreign referer
#ist = conn.getInputStream()
#isr = InputStreamReader(ist)
#br = BufferedReader(isr)
print("[BREACH] Received response: %d" % conn.getResponseCode())
if conn.getContentEncoding() != None:
print("[BREACH] Received Content-encoding: %s" %
(conn.getContentEncoding()))
return True
except:
print("[BREACH] Socket timeout or an error occurred")
return False
from javax.net.ssl import TrustManager, X509TrustManager
from jarray import array
from javax.net.ssl import SSLContext
class TrustAllX509TrustManager(X509TrustManager):
'''Define a custom TrustManager which will blindly accept all certificates'''
def checkClientTrusted(self, chain, auth):
pass
def checkServerTrusted(self, chain, auth):
pass
def getAcceptedIssuers(self):
return None
# Create a static reference to an SSLContext which will use
# our custom TrustManager
trust_managers = array([TrustAllX509TrustManager()], TrustManager)
TRUST_ALL_CONTEXT = SSLContext.getInstance("SSL")
TRUST_ALL_CONTEXT.init(None, trust_managers, None)
from javax.net.ssl import SSLContext
SSLContext.setDefault(TRUST_ALL_CONTEXT)
from javax.net.ssl import HostnameVerifier, HttpsURLConnection
class AllHostsVerifier(HostnameVerifier):
def verify(self, urlHostname, session):
return True
HttpsURLConnection.setDefaultHostnameVerifier(AllHostsVerifier())