def setApiKeyFile(self, apiKeyFile):
if apiKeyFile == '' or apiKeyFile is None:
pluginException = PluginException(
message='The file specified is invalid.',
trace="setApiKeyFile with args apiKeyFile=%s " % (apiKeyFile),
plugin="shodan",
method="setApiKeyFile")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The file specified is invalid. "
raise pluginException
if os.path.exists(apiKeyFile) == False or os.path.isfile(
apiKeyFile) == False:
pluginException = PluginException(
message='The file specified is invalid.',
trace="setApiKeyFile with args apiKeyFile=%s " % (apiKeyFile),
plugin="shodan",
method="setApiKeyFile")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The file specified is invalid. "
raise pluginException
shodanKeyString = open(apiKeyFile).readline().rstrip('\n')
self.apiKey = shodanKeyString
print "[*] Shodan Key established!"
def setTarget(self, relayIp):
if relayIp is not None and relayIp != '':
if is_valid_ipv4_address(relayIp) or is_valid_ipv6_address(
relayIp):
self.hbExploit = HeartBleedExploit(relayIp)
print "[+] Target %s setted." % (relayIp)
return True
else:
pluginException = PluginException(
message='IP Address is Invalid ',
trace="setTarget with args %s " % (relayIp),
plugin="heartBleedPlugin",
method="setTarget")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
raise pluginException
else:
pluginException = PluginException(
message='IP Address is None or empty. Invalid value',
trace="setTarget with args %s " % (relayIp),
plugin="heartBleedPlugin",
method="setTarget")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
raise pluginException
def basicSearchByRelay(self, basicSearch, relay):
if basicSearch == None or basicSearch == '':
pluginException = PluginException(
message='The query specified is invalid.',
trace="basicSearchAllRelays with args basicSearch=%s , relay=%s"
% (basicSearch, relay),
plugin="shodan",
method="basicSearchAllRelays")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The query specified is invalid. "
raise pluginException
if is_valid_ipv4_address(relay) == False and is_valid_ipv6_address(
relay) == False and is_valid_domain(relay) == False:
pluginException = PluginException(
message='The relay specified is invalid. %s ' % (relay),
trace="basicSearchByRelay with args basicSearch=%s , relay=%s "
% (basicSearch, relay),
plugin="shodan",
method="basicSearchByRelay")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print '[-] The relay specified is invalid. %s ' % (relay)
raise pluginException
if hasattr(self, 'apiKey') and self.apiKey is not None:
shodanApi = shodan.Shodan(self.apiKey)
for node in self.torNodes:
if relay is not None and node.host == relay:
results = shodanApi.search(basicSearch + "net:" +
node.host)
table = Texttable()
table.set_cols_align(["l"])
table.set_cols_valign(["m"])
table.set_cols_width([55])
rows = [["Data"]]
if len(results['matches']) > 0:
print results
print "[*] Data for: %s " % (node.host)
for service in results['matches']:
rows.append(
[service['ip_str'] + "\n" + service['data']])
table.add_rows(rows)
print table.draw() + "\n"
else:
print "[*] No results for: %s " % (node.host)
else:
print "[*] Shodan API key not set. This is mandatory to perform searches using Shodan"
def basicSearchQuery(self, basicSearch, limit=10):
if limit == None or limit < 0:
pluginException = PluginException(
message='The limit specified is invalid.',
trace="basicSearchQuery with args basicSearch=%s , limit=%s " %
(basicSearch, str(limit)),
plugin="shodan",
method="basicSearchQuery")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The limit specified is invalid. "
raise pluginException
if basicSearch == None or basicSearch == '':
pluginException = PluginException(
message='The query specified is invalid.',
trace="basicSearchQuery with args basicSearch=%s , limit=%s " %
(basicSearch, str(limit)),
plugin="shodan",
method="basicSearchQuery")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The query specified is invalid. "
raise pluginException
if hasattr(self, 'apiKey') and self.apiKey is not None:
shodanApi = shodan.Shodan(self.apiKey)
results = shodanApi.search(basicSearch)
count = 0
table = Texttable()
table.set_cols_align(["l"])
table.set_cols_valign(["m"])
table.set_cols_width([55])
rows = [["Data"]]
for service in results['matches']:
if count == limit:
break
else:
count += 1
rows.append([service['ip_str'] + "\n" + service['data']])
table.add_rows(rows)
print table.draw() + "\n"
else:
print "[*] Shodan API key not set. This is mandatory to perform searches using Shodan"
def simpleStemmingAllRelays(self, queryTerms, httpMethod="GET", port=None):
if is_valid_port(port) == False:
pluginException = PluginException(
message='The port specified is invalid. %s ' % (str(port)),
trace=
"simpleStemmingAllRelays with args port=%s , queryTerms=%s " %
(str(port), queryTerms),
plugin="stemming",
method="simpleStemmingAllRelays")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The port specified is invalid. %s " % (str(port))
raise pluginException
for node in self.torNodes:
if port is not None and port in node.openPorts:
response = requests.get(
node.host + ":" + str(port),
timeout=self.pluginConfigs['timeOutRequests'])
self.__validateResponse(response, queryTerms)
else:
for port in node.openPorts:
if port.port in self.webPorts:
try:
response = self.serviceConnector.performHTTPConnection(
"http://" + node.host + ":" + str(port.port),
method=httpMethod)
self.__validateResponse(response, queryTerms)
except requests.ConnectionError:
continue
except requests.Timeout:
continue
def setApiKey(self, apiKey):
if apiKey == '' or apiKey is None:
pluginException = PluginException(
message='The Shodan key specified is invalid.',
trace="setApiKey with args apiKey=%s " % (apiKey),
plugin="shodan",
method="setApiKey")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The Shodan key specified is invalid. "
raise pluginException
self.apiKey = apiKey
print "[*] Shodan Key established!"
def dirBruterOnHiddenService(self, hiddenService, dictFile=''):
if is_valid_onion_address(hiddenService) == False:
pluginException = PluginException(
message=
"Invalid Onion Address %s must contain 16 characters. The TLD must be .onion"
% (hiddenService),
trace=
"dirBruterOnHiddenService with args hiddenService=%s, dictFile=%s "
% (hiddenService, dictFile),
plugin="dirBruter",
method="dirBruterOnHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenService)
raise pluginException
self.dirBruterOnRelay(hiddenService, dictFile=dictFile, proxy=True)
def dirBruterOnAllRelays(self, port=80, dictFile=''):
if is_valid_port(port) == False:
pluginException = PluginException(
message='The port specified is invalid. %s ' % (str(port)),
trace="dirBruterOnAllRelays with args port=%s , dictFile=%s " %
(str(port), dictFile),
plugin="dirBruter",
method="dirBruterOnAllRelays")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The port specified is invalid. %s " % (str(port))
raise pluginException
for relay in self.bruteForceData:
self.dirBruterOnRelay("http://" + relay,
dictFile=dictFile,
proxy=False)
def setDictForBruter(self, dictFile):
if dictFile == None or os.path.exists(
dictFile) == False or os.path.isfile(dictFile) == False:
pluginException = PluginException(
message='The regular expresion specified is invalid. %s ' %
(dictFile),
trace="deepWebCrawlerPlugin with args dictFile=%s " %
(str(dictFile)),
plugin="crawler",
method="setDictForBruter")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The file selected doesn't exists or is a directory."
raise pluginException
print "[+] Setting Dictionary File ... %s " % (dictFile)
self.dictFile = dictFile
def stemmingHiddenService(self, onionSite, queryTerms):
if is_valid_url(onionSite) == False:
pluginException = PluginException(
message="The URL specified is invalid. %s " % (onionSite),
trace=
"stemmingHiddenService with args onionSite=%s, queryTerms=%s "
% (onionSite, queryTerms),
plugin="stemming",
method="stemmingHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The URL specified is invalid. %s " % (onionSite)
raise pluginException
response = self.serviceConnector.performHTTPConnectionHiddenService(
onionSite, method="GET")
self.__validateResponse(response, queryTerms)
def setExtractorRulesDeny(self, extractorRulesDeny):
if is_valid_regex(extractorRulesDeny) == False:
pluginException = PluginException(
message='The regular expresion specified is invalid. %s ' %
(extractorRulesDeny),
trace="deepWebCrawlerPlugin with args extractorRulesDeny=%s " %
(str(extractorRulesDeny)),
plugin="crawler",
method="setExtractorRulesDeny")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print '[-] The regular expresion specified is invalid. %s ' % (
extractorRulesDeny)
raise pluginException
print "[+] Setting deny rules ... %s " % (extractorRulesDeny)
self.extractorRulesDeny = extractorRulesDeny
def setCrawlRulesImages(self, crawlRulesImages):
if is_valid_regex(crawlRulesImages) == False:
pluginException = PluginException(
message='The regular expresion specified is invalid. %s ' %
(crawlRulesImages),
trace="deepWebCrawlerPlugin with args crawlRulesImages=%s " %
(str(crawlRulesImages)),
plugin="crawler",
method="setCrawlRulesImages")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print '[-] The regular expresion specified is invalid. %s ' % (
crawlRulesImages)
raise pluginException
print "[+] Setting rules for the images extractor... %s " % (
crawlRulesImages)
self.crawlRulesImages = crawlRulesImages
def startHTTPHiddenService(self,
serviceDir,
servicePort=8080,
hiddenserviceDir=None,
hiddenservicePort=80,
serviceInterface='127.0.0.1',
socksPort=9152,
orPort=9000):
if is_valid_ipv4_address(
serviceInterface) == False and is_valid_ipv6_address(
serviceInterface) == False:
pluginException = PluginException(
message=
'The Service Interface is invalid. Try to use the default value without specify the parameter "serviceInterface" ',
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print '[-] The Service Interface is invalid. Try to use the default value without specify the parameter "serviceInterface". '
raise pluginException
if is_valid_port(hiddenservicePort) == False:
pluginException = PluginException(
message='The hidden service port is invalid.',
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The hidden service port is invalid. "
raise pluginException
if is_valid_port(socksPort) == False:
pluginException = PluginException(
message='The socks port is invalid.',
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The socks port is invalid. "
raise pluginException
if is_valid_port(orPort) == False:
pluginException = PluginException(
message='The OR port is invalid.',
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The OR port is invalid. "
raise pluginException
if is_valid_port(servicePort) == False:
pluginException = PluginException(
message='The Service port is invalid.',
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The Service port is invalid. "
raise pluginException
self.hiddenservicePort = hiddenservicePort
config = txtorcon.TorConfig()
config.SOCKSPort = socksPort
config.ORPort = orPort
if hiddenserviceDir is None:
print "[+] HiddenServiceDir not specified... Generating a temporal file."
hiddenserviceDir = self.__createTemporal()
if os.path.exists(hiddenserviceDir) == False:
print "[+] The HiddenServiceDir specified does not exists... Generating a temporal file."
hiddenserviceDir = self.__createTemporal()
if serviceDir is None or os.path.exists(serviceDir) == False:
pluginException = PluginException(
"The specified Server directory is not valid.",
trace=
"startHTTPHiddenService with args serviceDir=%s , servicePort=%s , hiddenserviceDir=%s , hiddenservicePort=%s , serviceInterface=%s , socksPort=%s , orPort=%s"
% (serviceDir, str(servicePort), hiddenserviceDir,
str(hiddenservicePort), serviceInterface, str(socksPort),
str(orPort)),
plugin="maliciousHiddenServicePlugin",
method="startHTTPHiddenService")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The specified Server directory is not valid. You must specify a valid directory where resources like HTML pages, images, CSS and stuff like that are located. The directory will be used to start a simple HTTP Server."
raise pluginException
config.HiddenServices = [
txtorcon.HiddenService(config, hiddenserviceDir, [
"%s %s:%s" %
(str(hiddenservicePort), serviceInterface, str(servicePort))
])
]
config.save()
root = static.File(serviceDir)
root.putChild("gatherUserInfo", GatherInformation())
site = server.Site(root)
hs_endpoint = TCP4ServerEndpoint(reactor,
servicePort,
interface=serviceInterface)
hs_endpoint.listen(site)
try:
d = txtorcon.launch_tor(
config,
reactor,
progress_updates=self.__updates,
)
except txtorcon.TorNotFound as torBinaryNotFound:
print "[-] Tor binary not found in the system path. Using the property 'torExecutablePath' located in the config/config.py file."
d = txtorcon.launch_tor(
config,
reactor,
tor_binary=tortazoConfig.torExecutablePath,
progress_updates=self.__updates,
)
d.addCallback(functools.partial(self.__setup_complete, config))
d.addErrback(self.__setup_failed)
reactor.run()
return True
def dirBruterOnRelay(self, site, dictFile='', proxy=False):
if is_valid_url(site) == False:
pluginException = PluginException(
message="The URL specified is invalid. %s " % (site),
trace=
"dirBruterOnRelay with args site=%s, dictFile=%s, proxy=%s " %
(site, dictFile, str(proxy)),
plugin="dirBruter",
method="dirBruterOnRelay")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The URL specified is invalid. %s " % (site)
raise pluginException
print "\n[+] Trying to find directories in the webserver %s " % (site)
print "[+] Verifying if the path %s is reachable ... " % (site)
try:
if proxy:
initialResponse = self.serviceConnector.performHTTPConnectionHiddenService(
site)
else:
initialResponse = self.serviceConnector.performHTTPConnection(
site)
if initialResponse.status_code in range(
400, 499) or initialResponse.status_code in range(
500, 599):
print "[-] The web server responded with an HTTP error Code ... HTTP %s " % (
str(initialResponse.status_code))
else:
if dictFile == '' or dictFile is None:
print "[+] No specified 'dictFile'. Using FuzzDB Project to execute the attack."
print "[+] Starting the attack using the FuzzDB Files ... This could take some time."
dirList = self.fuzzDBReader.getDirListFromFuzzDB()
for dir in dirList:
try:
resource = dir.replace('//', '/')
if proxy:
response = self.serviceConnector.performHTTPConnectionHiddenService(
site + resource)
else:
response = self.serviceConnector.performHTTPConnection(
site + resource)
if response.status_code in range(200, 299):
print "[+] Resource found! %s in %s" % (
dir, site + resource)
except InvalidURL:
continue
else:
print "[+] Using the 'dictFile' stored in %s. Verifing the file. " % (
dictFile, self.separator)
if os.path.exists(dictFile) == False or os.path.isfile(
dictFile) == False:
print "[-] The dictFile selected doesn't exists or is a directory."
return
else:
print "Executing the dictionary attack. Be patient."
for resource in open(dictFile, "r").readlines():
try:
if proxy:
response = self.serviceConnector.performHTTPConnectionHiddenService(
site + resource)
else:
response = self.serviceConnector.performHTTPConnection(
site + resource)
if response.status_code in range(200, 299):
print "[+] Resource found! %s in %s" % (
dir, site + resource)
except InvalidURL:
continue
except ConnectionError:
pluginException = PluginException(
message=
"Seems that the webserver in path %s is not reachable. Aborting the attack..."
% (site),
trace=
"dirBruterOnRelay with args site=%s, dictFile=%s, proxy=%s " %
(site, dictFile, str(proxy)),
plugin="dirBruter",
method="dirBruterOnRelay")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Seems that the webserver in path %s is not reachable. Aborting the attack..." % (
site)
raise pluginException
except Timeout:
pluginException = PluginException(
message=
"Seems that the webserver in path %s is not reachable. Aborting the attack..."
% (site),
trace=
"dirBruterOnRelay with args site=%s, dictFile=%s, proxy=%s " %
(site, dictFile, str(proxy)),
plugin="dirBruter",
method="dirBruterOnRelay")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Seems that the webserver in path %s is not reachable. Aborting the attack..." % (
site)
raise pluginException
def compareWebSiteWithHiddenWebSite(self, webSite, hiddenWebSite):
if webSite == '' or webSite is None:
pluginException = PluginException(
message="The URL specified is invalid. %s " % (webSite),
trace=
"compareWebSiteWithHiddenWebSite with args webSite=%s, hiddenWebSite=%s"
% (webSite, hiddenWebSite),
plugin="crawler",
method="compareWebSiteWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The URL specified is invalid. %s " % (webSite)
raise pluginException
if hiddenWebSite == '' or hiddenWebSite is None:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (hiddenWebSite),
trace=
"compareWebSiteWithHiddenWebSite with args webSite=%s, hiddenWebSite=%s"
% (webSite, hiddenWebSite),
plugin="crawler",
method="compareWebSiteWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
if hiddenWebSite.startswith('http://') == False:
hiddenWebSite = "http://" + hiddenWebSite
if webSite.startswith('http://') == False:
webSite = "http://" + webSite
if is_valid_onion_address(hiddenWebSite) == False:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (hiddenWebSite),
trace=
"compareWebSiteWithHiddenWebSite with args webSite=%s, hiddenWebSite=%s"
% (webSite, hiddenWebSite),
plugin="crawler",
method="compareWebSiteWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
if is_valid_url(webSite) == False:
pluginException = PluginException(
message="The URL specified is invalid. %s " % (webSite),
trace=
"compareWebSiteWithHiddenWebSite with args webSite=%s, hiddenWebSite=%s"
% (webSite, hiddenWebSite),
plugin="crawler",
method="compareWebSiteWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The URL specified is invalid. %s " % (webSite)
raise pluginException
try:
responseHidden = self.serviceConnector.performHTTPConnectionHiddenService(
hiddenWebSite, method="GET")
except Exception as exc:
import sys
print sys.exc_info()
print "[-] Exception connecting to the hidden service. Is the hidden service up and running? " + str(
exc.message)
return
ratio = 0
if responseHidden.status_code == 200:
try:
responseRelay = self.serviceConnector.performHTTPConnection(
webSite, method="GET")
if responseRelay.status_code == 200:
print "[+] Executing the matcher tool against the responses."
ratio = difflib.SequenceMatcher(
None, responseHidden.content,
responseRelay.content).ratio()
print "[+] Match ration between the web sites: %s " % (
str(ratio))
else:
print "[-] The website returned an non HTTP 200 code. %s " % (
str(responseRelay.status_code))
except Exception as exc:
print "[-] Exception connecting to the web service. Is the web service up and running? " + str(
exc.message)
return
else:
print "[-] The Hidden website returned an non HTTP 200 code. %s " % (
str(responseHidden.status_code))
print "[+] The percentage of equivalence between the contents of web sites found in the relays and hidden services are: \n"
table = Texttable()
table.set_cols_align(["l", "l", "c"])
table.set_cols_valign(["m", "m", "m"])
table.set_cols_width([15, 15, 15])
elements = [["Hidden Service", "WebSite", "Percentage"]]
elements.append([hiddenWebSite, webSite, str(ratio)])
table.add_rows(elements)
print table.draw() + "\n"
def crawlOnionWebSite(self,
hiddenWebSite,
hiddenWebSitePort=80,
socatTcpListenPort=8765,
crawlImages=False,
crawlLinks=True,
crawlContents=True,
crawlFormData=False,
useRandomUserAgents=True,
deepLinks=None,
bruterOnProtectedResource=False):
if hiddenWebSite == '' or hiddenWebSite is None:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (hiddenWebSite),
trace="crawlOnionWebSite with args hiddenWebSite=%s" %
(hiddenWebSite),
plugin="crawler",
method="crawlOnionWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
if hiddenWebSite.startswith('http://'):
onionSite = hiddenWebSite.replace('http://', "")
else:
onionSite = hiddenWebSite
if is_valid_onion_address(onionSite) == False:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (onionSite),
trace="crawlOnionWebSite with args hiddenWebSite=%s" %
(hiddenWebSite),
plugin="crawler",
method="crawlOnionWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
if is_valid_port(hiddenWebSitePort) == False:
pluginException = PluginException(
message='[-] The port specified is invalid. %s ' %
(str(hiddenWebSitePort)),
trace="crawlOnionWebSite with args hiddenWebSitePort=%s , " %
(str(hiddenWebSitePort)),
plugin="crawler",
method="crawlOnionWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The port specified is invalid. "
raise pluginException
if is_valid_port(socatTcpListenPort) == False:
pluginException = PluginException(
message='[-] The port specified is invalid. %s ' %
(str(socatTcpListenPort)),
trace="crawlOnionWebSite with args hiddenWebSitePort=%s , " %
(str(socatTcpListenPort)),
plugin="crawler",
method="crawlOnionWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The port specified is invalid. "
raise pluginException
onionSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
onionSocket.settimeout(1)
result = onionSocket.connect_ex(('127.0.0.1', socatTcpListenPort))
if result == 0:
pluginException = PluginException(
message="The selected local port " + str(socatTcpListenPort) +
" is being used by another process. Please, select an port available in this machine",
trace="crawlOnionWebSite with args socatTcpListenPort=%s , " %
(str(socatTcpListenPort)),
plugin="crawler",
method="crawlOnionWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] The selected local port " + str(
socatTcpListenPort
) + " is being used by another process. Please, select an port available in this machine"
raise pluginException
extraPath = onionSite[onionSite.find('.onion') + 6:]
#Extracts the Onion Site, everything before ".onion"
onionSite = onionSite[:onionSite.find('.onion') + 6]
try:
print "[+] Starting a local proxy with Socat to forward requests to the hidden service through the local machine and the local Socks Proxy... "
socatProcess = self.serviceConnector.startLocalSocatTunnel(
socatTcpListenPort,
onionSite,
hiddenWebSitePort,
socksPort=self.serviceConnector.socksPort)
sleep(5)
print "[+] Socat process started! PID: " + str(socatProcess.pid)
self.__crawl(hiddenWebSite,
socatTcpListenPort,
extraPath=extraPath,
crawlImages=crawlImages,
crawlLinks=crawlLinks,
crawlContents=crawlContents,
crawlFormData=crawlFormData,
useRandomUserAgents=useRandomUserAgents,
deepLinks=deepLinks,
bruterOnProtectedResource=bruterOnProtectedResource)
os.killpg(socatProcess.pid, signal.SIGTERM)
print "[+] Socat process killed."
except Exception as exce:
print "[+] The following exception was raised, however, shutting down the local Socat tunnel..."
print sys.exc_info()
print exce
os.killpg(socatProcess.pid, signal.SIGTERM)
sleep(5)
def compareRelaysWithHiddenWebSite(self, hiddenWebSite):
if hiddenWebSite == '' or hiddenWebSite is None:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (hiddenWebSite),
trace=
"compareRelaysWithHiddenWebSite with args hiddenWebSite=%s" %
(hiddenWebSite),
plugin="crawler",
method="compareRelaysWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
if hiddenWebSite.startswith('http://') == False:
hiddenWebSite = "http://" + hiddenWebSite
if is_valid_onion_address(hiddenWebSite) == False:
pluginException = PluginException(
message=
"Invalid Onion Adress %s must contain 16 characters. The TLD must be .onion"
% (hiddenWebSite),
trace=
"compareRelaysWithHiddenWebSite with args hiddenWebSite=%s" %
(hiddenWebSite),
plugin="crawler",
method="compareRelaysWithHiddenWebSite")
if self.runFromInterpreter:
showTrace(pluginException)
return
else:
print "[-] Invalid Onion Address %s must contain 16 characters. The TLD must be .onion" % (
hiddenWebSite)
raise pluginException
try:
responseHidden = self.serviceConnector.performHTTPConnectionHiddenService(
hiddenWebSite, method="GET")
except Exception as exc:
print "[-] Exception connecting to the hidden service. Is the hidden service up and running? " + str(
exc.message)
return
ratios = {}
for node in self.torNodes:
if responseHidden.status_code == 200:
try:
responseRelay = self.serviceConnector.performHTTPConnection(
'http://' + node.host, method="GET")
if responseRelay.status_code == 200:
print "[+] Executing the matcher tool against the responses."
ratio = difflib.SequenceMatcher(
None, responseHidden.content,
responseRelay.content).ratio()
ratios[node.host] = str(ratio)
except:
continue
print "[+] The percentage of equivalence between the contents of web sites found in the relays and hidden services are: \n"
table = Texttable()
table.set_cols_align(["l", "l", "c"])
table.set_cols_valign(["m", "m", "m"])
table.set_cols_width([15, 15, 15])
elements = [["Hidden Service", "Relay", "Percentage"]]
for key in ratios.keys():
elements.append([hiddenWebSite, str(key), str(ratios[key])])
table.add_rows(elements)
print table.draw() + "\\n"