def parser_subject(self):
try:
#certsearch?include_expired=true&include_subdomains=true&domain=jd.com
url = '{0}/certsearch?domain={1}&include_expired=true&include_subdomains=true'.format(
self.website, self.domain)
if self.req(url):
result = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-4]
total_page = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-1]
current_page = (self.result[6:-1]).replace("\n", "").replace(
"[", "").replace("]", "").split(",")[-2]
self.token = ast.literal_eval(result)
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
if domains != None:
self.dns_names.extend(domains)
'''
while current_page < total_page:#重复请求，页面未变，该如何修改页面呢？
url = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p={0}".format(self.token)
if self.req(url):
print "xxxxx"
current_page = \
(self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
print current_page
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
self.dns_names.extend(domains)
else:
break
'''
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return
def parser_subject(self):
try:
url = self.search_api.format(self.domain)
if self.req(url):
formated_string = (self.result[6:-1]).replace(
"\n", "").replace("[", "").replace("]", "")
self.token = formated_string.split(",")[-4]
#print result
#total_page = (self.result[6:-1]).replace("\n","").replace("[","").replace("]","").split(",")[-1]
#current_page = (self.result[6:-1]).replace("\n", "").replace("[", "").replace("]", "").split(",")[-2]
#it seems that no need to switch pages, the main purpose of this API is to get the related domains. it's enough using items in one page
rawres = parser(self.result, self.domain)
domains = rawres.hostnames()
if domains != None:
self.domain_name.extend(domains)
for item in formated_string.split(","): #
if len(
item
) >= 40 and " " not in item: #sSWg6vIw46sI1eNhDlilAaanXC9htQlVuMuHJWqyNr8=
item = item.strip("\"")
#item = unicode(item).decode()
item = item.replace("\\u003d", "=")
self.hash_codes.append(item)
except Exception as e:
logger.error("Error in {0}: {1}".format(
__file__.split('/')[-1], e))
return
def fetch_chinaz(self):
"""get subdomains from alexa.chinaz.com"""
url = '{0}?domain={1}'.format(self.url, self.domain)
r = http_request_get(url).content
rawres = parser(r, self.domain)
result = rawres.hostnames()
for sub in result:
self.domain_name.append(sub)
def fetch(self):
url = '{0}hostsearch/?q={1}'.format(self.url,self.domain)
try:
r = req.get(url,proxies = self.proxy).content
rawres = parser(r, self.domain)
result = rawres.hostnames()
for sub in result:
self.domain_name.append(sub)
except Exception,e:
logger.error("Error in {0}: {1}".format(__file__.split('/')[-1], e))
def fetch_chinaz(self):
"""get subdomains from alexa.chinaz.com"""
url = '{0}?domain={1}'.format(self.url,self.domain)
try:
r = req.get(url,proxies = self.proxy).content
rawres = parser(r, self.domain)
result = rawres.hostnames()
for sub in result:
self.domain_name.append(sub)
except Exception,e:
logger.error("Error in {0}: {1}".format(__file__.split('/')[-1], e))
def run(self):
try:
url = "{0}/getData.php?e=subdomains_container&q={1}&t=0&rt=10&p=1".format(self.website, self.domain)
# content = curl_get_content(url).get('resp')
content = http_request_get(url).content
rawres = parser(content, self.domain)
self.domain_name = rawres.hostnames()
except Exception as e:
logger.error(str(e))
logger.info("{0} found {1} domains".format(self.engine_name, len(self.domain_name)))
return self.domain_name,self.smiliar_domain_name,self.email
def run(self):
try:
url = "{0}/getData.php?e=subdomains_container&q={1}&t=0&rt=10&p=1".format(self.website, self.domain)
# content = curl_get_content(url).get('resp')
content = req.get(url,proxies = self.proxy).content
rawres = parser(content, self.domain)
self.domain_name = rawres.hostnames()
except Exception as e:
logger.error("Error in {0}: {1}".format(__file__.split('/')[-1], e))
logger.info("{0} found {1} domains".format(self.engine_name, len(self.domain_name)))
return self.domain_name,self.smiliar_domain_name,self.related_domain_name,self.email
def get_hostnames(self):
rawres = myparser.parser(self.results, self.domain)
return rawres.hostnames()
def get_emails(self):
rawres = myparser.parser(self.results, self.domain)
return rawres.emails()
def get_hostnames(self):
rawres = parser(self.totalresults, self.word)
return rawres.hostnames()
def get_hostnames(self):
rawres = myparser.parser(self.total_results, self.key_word)
return rawres.hostnames()
def get_hostnames(self):
rawres = parser(self.resp, self.domain)
return rawres.hostnames()
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
#print "%s domain(s) found in Baidu" %len(rawres.hostnames())
return rawres.hostnames()
def get_emails(self):
rawres = myparser.parser(self.totalresults, self.word)
#print "%s email(s) found in Baidu" %len(rawres.emails())
return rawres.emails()
def get_profiles(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.profiles()
def get_files(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.fileurls(self.files)
def get_email(self):
rawres = parser(self.resp, self.domain)
return rawres.emails()
def get_allhostnames(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.hostnames_all()
def get_people(self):
rawres = myparser.parser(self.totalresults, self.word)
return rawres.people_jigsaw()
