def checkrepeat(self, ip, tmp_ip):
'''检查是否重复字符'''
for tmp in tmp_ip:
if ip == tmp:
logger.log('INFOR', '[-]%s检查重复' % ip)
return True
return False
def request(self):
while True:
# if(urls_queue.empty() == False:
if(self.urls_queue.qsize()==0):
time.sleep(config_aoftest.queue_request_sleep)
# print(urls_queue.qsize())
req = self.urls_queue.get()
proxies = {
'http': 'http://'+config_aoftest.proxy,
'https': 'http://'+config_aoftest.proxy,
}
urls0 =req['url']
headers0 =req['headers']
method0=req['method']
# print(method0)
data0=req['data']
if method0 == 'GET':
try:
a = requests.get(urls0, headers=headers0, proxies=proxies,timeout=15,verify=False)
# time.sleep(0.5)
except:
# print('get错误')
pass
elif method0 == 'POST':
try:
requests.post(urls0, headers=headers0,data=data0, proxies=proxies,timeout=15,verify=False)
# time.sleep(0.5)
except:
# print('post错误')
pass
else:
logger.log('ALERT', '[-]存在其他HTTP方法请求，xray不支持')
def checkrepeat(self, s, fs): ## 查看url是否重复
# print(s)
if s in fs:
logger.log('INFOR', '[-]搜狗url扩展重复')
return True
else:
return False
def nmapscan(self):
nm = nmap.PortScanner()
logger.log('INFOR', '[+]开始nmap扫描')
nm.scan('None', None, config_aoftest.nmap_cmd + self.out + '/ip.txt')
with open(out + '/nmap.xml', 'w', encoding='utf-8') as file:
file.write(nm.get_nmap_last_output())
logger.log('INFOR', '[+]结束nmap扫描')
def ofa(self):
f = open(self.target, 'r')
folder = self.out
if not os.path.exists(folder): #判断是否存在文件夹如果不存在则创建为文件夹
os.makedirs(folder) #makedirs 创建文件时如果路径不存在会创建这个路径
logger.log('INFOR', "new folder")
else:
logger.log('INFOR', "There is this folder!")
lines = f.readlines()
for domain in lines:
domain = domain.strip('\n')
OneForAll(target=domain, alive=True, format='json').run()
f.close()
def collectUrl(self):
page_num = 0
while True:
url = 'https://www.baidu.com/s?wd=' + self.keyword + '&pn=' + str(
page_num)
page_num = page_num + 10
try:
resp = requests.get(url, headers=self.headers,
timeout=6) #得到url返回的搜索列表，包含10个值
# print(resp.text)
soup = bs(resp.text, features="html.parser")
bqs = soup.find_all(name='a',
attrs={
'data-click': re.compile(r'.'),
'class': None
})
# print(bqs)
if len(bqs):
for bq in bqs:
r = requests.get(bq['href'],
headers=self.headers,
timeout=3)
if r.status_code == 200:
logger.log('INFOR', '[+]百度url扩展:%s' % r.url)
with open(self.writePath, 'r') as f:
s = r.url + '\n'
fs = f.read()
if self.checkrepeat(s, fs):
continue
with open(self.writePath, 'a') as f:
f.write(r.url + '\n')
else:
logger.log('INFOR', '[-]百度搜索无响应') ##若返回列表为空，则只记录原始域名
with open(self.writePath, 'a') as f:
f.write('http://' + self.domain + '\n')
if '&pn={next_pn}&'.format(next_pn=page_num) not in resp.text:
# print(out)
break
except Exception as e: ##捕获到错误，记录原始域名并跳出循环
print(e)
with open(self.writePath, 'a') as f:
f.write('http://' + self.domain + '\n')
break
if page_num >= self.limit_num: # 搜索条数限制
break
def collectUrl(self):
page_num = 0
while True:
url = 'https://www.sogou.com/web?query=' + self.keyword + '&page=' + str(
page_num)
page_num = page_num + 1
try:
resp = requests.get(url, headers=self.headers,
timeout=6) #得到url返回的搜索列表，包含10个值
# print(resp.text)
soup = bs(resp.text, features="html.parser")
bqs = soup.find_all(name='a', attrs={'name': 'dttl'})
if len(bqs):
for bq in bqs:
href = 'https://www.sogou.com' + bq['href']
r = requests.get(href, headers=self.headers, timeout=3)
if r.status_code == 200:
p1 = re.compile(r'[(]\"(.*?)\"[)]', re.S)
url = re.findall(p1, r.text)
logger.log('INFOR', '[+]搜狗url扩展:%s' % url[0])
with open(self.writePath, 'r') as f:
s = url[0] + '\n'
fs = f.read()
if self.checkrepeat(s, fs):
continue
with open(self.writePath, 'a') as f:
f.write(url[0] + '\n')
else:
logger.log('INFOR', '[-]搜狗搜索无响应') ##若返回列表为空，则只记录原始域名
with open(self.writePath, 'a') as f:
f.write('http://' + self.domain + '\n')
if '<a id="sogou_next"' not in resp.text:
# print('out')
break
except Exception as e:
print(e)
with open(self.writePath, 'a') as f:
f.write('http://' + self.domain + '\n')
break
if page_num * 10 >= self.limit_num: # 搜索条数限制
break
def craw_thread(self):
'''
多线程爬虫
'''
t = threading.Thread(target=request)
t.start()
file = open(self.out+file_url, 'r')
f = open(self.out+'/request.txt', 'a')
for line in file.readlines():
line = line.strip('\n')
logger.log('INFOR', '[+]开始爬虫%s' % line)
cmd = config_aoftest.craw_thread_cmd
cmd.append(line)
rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
output, error = rsp.communicate(timeout=config_aoftest.craw_timeout)
except subprocess.TimeoutExpired:
logger.log('ERROR', "[-]爬虫超时")
rsp.kill()
continue
try:
result = json.loads(output.decode().split("--[Mission Complete]--")[1])
except:
logger.log('ERROR', '[-]爬虫json解析错误')
continue
req_list = result["req_list"]
sub_domain = result["sub_domain_list"]
logger.log('INFOR', "[+][crawl ok]\n")
if req_list:
for req in req_list:
f.write(req['url']+'\n')
self.urls_queue.put(req)
else:
logger.log('ERROR', "[-]req_list不存在\n")
continue
time.sleep(config_aoftest.craw_thread_sleep)
f.close()
file.close()
def statistic(self):
f = open(self.target, 'r')
f1 = open(self.out + '/subdomain.txt', 'a')
f2 = open(self.out + '/ip.txt', 'a')
f3 = open(self.out + '/url.txt', 'a')
lines = f.readlines()
for domain in lines:
domain = domain.strip('\n')
sub_file = pathlib.Path(__file__).parent.resolve().joinpath(
'oneforall/results', domain + '.json')
if not sub_file.is_file():
logger.log('ALERT', '[-]子域名文件：%s未发现' % sub_file)
continue
with open(sub_file, 'r', encoding='utf-8') as file:
tmp_str = file.read()
try:
tmp_dict = json.loads(tmp_str)
except:
logger.log('ERROR', '[-]子域名文件:%s解析json格式错误' % sub_file)
continue
tmp_ip = list()
for subdomain_dict in tmp_dict:
subdomain = subdomain_dict['subdomain'] # 取子域名
f1.write(subdomain + '\n')
url = subdomain_dict['url']
f3.write(url + '\n')
status = subdomain_dict['status']
if status is not None:
ip_str = subdomain_dict['content']
ip_list = ip_str.split(',')
for ip in ip_list:
if self.checkip(ip):
logger.log('INFOR', '[-]ip%s invaild' % ip)
continue
if self.checkrepeat(ip, tmp_ip): #如果检查出ip已存在，将不记录ip表中
logger.log('INFOR', '[-]ip%s 已存在' % ip)
continue
f2.write(ip + '\n')
tmp_ip.append(ip)
f.close()
f1.close()
f2.close()
f3.close()
def craw_alone(self):
'''
单独爬虫模块
'''
file = open(self.target, 'r')
if not os.path.exists(self.out): #判断是否存在文件夹如果不存在则创建为文件>夹
os.makedirs(self.out) #makedirs 创建文件时如果路径不存在会创建这个路径
logger.log('INFOR', "new folder... ")
else:
logger.log('INFOR', "There is this folder!")
f = open(self.out+'/request.txt', 'a')
for line in file.readlines():
line = line.strip('\n')
logger.log('INFOR', '[+]开始爬虫%s' % line)
cmd = config_aoftest.craw_cmd
# print(cmd)
cmd.append(line)
rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# print("before")
try:
output, error = rsp.communicate(timeout=config_aoftest.craw_timeout)
except subprocess.TimeoutExpired:
logger.log('ERROR', "[-]爬虫超时")
rsp.kill()
continue
# print("after")
try:
result = json.loads(output.decode().split("--[Mission Complete]--")[1])
except:
logger.log('ERROR', '[-]爬虫json解析错误')
continue
req_list = result["req_list"]
sub_domain = result["sub_domain_list"]
logger.log('INFOR', "[+][crawl ok]\n")
if req_list:
for req in req_list:
f.write(req['url']+'\n')
else:
logger.log('ERROR', "[-]req_list不存在\n")
continue
time.sleep(config_aoftest.craw_sleep)
f.close()
file.close()