def CScan(ip):
try:
rep = requests.get("http://" + ip, headers=config.GetHeaders(), timeout=2, verify=False)
if rep.status_code != 404:
title = re.findall(r'<title>(.*?)</title>', rep.text)
if title:
return "[T]" + ip + ' >>> ' + title[0] + "\n"
else:
return "[H]" + ip + " >>> have reason\n"
except Exception as e:
pass
def urlCheck(target):
try:
print("now url live check: {}".format(target))
rep = requests.get(target,
headers=config.GetHeaders(),
timeout=2,
verify=False)
if rep.status_code != 404:
return True
except Exception as e:
# print(e)
return False
return False
def crawlergoGet(target):
print("Now crawlergoGet : {}".format(target))
try:
cmd = [config.crawlergo_Path, "-c", config.Chrome_Path, "--custom-headers", json.dumps(config.GetHeaders()), "-t", "10", "-f",
"smart", "-o", "json", target]
rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = rsp.communicate()
# "--[Mission Complete]--" 是任务结束的分隔字符串
result = simplejson.loads(output.decode().split("--[Mission Complete]--")[1])
# print(result)
req_list = result["req_list"]
except Exception as e:
print(e)
req_list=[]
pass
print("target {} crawlergo end~".format(target))
print("crawlergo get url number {}".format(len(req_list)))
return removeDuplicates(req_list)