def dic_scan(web, dictionary_loc, export_filename="", to=0.4, proxy=None,ua=None,ignore_text=""):
if 0 == len(export_filename):
export_filename = web
web = web_deal(web)[0]
export_filename = result.initialize_webframe(web_deal(web)[1]) # use result to create web form
dic_f = open(dictionary_loc) # open dictionary
line = dic_f.readline()
web_length = len(web)
while line: # read one line by line
line = line.strip('\n') # remove the line feed
if line.startswith("/"):
web = web + line
else:
web = web + "/" + line
try:
code = requests.get(web, timeout=to,proxies=proxy,headers=ua).status_code
if "" != ignore_text and ignore_text not in requests.get(web).text:
printweb(code,web)
result.export_result(export_filename, web,web+"---"+str(code))
elif "" == ignore_text:
printweb(code,web)
result.export_result(export_filename, web,web+"---"+str(code))
else:
pass
except:
printf(web+"\t\t\tConnect wrong!!!","error")
web = web[0:web_length]
line = dic_f.readline()
if line == None:
break
def initialize_webframe(filename): # init the web form
web_frame = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>WMIN result</title>
</head>
<body>
<center>
<h1>
<b>
Website Miner
</b>
</h1>
</center>
</body>
</html>"""
export_filename = filename + ".html" # set output filename
try:
export_file = open(export_filename, "w") # use write to open file
except:
printf("result file can't be created!","error")
export_file.write(web_frame) # write to web form
export_file.close()
return export_filename # return filename
def initialize_webframe(filename): # init the web form
web_frame = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>WMIN result</title>
</head>
<body>
<center>
<h1>
<b>
Website Miner
</b>
</h1>
</center>
</body>
</html>"""
export_filename = filename + ".html" # set output filename
try:
export_file = open(export_filename, "w") # use write to open file
except:
printf("result file can't be created!", "error")
export_file.write(web_frame) # write to web form
export_file.close()
return export_filename # return filename
def export_result(export_file, web,web_text): # export result file
try: # open file
export_f = open(export_file, "a+") #use add to open file
export_f.seek(274, 0) # turn after to</center>
except:
printf("Cann't export the result!!!","error")
# set format to html
export_web = "<li><a href=\"{0}\">{1}</a></li>".format(web, web_text)
export_web_len = len(export_web)
export_f.write(export_web)
export_f.seek(0, 0) # goto file's head
export_f.close()
def export_result(export_file, web, web_text): # export result file
try: # open file
export_f = open(export_file, "a+") #use add to open file
export_f.seek(274, 0) # turn after to</center>
except:
printf("Cann't export the result!!!", "error")
# set format to html
export_web = "<li><a href=\"{0}\">{1}</a></li>".format(web, web_text)
export_web_len = len(export_web)
export_f.write(export_web)
export_f.seek(0, 0) # goto file's head
export_f.close()
def get_info(web,timeout=0.4,proxy=None,ua=None):
try:
printf("Server:\t"+requests.get(web_deal(web)[0],timeout=timeout,proxies=proxy,headers=ua).headers["Server"],"normal")
except:
printf("Can\'t get server,Connect wrong","error")
try:
printf("IP:\t"+socket.gethostbyname(web_deal(web)[1]),"normal")
except:
printf("Can\'t get ip,Connect wrong","error")
proxy = None
ua = None
ignore_text = ""
url = ""
dictionary = ""
use_help = 0
U = False
D = False
if version_info.major == 3:
from printf.py3 import printf
else:
from printf.py2 import printf
if len(argv) <= 1:
printf(info)
else:
argv.pop(0) # delete the argv[0]
for i in range(0, len(argv)):
if "-u" == argv[i]: # set url
try:
url = argv[i + 1]
U = True
except IndexError:
printf("-u No argv!","error")
elif "-d" == argv[i]: # set dictionary
try:
dictionary = argv[i + 1]
proxy = None
ua = None
ignore_text = ""
url = ""
dictionary = ""
use_help = 0
U = False
D = False
if version_info.major == 3:
from printf.py3 import printf
else:
from printf.py2 import printf
if len(argv) <= 1:
printf(info)
else:
argv.pop(0) # delete the argv[0]
for i in range(0, len(argv)):
if "-u" == argv[i]: # set url
try:
url = argv[i + 1]
U = True
except IndexError:
printf("-u No argv!", "error")
elif "-d" == argv[i]: # set dictionary
try:
dictionary = argv[i + 1]