def shell_main():
print("""
输入 url， 响应数， 最大搜索页数， 空格隔开, 回车结束一次输入
比如\n
http://h.nimingban.com/f/%E7%BB%BC%E5%90%88%E7%89%881 20 10\n
连续两次回车开始程序
""")
tasks = []
while True:
ip = input()
if not ip:
break
url, response_gt, max_page = ip.split()
tasks.append({
'url': url,
'response_gt': int(response_gt),
'max_page': int(max_page),
})
engine = Engine()
engine.set_init_tasks(tasks)
engine.start()
print('-----------Engine Start----------')
while engine.is_running:
rs = engine.get_one_result()
if not rs:
time.sleep(0.5)
continue
print(rs.link, 'image', rs.image_url or 'None')
print(rs.text)
print('-'*40)
print('------------Engine Stop---------------')
def do_crawler(self, para):
print "do_crawler"
#create crawler engin
crawler_engine = Engine()
#start engine
crawler_engine.start()
#stop engin
crawler_engine.stop()
def do_crawler(self,para): print "do_crawler" #create crawler engin crawler_engine = Engine() #start engine crawler_engine.start( ) #stop engin crawler_engine.stop()
def main():
try:
#create crawler engin
crawler_engine = Engine()
#start engine
crawler_engine.start( )
#hold the main thread here, wait for any input to finish
raw_input("")
#stop engin
crawler_engine.stop()
except (Exception) as e:
#Log().debug(e)
sys.exit(0)
class Crawler():
def __init__(self):
os.chdir('./global_vars')
from core.engine import Engine
os.chdir('..')
self.engine = Engine()
def _start_engine(self):
self.engine.start()
def _stop_engine(self):
self.engine.stop()
def crawl(self): # entrance method
self._start_engine()
def stop(self):
self._stop_engine()
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-u", "--url", dest="url", help="target URL")
(options, args) = parser.parse_args()
#options.url=targetUrl
if options.url is None:
parser.print_help()
exit()
t = Target(options.url)
s = Engine(t)
s.addOption("crawl", True)
s.addOption("forms", True)
s.addOption("threads", 1)
if s.start():
exit()
def main():
banner()
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-u", "--url", dest="url", help="target URL")
parser.add_option("--post", dest="post", default=False, action="store_true",
help="try a post request to target url")
parser.add_option("--data", dest="post_data", help="posta data to use")
parser.add_option("--threads", dest="threads", default=1,
help="number of threads")
parser.add_option("--http-proxy", dest="http_proxy",
help="scan behind given proxy (format: 127.0.0.1:80)")
parser.add_option("--tor", dest="tor", default=False, action="store_true",
help="scan behind default Tor")
parser.add_option("--crawl", dest="crawl", default=False, action="store_true",
help="crawl target url for other links to test")
parser.add_option("--forms", dest="forms", default=False, action="store_true",
help="crawl target url looking for forms to test")
parser.add_option("--user-agent", dest="user_agent",
help="provide an user agent")
parser.add_option("--random-agent", dest="random_agent", default=False,
action="store_true",
help="perform scan with random user agents")
parser.add_option("--cookie", dest="cookie",
help="use a cookie to perform scans")
parser.add_option("--dom", dest="dom", default=False, action="store_true",
help="basic heuristic to detect dom xss")
(options, args) = parser.parse_args()
if options.url is None:
parser.print_help()
exit()
# Build a first target
print "[+] TARGET: %s" % options.url
if options.post is True:
print " |- METHOD: POST"
if options.post_data is not None:
print " |- POST data: %s" % options.post_data
t = Target(options.url, method = 'POST', data = options.post_data)
else:
error('No POST data specified: use --data', ' |- ')
exit()
else:
print " |- METHOD: GET"
t = Target(options.url)
# Build a scanner
s = Engine(t)
# Lets parse options for some proxy setting
if options.http_proxy is not None and options.tor is True:
error('No --tor and --http-proxy together!', ' |- ')
exit()
elif options.tor is False and options.http_proxy is not None:
s.addOption("http-proxy", options.http_proxy)
print " |- PROXY: %s" % options.http_proxy
elif options.tor is True:
s.addOption("http-proxy", "127.0.0.1:8118")
print " |- PROXY: 127.0.0.1:8118"
# User Agent option provided?
if options.user_agent is not None and options.random_agent is True:
error('No --user-agent and --random-agent together!', ' |- ')
elif options.random_agent is False and options.user_agent is not None:
s.addOption("ua", options.user_agent)
print " |- USER-AGENT: %s" % options.user_agent
elif options.random_agent is True:
s.addOption("ua", "RANDOM")
print " |- USER-AGENT: RANDOM"
# Cookies?
if options.cookie is not None:
s.addOption("cookie", options.cookie)
print " |- COOKIE: %s" % options.cookie
# Do you want to crawl?
if options.crawl is True:
s.addOption("crawl", True)
# Do you want to crawl forms?
if options.forms is True:
s.addOption("forms", True)
# Dom scan?
if options.dom is True:
s.addOption("dom", True)
# How many threads?
s.addOption("threads", int(options.threads))
# Start the scanning
if s.start():
exit()
def main():
manager, port, downloaders = parseCommandLineArgs()
engine = Engine(downloaders, manager, port)
engine.start()
raw_input("press any key to stop....\n")
engine.stop()
# project_dir/main.py
# from core.engine import Engine # 导入引擎
#
# from .spiders.baidu import BaiduSpider
# from .spiders.douban import DoubanSpider
# from .pipelines import BaiduPipeline, DoubanPipeline
# # 此处新增
# from .spider_middlewares import TestDownloaderMiddleware1, TestDownloaderMiddleware2
# from .downloader_middlewares import TestSpiderMiddleware1, TestSpiderMiddleware2
# if __name__ == '__main__':
# baidu_spider = BaiduSpider() # 实例化爬虫对象
# douban_spider = DoubanSpider() # 实例化爬虫对象
# spiders = {BaiduSpider.name: baidu_spider, DoubanSpider.name: douban_spider}
# pipelines = [BaiduPipeline(), DoubanPipeline()] # 管道们
# spider_mids = [TestSpiderMiddleware1(), TestSpiderMiddleware2()] # 多个爬虫中间件
# downloader_mids = [TestDownloaderMiddleware1(), TestDownloaderMiddleware2()] # 多个下载中间件
# # engine = Engine(spiders) # 传入爬虫对象
# engine = Engine(spiders, pipelines=pipelines,
# spider_mids=spider_mids, downloader_mids=downloader_mids) # 传入爬虫对象
# engine.start() # 启动引擎
# project_dir/main.py
from core.engine import Engine # 导入引擎
if __name__ == '__main__':
engine = Engine() # 创建引擎对象
engine.start() # 启动引擎
def main():
banner()
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-u", "--url", dest="url", help="target URL")
parser.add_option("--post",
dest="post",
default=False,
action="store_true",
help="try a post request to target url")
parser.add_option("--data", dest="post_data", help="posta data to use")
parser.add_option("--threads",
dest="threads",
default=1,
help="number of threads")
parser.add_option("--http-proxy",
dest="http_proxy",
help="scan behind given proxy (format: 127.0.0.1:80)")
parser.add_option("--tor",
dest="tor",
default=False,
action="store_true",
help="scan behind default Tor")
parser.add_option("--crawl",
dest="crawl",
default=False,
action="store_true",
help="crawl target url for other links to test")
parser.add_option("--forms",
dest="forms",
default=False,
action="store_true",
help="crawl target url looking for forms to test")
parser.add_option("--user-agent",
dest="user_agent",
help="provide an user agent")
parser.add_option("--random-agent",
dest="random_agent",
default=False,
action="store_true",
help="perform scan with random user agents")
parser.add_option("--cookie",
dest="cookie",
help="use a cookie to perform scans")
parser.add_option("--dom",
dest="dom",
default=False,
action="store_true",
help="basic heuristic to detect dom xss")
(options, args) = parser.parse_args()
if options.url is None:
parser.print_help()
exit()
# Build a first target
print "[+] TARGET: %s" % options.url
if options.post is True:
print " |- METHOD: POST"
if options.post_data is not None:
print " |- POST data: %s" % options.post_data
t = Target(options.url, method='POST', data=options.post_data)
else:
error('No POST data specified: use --data', ' |- ')
exit()
else:
print " |- METHOD: GET"
t = Target(options.url)
# Build a scanner
s = Engine(t)
# Lets parse options for some proxy setting
if options.http_proxy is not None and options.tor is True:
error('No --tor and --http-proxy together!', ' |- ')
exit()
elif options.tor is False and options.http_proxy is not None:
s.addOption("http-proxy", options.http_proxy)
print " |- PROXY: %s" % options.http_proxy
elif options.tor is True:
s.addOption("http-proxy", "127.0.0.1:8118")
print " |- PROXY: 127.0.0.1:8118"
# User Agent option provided?
if options.user_agent is not None and options.random_agent is True:
error('No --user-agent and --random-agent together!', ' |- ')
elif options.random_agent is False and options.user_agent is not None:
s.addOption("ua", options.user_agent)
print " |- USER-AGENT: %s" % options.user_agent
elif options.random_agent is True:
s.addOption("ua", "RANDOM")
print " |- USER-AGENT: RANDOM"
# Cookies?
if options.cookie is not None:
s.addOption("cookie", options.cookie)
print " |- COOKIE: %s" % options.cookie
# Do you want to crawl?
if options.crawl is True:
s.addOption("crawl", True)
# Do you want to crawl forms?
if options.forms is True:
s.addOption("forms", True)
# Dom scan?
if options.dom is True:
s.addOption("dom", True)
# How many threads?
s.addOption("threads", int(options.threads))
# Start the scanning
if s.start():
exit()
def main():
manager, port, downloaders = parseCommandLineArgs()
engine = Engine(downloaders, manager, port)
engine.start()
raw_input("press any key to stop....\n")
engine.stop()
