def shell_main(): print(""" 输入 url, 响应数, 最大搜索页数, 空格隔开, 回车结束一次输入 比如\n http://h.nimingban.com/f/%E7%BB%BC%E5%90%88%E7%89%881 20 10\n 连续两次回车开始程序 """) tasks = [] while True: ip = input() if not ip: break url, response_gt, max_page = ip.split() tasks.append({ 'url': url, 'response_gt': int(response_gt), 'max_page': int(max_page), }) engine = Engine() engine.set_init_tasks(tasks) engine.start() print('-----------Engine Start----------') while engine.is_running: rs = engine.get_one_result() if not rs: time.sleep(0.5) continue print(rs.link, 'image', rs.image_url or 'None') print(rs.text) print('-'*40) print('------------Engine Stop---------------')
def do_crawler(self, para): print "do_crawler" #create crawler engin crawler_engine = Engine() #start engine crawler_engine.start() #stop engin crawler_engine.stop()
def do_crawler(self,para): print "do_crawler" #create crawler engin crawler_engine = Engine() #start engine crawler_engine.start( ) #stop engin crawler_engine.stop()
def main(): try: #create crawler engin crawler_engine = Engine() #start engine crawler_engine.start( ) #hold the main thread here, wait for any input to finish raw_input("") #stop engin crawler_engine.stop() except (Exception) as e: #Log().debug(e) sys.exit(0)
class Crawler(): def __init__(self): os.chdir('./global_vars') from core.engine import Engine os.chdir('..') self.engine = Engine() def _start_engine(self): self.engine.start() def _stop_engine(self): self.engine.stop() def crawl(self): # entrance method self._start_engine() def stop(self): self._stop_engine()
def main(): usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-u", "--url", dest="url", help="target URL") (options, args) = parser.parse_args() #options.url=targetUrl if options.url is None: parser.print_help() exit() t = Target(options.url) s = Engine(t) s.addOption("crawl", True) s.addOption("forms", True) s.addOption("threads", 1) if s.start(): exit()
def main(): banner() usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-u", "--url", dest="url", help="target URL") parser.add_option("--post", dest="post", default=False, action="store_true", help="try a post request to target url") parser.add_option("--data", dest="post_data", help="posta data to use") parser.add_option("--threads", dest="threads", default=1, help="number of threads") parser.add_option("--http-proxy", dest="http_proxy", help="scan behind given proxy (format: 127.0.0.1:80)") parser.add_option("--tor", dest="tor", default=False, action="store_true", help="scan behind default Tor") parser.add_option("--crawl", dest="crawl", default=False, action="store_true", help="crawl target url for other links to test") parser.add_option("--forms", dest="forms", default=False, action="store_true", help="crawl target url looking for forms to test") parser.add_option("--user-agent", dest="user_agent", help="provide an user agent") parser.add_option("--random-agent", dest="random_agent", default=False, action="store_true", help="perform scan with random user agents") parser.add_option("--cookie", dest="cookie", help="use a cookie to perform scans") parser.add_option("--dom", dest="dom", default=False, action="store_true", help="basic heuristic to detect dom xss") (options, args) = parser.parse_args() if options.url is None: parser.print_help() exit() # Build a first target print "[+] TARGET: %s" % options.url if options.post is True: print " |- METHOD: POST" if options.post_data is not None: print " |- POST data: %s" % options.post_data t = Target(options.url, method = 'POST', data = options.post_data) else: error('No POST data specified: use --data', ' |- ') exit() else: print " |- METHOD: GET" t = Target(options.url) # Build a scanner s = Engine(t) # Lets parse options for some proxy setting if options.http_proxy is not None and options.tor is True: error('No --tor and --http-proxy together!', ' |- ') exit() elif options.tor is False and options.http_proxy is not None: s.addOption("http-proxy", options.http_proxy) print " |- PROXY: %s" % options.http_proxy elif options.tor is True: s.addOption("http-proxy", "127.0.0.1:8118") print " |- PROXY: 127.0.0.1:8118" # User Agent option provided? if options.user_agent is not None and options.random_agent is True: error('No --user-agent and --random-agent together!', ' |- ') elif options.random_agent is False and options.user_agent is not None: s.addOption("ua", options.user_agent) print " |- USER-AGENT: %s" % options.user_agent elif options.random_agent is True: s.addOption("ua", "RANDOM") print " |- USER-AGENT: RANDOM" # Cookies? if options.cookie is not None: s.addOption("cookie", options.cookie) print " |- COOKIE: %s" % options.cookie # Do you want to crawl? if options.crawl is True: s.addOption("crawl", True) # Do you want to crawl forms? if options.forms is True: s.addOption("forms", True) # Dom scan? if options.dom is True: s.addOption("dom", True) # How many threads? s.addOption("threads", int(options.threads)) # Start the scanning if s.start(): exit()
def main(): manager, port, downloaders = parseCommandLineArgs() engine = Engine(downloaders, manager, port) engine.start() raw_input("press any key to stop....\n") engine.stop()
# project_dir/main.py # from core.engine import Engine # 导入引擎 # # from .spiders.baidu import BaiduSpider # from .spiders.douban import DoubanSpider # from .pipelines import BaiduPipeline, DoubanPipeline # # 此处新增 # from .spider_middlewares import TestDownloaderMiddleware1, TestDownloaderMiddleware2 # from .downloader_middlewares import TestSpiderMiddleware1, TestSpiderMiddleware2 # if __name__ == '__main__': # baidu_spider = BaiduSpider() # 实例化爬虫对象 # douban_spider = DoubanSpider() # 实例化爬虫对象 # spiders = {BaiduSpider.name: baidu_spider, DoubanSpider.name: douban_spider} # pipelines = [BaiduPipeline(), DoubanPipeline()] # 管道们 # spider_mids = [TestSpiderMiddleware1(), TestSpiderMiddleware2()] # 多个爬虫中间件 # downloader_mids = [TestDownloaderMiddleware1(), TestDownloaderMiddleware2()] # 多个下载中间件 # # engine = Engine(spiders) # 传入爬虫对象 # engine = Engine(spiders, pipelines=pipelines, # spider_mids=spider_mids, downloader_mids=downloader_mids) # 传入爬虫对象 # engine.start() # 启动引擎 # project_dir/main.py from core.engine import Engine # 导入引擎 if __name__ == '__main__': engine = Engine() # 创建引擎对象 engine.start() # 启动引擎
def main(): banner() usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-u", "--url", dest="url", help="target URL") parser.add_option("--post", dest="post", default=False, action="store_true", help="try a post request to target url") parser.add_option("--data", dest="post_data", help="posta data to use") parser.add_option("--threads", dest="threads", default=1, help="number of threads") parser.add_option("--http-proxy", dest="http_proxy", help="scan behind given proxy (format: 127.0.0.1:80)") parser.add_option("--tor", dest="tor", default=False, action="store_true", help="scan behind default Tor") parser.add_option("--crawl", dest="crawl", default=False, action="store_true", help="crawl target url for other links to test") parser.add_option("--forms", dest="forms", default=False, action="store_true", help="crawl target url looking for forms to test") parser.add_option("--user-agent", dest="user_agent", help="provide an user agent") parser.add_option("--random-agent", dest="random_agent", default=False, action="store_true", help="perform scan with random user agents") parser.add_option("--cookie", dest="cookie", help="use a cookie to perform scans") parser.add_option("--dom", dest="dom", default=False, action="store_true", help="basic heuristic to detect dom xss") (options, args) = parser.parse_args() if options.url is None: parser.print_help() exit() # Build a first target print "[+] TARGET: %s" % options.url if options.post is True: print " |- METHOD: POST" if options.post_data is not None: print " |- POST data: %s" % options.post_data t = Target(options.url, method='POST', data=options.post_data) else: error('No POST data specified: use --data', ' |- ') exit() else: print " |- METHOD: GET" t = Target(options.url) # Build a scanner s = Engine(t) # Lets parse options for some proxy setting if options.http_proxy is not None and options.tor is True: error('No --tor and --http-proxy together!', ' |- ') exit() elif options.tor is False and options.http_proxy is not None: s.addOption("http-proxy", options.http_proxy) print " |- PROXY: %s" % options.http_proxy elif options.tor is True: s.addOption("http-proxy", "127.0.0.1:8118") print " |- PROXY: 127.0.0.1:8118" # User Agent option provided? if options.user_agent is not None and options.random_agent is True: error('No --user-agent and --random-agent together!', ' |- ') elif options.random_agent is False and options.user_agent is not None: s.addOption("ua", options.user_agent) print " |- USER-AGENT: %s" % options.user_agent elif options.random_agent is True: s.addOption("ua", "RANDOM") print " |- USER-AGENT: RANDOM" # Cookies? if options.cookie is not None: s.addOption("cookie", options.cookie) print " |- COOKIE: %s" % options.cookie # Do you want to crawl? if options.crawl is True: s.addOption("crawl", True) # Do you want to crawl forms? if options.forms is True: s.addOption("forms", True) # Dom scan? if options.dom is True: s.addOption("dom", True) # How many threads? s.addOption("threads", int(options.threads)) # Start the scanning if s.start(): exit()