def createSpider(self, kwags):
        site = kwags.get("site", "")
        site = site

        str_ptn = kwags.get("ptns", "")
        ptn = re.compile(str_ptn)
        ptns = [ptn]

        onScapyDic = {"onScapy": OnScapy(), "print": print_OnScapy(), "logfile": logfile_OnScapy()}

        max_size = kwags.get("max_size", "100")
        max_size = int(max_size)

        onScapy = kwags.get("output_type", "onScapy")
        onscapy = onScapyDic[onScapy]
        logfileName = kwags.get("output_name", None)

        multi = kwags.get("multi_thread", "False")
        threadnumber = kwags.get("threadnumber", None)

        if not site:
            raise Exception("not site input")
        if logfileName and onScapy == "logfile":
            onscapy = logfile_OnScapy(logfileName)

        spiderCls = Spider

        if multi == "True":
            spiderCls = MultiSpider
        spider = spiderCls(site, ptns, max_size, onScapy=onscapy)

        if multi == "True" and threadnumber:
            threadnumber = int(threadnumber)
            spider.setThreadNumber(threadnumber)
        return spider
def test_cls_MultiScapy_scapy__logfile_OnScapy():
    site = "http://www.hao123.com/"
    max_size = 100
    ptns = producePtns()
    name = 'test_result/test_cls_MultiScapy_scapy__logfile_OnScapy'
    onscapy = logfile_OnScapy(name=name)
    multispider = MultiSpider(site,ptns,max_size=max_size,onScapy=onscapy)
    multispider.scapy()
def test_cls_Spider_scapy__logfile_OnScapy():
    site = "http://www.hao123.com/"
    max_size = 100
    logfile = "test_result/logfile"
    ptns = producePtns()
    onscapy = logfile_OnScapy(name=logfile)
    spider = Spider(site,ptns,max_size=max_size,onScapy=onscapy)
    spider.scapy()
def test_cls_logfile_OnScapy():
    name = "test_result/logfile"
    url,html = 'url','html'
    on_scapy = logfile_OnScapy(name=name)
    on_scapy.beforeScapy()
    on_scapy.filterHtml(url, html)
    on_scapy.finishScapy()
    url = url+'\n'
    f = open(name)
    logfile = f.read()
    f.close()
    assert logfile==url,('logfile_OnScapy class error',logfile,url)