Esempi in Python per get_dict_from_module

Esempio n. 1

File: run_duanzi_crawler.py Progetto: thanq/jiandan_duanzi

def crawl_duanzi():
    # 爬虫开始
    import jiandan_duanzi.settings as s

    setting_dict = get_dict_from_module(s)
    setting_dict["is_full_amount_spider"] = True
    setting_dict["DEPTH_LIMIT"] = 20
    process = CrawlerProcess(setting_dict)
    process.crawl(DuanziSpider)
    process.start()  # the script will block here until the crawling is finished

    print "@" * 100
    pass

Esempio n. 2

File: run_everyday_new_duanzi_crawler.py Progetto: thanq/jiandan_duanzi

def crawl_duanzi():
    #爬虫开始
    import jiandan_duanzi.settings as s
    s.ITEM_PIPELINES = {'jiandan_duanzi.pipelines.NewJiandanDuanziPipeline':100}
    setting_dict = get_dict_from_module(s)
    setting_dict['is_full_amount_spider'] = True
    setting_dict['DEPTH_LIMIT'] = 10

    process = CrawlerProcess(setting_dict)
    process.crawl(NewDuanziSpider)
    process.start() # the script will block here until the crawling is finished

    print '@'*100
    pass