def init_rule(): engine = db_connect() create_news_table(engine) Session = sessionmaker(bind=engine) with session_scope(Session) as session: artile_rule1 = ArticleRule( name='huxiu', allow_domains='huxiu.com', start_urls='http://www.huxiu.com/', next_page='', allow_url='/article/\d+/\d+\.html', extract_from='//div[@class="mod-info-flow"]', title_xpath='//div[@class="article-wrap"]/h1/text()', body_xpath='//div[@id="article_content"]/p//text()', publish_time_xpath='//span[@class="article-time"]/text()', source_site='虎嗅网', enable=1 ) artile_rule2 = ArticleRule( name='osc', allow_domains='oschina.net', start_urls='http://www.oschina.net/', next_page='', allow_url='/news/\d+/', extract_from='//div[@id="IndustryNews"]', title_xpath='//h1[@class="OSCTitle"]/text()', publish_time_xpath='//div[@class="PubDate"]/text()', body_xpath='//div[starts-with(@class, "Body")]/p[position()>1]//text()', source_site='开源中国', enable=1 ) session.add(artile_rule1) session.add(artile_rule2)
def init_rule(): engine = db_connect() create_news_table(engine) Session = sessionmaker(bind=engine) with session_scope(Session) as session: artile_rule1 = ArticleRule( name='huxiu', allow_domains='huxiu.com', start_urls='http://www.huxiu.com/', next_page='', allow_url='/article/\d+/\d+\.html', extract_from='//div[@class="mod-info-flow"]', title_xpath='//div[@class="article-wrap"]/h1/text()', body_xpath='//div[@id="article_content"]/p//text()', publish_time_xpath='//span[@class="article-time"]/text()', source_site='虎嗅网', enable=1) artile_rule2 = ArticleRule( name='osc', allow_domains='oschina.net', start_urls='http://www.oschina.net/', next_page='', allow_url='/news/\d+/', extract_from='//div[@id="IndustryNews"]', title_xpath='//h1[@class="OSCTitle"]/text()', publish_time_xpath='//div[@class="PubDate"]/text()', body_xpath= '//div[starts-with(@class, "Body")]/p[position()>1]//text()', source_site='开源中国', enable=1) session.add(artile_rule1) session.add(artile_rule2)
def __init__(self): """ Initializes database connection and sessionmaker. Creates deals table. """ engine = db_connect() create_news_table(engine) # 初始化对象属性Session为可调用对象 self.Session = sessionmaker(bind=engine) self.recent_links = None self.nowtime = datetime.datetime.now()
def __init__(self): """ Initializes database connection and sessionmaker. Creates deals table. """ engine = db_connect() create_news_table(engine) # 初始化对象属性Session为可调用对象 self.Session = sessionmaker(bind=engine) self.recent_links = None self.nowtime = datetime.datetime.now()
def __init__(self): engine = db_connect() self.Session = sessionmaker(bind=engine)
def __init__(self): engine = db_connect() create_news_table(engine) self.Session = sessionmaker(bind=engine)
""" import logging from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings from scrapy.utils.log import configure_logging from coolscrapy.models import db_connect, create_news_table from coolscrapy.models import ArticleRule from sqlalchemy.orm import sessionmaker from coolscrapy.spiders.article_spider import ArticleSpider if __name__ == '__main__': settings = get_project_settings() configure_logging(settings) db = db_connect() Session = sessionmaker(bind=db) session = Session() rules = session.query(ArticleRule).filter(ArticleRule.enable == 1).all() session.close() runner = CrawlerRunner(settings) for rule in rules: # spider = ArticleSpider(rule) # instantiate every spider using rule # stop reactor when spider closes # runner.signals.connect(spider_closing, signal=signals.spider_closed) runner.crawl(ArticleSpider, rule=rule) d = runner.join() d.addBoth(lambda _: reactor.stop())
""" import logging from spiders.article_spider import ArticleSpider from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings from scrapy.utils.log import configure_logging from coolscrapy.models import db_connect from coolscrapy.models import ArticleRule from sqlalchemy.orm import sessionmaker if __name__ == '__main__': settings = get_project_settings() configure_logging(settings) db = db_connect() Session = sessionmaker(bind=db) session = Session() rules = session.query(ArticleRule).filter(ArticleRule.enable == 1).all() session.close() runner = CrawlerRunner(settings) for rule in rules: # spider = ArticleSpider(rule) # instantiate every spider using rule # stop reactor when spider closes # runner.signals.connect(spider_closing, signal=signals.spider_closed) runner.crawl(ArticleSpider, rule=rule) d = runner.join() d.addBoth(lambda _: reactor.stop())
def __init__(self): engine = db_connect() create_news_table(engine) self.Session = sessionmaker(bind=engine)
def __init__(self): logg.info("Init ProxyDatabasePipeline") engine = db_connect() self.Session = sessionmaker(bind=engine) logg.info(self.Session)