PROCESS = CrawlerProcess({
'LOG_ENABLED': '1',
'LOG_LEVEL': 'ERROR',
# 'LOG_LEVEL': 'DEBUG',
'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36',
'DOWNLOADER_MIDDLEWARES': {'__main__.CustomMiddleware': 900},# Need to be > 600 to be after the redirectMiddleware
'DOWNLOADER_CLIENTCONTEXTFACTORY': 'scrapy_patch.CustomContextFactory'
})
PROCESS.crawl(
DocumentationSpider,
config=CONFIG,
algolia_helper=ALGOLIA_HELPER,
strategy=STRATEGY
)
PROCESS.start()
PROCESS.stop()
CONFIG.destroy()
ALGOLIA_HELPER.commit_tmp_index()
print("")
print('Nb hits: ' + str(DocumentationSpider.NB_INDEXED))
CONFIG.update_nb_hits(DocumentationSpider.NB_INDEXED)
print("")
