"headers": {
"User-Agent": constants.USER_AGENT,
"Content-Type": constants.CONTENT_TYPE
},
"cookies": {
"language": constants.LANGUAGE,
"vts_usr_lg": constants.USER_COOKIE
},
"meta": {
"ticker": "", # ticker
"ReportType": "" # document type, use this so we can use TickerCrawlSpiderMiddleware
}
}
log_settings = utilities.log_settings(spiderName=name,
log_level="INFO",
log_formatter="scraper_vietstock.spiders.models.utilities.TickerSpiderLogFormatter"
)
middlewares_settings = {
'DOWNLOADER_MIDDLEWARES': {
'rotating_proxies.middlewares.RotatingProxyMiddleware': 610,
'rotating_proxies.middlewares.BanDetectionMiddleware': 620,
# 'scraper_vietstock.middlewares.TickerCrawlDownloaderMiddleware': 901,
# 'scraper_vietstock.fad_stats.TickerCrawlerStats': 850,
'scrapy.downloadermiddlewares.stats.DownloaderStats': None,
},
'SPIDER_MIDDLEWARES': {
# 'scraper_vietstock.middlewares.TickerCrawlSpiderMiddleware': 45
}
}
industry_list = {
"url": "https://finance.vietstock.vn/data/industrylist",
"headers": {
"User-Agent": constants.USER_AGENT
},
"cookies": {
"language": constants.LANGUAGE,
"__RequestVerificationToken": constants.REQ_VER_TOKEN_COOKIE
},
"meta": {
"bizType_id": "",
"bizType_title": "",
}
}
log_settings_regular = utilities.log_settings(spiderName=name_regular,
log_level="INFO")
middlewares_settings = {
'DOWNLOADER_MIDDLEWARES': {
'rotating_proxies.middlewares.RotatingProxyMiddleware': 610,
'rotating_proxies.middlewares.BanDetectionMiddleware': 620,
}
}
proxy_settings = {
'ROTATING_PROXY_LIST': constants.PRIVOXY_LOCAL_PROXY,
}
redis_settings = {
'REDIS_HOST': constants.REDIS_HOST,
'REDIS_PORT': constants.REDIS_PORT