import datetime
import redis
from redis.sentinel import Sentinel
from logger.log import crawler
from config.conf import (
get_redis_args,
get_share_host_count,
get_running_mode,
get_cookie_expire_time
)
mode = get_running_mode()
share_host_count = get_share_host_count()
redis_args = get_redis_args()
password = redis_args.get('password', '')
cookies_db = redis_args.get('cookies', 1)
urls_db = redis_args.get('urls', 2)
broker_db = redis_args.get('broker', 5)
backend_db = redis_args.get('backend', 6)
id_name_db = redis_args.get('id_name', 8)
cookie_expire_time = get_cookie_expire_time()
data_expire_time = redis_args.get('expire_time') * 60 * 60
sentinel_args = redis_args.get('sentinel', '')
if sentinel_args:
# default socket timeout is 2 secs
master_name = redis_args.get('master')
socket_timeout = int(redis_args.get('socket_timeout', 2))
sentinel = Sentinel([(args['host'], args['port']) for args in sentinel_args], password=password,
# coding:utf-8
import json
import socket
import datetime
import redis
from logger.log import crawler
from config.conf import (get_redis_args, get_share_host_count,
get_running_mode, get_cookie_expire_time)
mode = get_running_mode()
redis_args = get_redis_args()
share_host_count = get_share_host_count()
cookie_expire_time = get_cookie_expire_time()
# todo consider the concurrency when fetching or storing cookies
class Cookies(object):
rd_con = redis.StrictRedis(host=redis_args.get('host'),
port=redis_args.get('port'),
password=redis_args.get('password'),
db=redis_args.get('cookies'))
rd_con_broker = redis.StrictRedis(host=redis_args.get('host'),
port=redis_args.get('port'),
password=redis_args.get('password'),
db=redis_args.get('broker'))
@classmethod
def store_cookies(cls, name, cookies):
pickled_cookies = json.dumps({
import json
import redis
from config.conf import get_redis_args
REDIS_ARGS = get_redis_args()
urls_db = REDIS_ARGS.get('urls', 2)
host = REDIS_ARGS.get('host', '127.0.0.1')
port = REDIS_ARGS.get('port', 6379)
urls_con = redis.Redis(host=host, port=port,db=urls_db)
class Url_Parameter(object):
@classmethod
def store_parameter(cls, spider_name, parameter):
urls_con.rpush(spider_name, json.dumps(parameter))
@classmethod
def fetch_parameters(cls,spider_name):
parameters = urls_con.lrange(spider_name,1,20)
urls_con.ltrim(spider_name,20,-1)
return parameters