Exemple #1
0
import datetime

import redis
from redis.sentinel import Sentinel
from logger.log import crawler
from config.conf import (
    get_redis_args,
    get_share_host_count,
    get_running_mode,
    get_cookie_expire_time
)

mode = get_running_mode()
share_host_count = get_share_host_count()

redis_args = get_redis_args()
password = redis_args.get('password', '')
cookies_db = redis_args.get('cookies', 1)
urls_db = redis_args.get('urls', 2)
broker_db = redis_args.get('broker', 5)
backend_db = redis_args.get('backend', 6)
id_name_db = redis_args.get('id_name', 8)
cookie_expire_time = get_cookie_expire_time()
data_expire_time = redis_args.get('expire_time') * 60 * 60

sentinel_args = redis_args.get('sentinel', '')
if sentinel_args:
    # default socket timeout is 2 secs
    master_name = redis_args.get('master')
    socket_timeout = int(redis_args.get('socket_timeout', 2))
    sentinel = Sentinel([(args['host'], args['port']) for args in sentinel_args], password=password,
Exemple #2
0
# coding:utf-8
import json
import socket
import datetime

import redis
from logger.log import crawler
from config.conf import (get_redis_args, get_share_host_count,
                         get_running_mode, get_cookie_expire_time)

mode = get_running_mode()
redis_args = get_redis_args()
share_host_count = get_share_host_count()
cookie_expire_time = get_cookie_expire_time()


# todo consider the concurrency when fetching or storing cookies
class Cookies(object):
    rd_con = redis.StrictRedis(host=redis_args.get('host'),
                               port=redis_args.get('port'),
                               password=redis_args.get('password'),
                               db=redis_args.get('cookies'))

    rd_con_broker = redis.StrictRedis(host=redis_args.get('host'),
                                      port=redis_args.get('port'),
                                      password=redis_args.get('password'),
                                      db=redis_args.get('broker'))

    @classmethod
    def store_cookies(cls, name, cookies):
        pickled_cookies = json.dumps({
import json
import redis
from config.conf import get_redis_args
REDIS_ARGS = get_redis_args()
urls_db = REDIS_ARGS.get('urls', 2)
host = REDIS_ARGS.get('host', '127.0.0.1')
port = REDIS_ARGS.get('port', 6379)
urls_con = redis.Redis(host=host, port=port,db=urls_db)


class Url_Parameter(object):
    @classmethod
    def store_parameter(cls, spider_name, parameter):
        urls_con.rpush(spider_name, json.dumps(parameter))


    @classmethod
    def fetch_parameters(cls,spider_name):
        parameters = urls_con.lrange(spider_name,1,20)
        urls_con.ltrim(spider_name,20,-1)
        return parameters