Python get_cookie_expire_time示例

示例#1

# coding:utf-8
import json
import socket
import datetime

import redis
from logger.log import crawler
from config.conf import (get_redis_args, get_share_host_count,
                         get_running_mode, get_cookie_expire_time)

mode = get_running_mode()
redis_args = get_redis_args()
share_host_count = get_share_host_count()
cookie_expire_time = get_cookie_expire_time()


# todo consider the concurrency when fetching or storing cookies
class Cookies(object):
    rd_con = redis.StrictRedis(host=redis_args.get('host'),
                               port=redis_args.get('port'),
                               password=redis_args.get('password'),
                               db=redis_args.get('cookies'))

    rd_con_broker = redis.StrictRedis(host=redis_args.get('host'),
                                      port=redis_args.get('port'),
                                      password=redis_args.get('password'),
                                      db=redis_args.get('broker'))

    @classmethod
    def store_cookies(cls, name, cookies):
        pickled_cookies = json.dumps({

示例#2

文件： redis_db.py 项目： dittoyi/weibospider

    get_share_host_count,
    get_running_mode,
    get_cookie_expire_time
)

mode = get_running_mode()
share_host_count = get_share_host_count()

redis_args = get_redis_args()
password = redis_args.get('password', '')
cookies_db = redis_args.get('cookies', 1)
urls_db = redis_args.get('urls', 2)
broker_db = redis_args.get('broker', 5)
backend_db = redis_args.get('backend', 6)
id_name_db = redis_args.get('id_name', 8)
cookie_expire_time = get_cookie_expire_time()
data_expire_time = redis_args.get('expire_time') * 60 * 60

sentinel_args = redis_args.get('sentinel', '')
if sentinel_args:
    # default socket timeout is 2 secs
    master_name = redis_args.get('master')
    socket_timeout = int(redis_args.get('socket_timeout', 2))
    sentinel = Sentinel([(args['host'], args['port']) for args in sentinel_args], password=password,
                        socket_timeout=socket_timeout)
    cookies_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=cookies_db)
    broker_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=broker_db)
    urls_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=urls_db)
    id_name_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=id_name_db)
else:
    host = redis_args.get('host', '127.0.0.1')