Exemplos de get_running_mode em Python

Exemplo n.º 1

# coding:utf-8
import json
import socket
import datetime

import redis
from logger.log import crawler
from config.conf import (get_redis_args, get_share_host_count,
                         get_running_mode, get_cookie_expire_time)

mode = get_running_mode()
redis_args = get_redis_args()
share_host_count = get_share_host_count()
cookie_expire_time = get_cookie_expire_time()


# todo consider the concurrency when fetching or storing cookies
class Cookies(object):
    rd_con = redis.StrictRedis(host=redis_args.get('host'),
                               port=redis_args.get('port'),
                               password=redis_args.get('password'),
                               db=redis_args.get('cookies'))

    rd_con_broker = redis.StrictRedis(host=redis_args.get('host'),
                                      port=redis_args.get('port'),
                                      password=redis_args.get('password'),
                                      db=redis_args.get('broker'))

    @classmethod
    def store_cookies(cls, name, cookies):
        pickled_cookies = json.dumps({

Exemplo n.º 2

Arquivo: redis_db.py Projeto: dittoyi/weibospider

# coding:utf-8
import json
import socket
import datetime

import redis
from redis.sentinel import Sentinel
from logger.log import crawler
from config.conf import (
    get_redis_args,
    get_share_host_count,
    get_running_mode,
    get_cookie_expire_time
)

mode = get_running_mode()
share_host_count = get_share_host_count()

redis_args = get_redis_args()
password = redis_args.get('password', '')
cookies_db = redis_args.get('cookies', 1)
urls_db = redis_args.get('urls', 2)
broker_db = redis_args.get('broker', 5)
backend_db = redis_args.get('backend', 6)
id_name_db = redis_args.get('id_name', 8)
cookie_expire_time = get_cookie_expire_time()
data_expire_time = redis_args.get('expire_time') * 60 * 60

sentinel_args = redis_args.get('sentinel', '')
if sentinel_args:
    # default socket timeout is 2 secs