from logger import crawler from config import (get_redis_args, get_share_host_count, get_running_mode, get_cookie_expire_time) MODE = get_running_mode() SHARE_HOST_COUNT = get_share_host_count() REDIS_ARGS = get_redis_args() password = REDIS_ARGS.get('password', '') cookies_db = REDIS_ARGS.get('cookies', 1) urls_db = REDIS_ARGS.get('urls', 2) broker_db = REDIS_ARGS.get('broker', 5) backend_db = REDIS_ARGS.get('backend', 6) id_name_db = REDIS_ARGS.get('id_name', 8) cookie_expire_time = get_cookie_expire_time() data_expire_time = REDIS_ARGS.get('expire_time') * 60 * 60 sentinel_args = REDIS_ARGS.get('sentinel', '') if sentinel_args: # default socket timeout is 2 secs master_name = REDIS_ARGS.get('master') socket_timeout = int(REDIS_ARGS.get('socket_timeout', 2)) sentinel = Sentinel([(args['host'], args['port']) for args in sentinel_args], password=password, socket_timeout=socket_timeout) cookies_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=cookies_db) broker_con = sentinel.master_for(master_name,
get_running_mode, get_cookie_expire_time ) MODE = get_running_mode() SHARE_HOST_COUNT = get_share_host_count() REDIS_ARGS = get_redis_args() password = REDIS_ARGS.get('password', '') cookies_db = REDIS_ARGS.get('cookies', 1) urls_db = REDIS_ARGS.get('urls', 2) broker_db = REDIS_ARGS.get('broker', 5) backend_db = REDIS_ARGS.get('backend', 6) id_name_db = REDIS_ARGS.get('id_name', 8) cookie_expire_time = get_cookie_expire_time() data_expire_time = REDIS_ARGS.get('expire_time') * 60 * 60 sentinel_args = REDIS_ARGS.get('sentinel', '') if sentinel_args: # default socket timeout is 2 secs master_name = REDIS_ARGS.get('master') socket_timeout = int(REDIS_ARGS.get('socket_timeout', 2)) sentinel = Sentinel([(args['host'], args['port']) for args in sentinel_args], password=password, socket_timeout=socket_timeout) cookies_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=cookies_db) broker_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=broker_db) urls_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=urls_db) id_name_con = sentinel.master_for(master_name, socket_timeout=socket_timeout, db=id_name_db) else: host = REDIS_ARGS.get('host', '127.0.0.1')