Python _default_redis Examples

Example #1

 def __init__(self, host, port, api_key, per_token_hours_limit,
              buffer_size):
     r = _default_redis(host, port)
     self.req_count = _default_req_count(r, api_key=api_key)
     self.tk_alive = _default_tk_alive(r, api_key=api_key)
     self.per_token_hours_limit = per_token_hours_limit
     self.buffer_size = buffer_size

Example #2

File: middlewares.py Project: lijiahong/utils4scrapy

 def __init__(self, host, port, api_key, api_secret, per_token_hours_limit, per_ip_hours_limit, buffer_size, ip_address):
     r = _default_redis(host, port)
     self.req_count = _v1_req_count(r, api_key=api_key, api_secret=api_secret)
     self.req_ip_count = _ip_req_count(r)
     self.localIp = ip_address
     self.tk_alive = _v1_tk_alive(r, api_key=api_key, api_secret=api_secret)
     self.per_token_hours_limit = per_token_hours_limit
     self.per_ip_hours_limit = per_ip_hours_limit
     self.buffer_size = buffer_size
     self.api_key = api_key
     self.api_secret = api_secret

Example #3

File: clear_spider.py Project: MOON-CLJ/utils4scrapy

REDIS_HOST = 'localhost'
REDIS_PORT = 6379

#prod
"""
REDIS_HOST = '219.224.135.60'
REDIS_PORT = 6379
"""

QUEUE_KEY = '%(spider)s:requests'
DUPEFILTER_KEY = '%(spider)s:dupefilter'

spider_name = sys.argv[1]
not_del = False
if len(sys.argv) > 2 and sys.argv[2] == 'st':
    not_del = True

r = _default_redis(REDIS_HOST, REDIS_PORT)

try:
    print 'scheduled requests: %s' % r.llen(QUEUE_KEY % {'spider': spider_name})
except redis.exceptions.ResponseError:
    print 'scheduled requests: %s' % r.zcard(QUEUE_KEY % {'spider': spider_name})

print 'dupefiler requests: %s' % r.scard(DUPEFILTER_KEY % {'spider': spider_name})

if not not_del:
    r.delete(QUEUE_KEY % {'spider': spider_name})
    r.delete(DUPEFILTER_KEY % {'spider': spider_name})
    print 'clear %s' % spider_name

Example #4

 def __init__(self, host, port, api_key):
     r = _default_redis(host, port)
     self.req_count = _default_req_count(r, api_key=api_key)
     self.tk_alive = _default_tk_alive(r, api_key=api_key)

Example #5

File: middlewares.py Project: MOON-CLJ/utils4scrapy

 def __init__(self, host, port, api_key, per_token_hours_limit, buffer_size):
     r = _default_redis(host, port)
     self.req_count = _default_req_count(r, api_key=api_key)
     self.tk_alive = _default_tk_alive(r, api_key=api_key)
     self.per_token_hours_limit = per_token_hours_limit
     self.buffer_size = buffer_size

Example #6

File: middlewares.py Project: MOON-CLJ/utils4scrapy

 def __init__(self, host, port, api_key):
     r = _default_redis(host, port)
     self.req_count = _default_req_count(r, api_key=api_key)
     self.tk_alive = _default_tk_alive(r, api_key=api_key)

Example #7

File: auto_reset_ip_req_count.py Project: lijiahong/utils4scrapy

# -*- coding: utf-8 -*-

from tk_maintain import reset_ip_req_count, _ip_req_count, _default_redis

r = _default_redis()
req_ip_count = _ip_req_count(r)
reset_ip_req_count(req_ip_count)


# windows cron jobs
# schtasks /create /sc hourly /st 00:00:00 /tn PythonIpResetCountTask /TR "C:\python27\python.exe E:\scrapy_weibo_v1\install\utils4scrapy-master\utils4scrapy\auto_reset_ip_count.py"
# schtasks /delete /tn PythonIpResetCountTask

# linux cron jobs
# cd /etc/crontab;
# vim /etc/crontab;
# 0 * * * * root cd /home/mirage/linhao/scrapy_weibo/install/utils4scrapy-master/utils4scrapy;python auto_reset_ip_req_count.py

Example #8

File: clear_spider.py Project: MOON-CLJ/utils4scrapy

#prod
"""
REDIS_HOST = '219.224.135.60'
REDIS_PORT = 6379
"""

QUEUE_KEY = '%(spider)s:requests'
DUPEFILTER_KEY = '%(spider)s:dupefilter'

spider_name = sys.argv[1]
not_del = False
if len(sys.argv) > 2 and sys.argv[2] == 'st':
    not_del = True

r = _default_redis(REDIS_HOST, REDIS_PORT)

try:
    print 'scheduled requests: %s' % r.llen(
        QUEUE_KEY % {'spider': spider_name})
except redis.exceptions.ResponseError:
    print 'scheduled requests: %s' % r.zcard(
        QUEUE_KEY % {'spider': spider_name})

print 'dupefiler requests: %s' % r.scard(
    DUPEFILTER_KEY % {'spider': spider_name})

if not not_del:
    r.delete(QUEUE_KEY % {'spider': spider_name})
    r.delete(DUPEFILTER_KEY % {'spider': spider_name})
    print 'clear %s' % spider_name

Example #9

File: middlewares.py Project: lijiahong/utils4scrapy

 def __init__(self, host, port, api_key):
     r = _default_redis(host, port)
     self.req_count = _default_req_count(r, api_key=api_key)
     self.tk_alive = _default_tk_alive(r, api_key=api_key)
     self.users_not_exist = _default_not_exist_users(r, 'user_info_spider')