def __init__(self, host, port, api_key, per_token_hours_limit,
buffer_size):
r = _default_redis(host, port)
self.req_count = _default_req_count(r, api_key=api_key)
self.tk_alive = _default_tk_alive(r, api_key=api_key)
self.per_token_hours_limit = per_token_hours_limit
self.buffer_size = buffer_size
def __init__(self, host, port, api_key, api_secret, per_token_hours_limit, per_ip_hours_limit, buffer_size, ip_address):
r = _default_redis(host, port)
self.req_count = _v1_req_count(r, api_key=api_key, api_secret=api_secret)
self.req_ip_count = _ip_req_count(r)
self.localIp = ip_address
self.tk_alive = _v1_tk_alive(r, api_key=api_key, api_secret=api_secret)
self.per_token_hours_limit = per_token_hours_limit
self.per_ip_hours_limit = per_ip_hours_limit
self.buffer_size = buffer_size
self.api_key = api_key
self.api_secret = api_secret
REDIS_HOST = 'localhost'
REDIS_PORT = 6379
#prod
"""
REDIS_HOST = '219.224.135.60'
REDIS_PORT = 6379
"""
QUEUE_KEY = '%(spider)s:requests'
DUPEFILTER_KEY = '%(spider)s:dupefilter'
spider_name = sys.argv[1]
not_del = False
if len(sys.argv) > 2 and sys.argv[2] == 'st':
not_del = True
r = _default_redis(REDIS_HOST, REDIS_PORT)
try:
print 'scheduled requests: %s' % r.llen(QUEUE_KEY % {'spider': spider_name})
except redis.exceptions.ResponseError:
print 'scheduled requests: %s' % r.zcard(QUEUE_KEY % {'spider': spider_name})
print 'dupefiler requests: %s' % r.scard(DUPEFILTER_KEY % {'spider': spider_name})
if not not_del:
r.delete(QUEUE_KEY % {'spider': spider_name})
r.delete(DUPEFILTER_KEY % {'spider': spider_name})
print 'clear %s' % spider_name
def __init__(self, host, port, api_key):
r = _default_redis(host, port)
self.req_count = _default_req_count(r, api_key=api_key)
self.tk_alive = _default_tk_alive(r, api_key=api_key)
def __init__(self, host, port, api_key, per_token_hours_limit, buffer_size):
r = _default_redis(host, port)
self.req_count = _default_req_count(r, api_key=api_key)
self.tk_alive = _default_tk_alive(r, api_key=api_key)
self.per_token_hours_limit = per_token_hours_limit
self.buffer_size = buffer_size
def __init__(self, host, port, api_key):
r = _default_redis(host, port)
self.req_count = _default_req_count(r, api_key=api_key)
self.tk_alive = _default_tk_alive(r, api_key=api_key)
# -*- coding: utf-8 -*- from tk_maintain import reset_ip_req_count, _ip_req_count, _default_redis r = _default_redis() req_ip_count = _ip_req_count(r) reset_ip_req_count(req_ip_count) # windows cron jobs # schtasks /create /sc hourly /st 00:00:00 /tn PythonIpResetCountTask /TR "C:\python27\python.exe E:\scrapy_weibo_v1\install\utils4scrapy-master\utils4scrapy\auto_reset_ip_count.py" # schtasks /delete /tn PythonIpResetCountTask # linux cron jobs # cd /etc/crontab; # vim /etc/crontab; # 0 * * * * root cd /home/mirage/linhao/scrapy_weibo/install/utils4scrapy-master/utils4scrapy;python auto_reset_ip_req_count.py
#prod
"""
REDIS_HOST = '219.224.135.60'
REDIS_PORT = 6379
"""
QUEUE_KEY = '%(spider)s:requests'
DUPEFILTER_KEY = '%(spider)s:dupefilter'
spider_name = sys.argv[1]
not_del = False
if len(sys.argv) > 2 and sys.argv[2] == 'st':
not_del = True
r = _default_redis(REDIS_HOST, REDIS_PORT)
try:
print 'scheduled requests: %s' % r.llen(
QUEUE_KEY % {'spider': spider_name})
except redis.exceptions.ResponseError:
print 'scheduled requests: %s' % r.zcard(
QUEUE_KEY % {'spider': spider_name})
print 'dupefiler requests: %s' % r.scard(
DUPEFILTER_KEY % {'spider': spider_name})
if not not_del:
r.delete(QUEUE_KEY % {'spider': spider_name})
r.delete(DUPEFILTER_KEY % {'spider': spider_name})
print 'clear %s' % spider_name
def __init__(self, host, port, api_key):
r = _default_redis(host, port)
self.req_count = _default_req_count(r, api_key=api_key)
self.tk_alive = _default_tk_alive(r, api_key=api_key)
self.users_not_exist = _default_not_exist_users(r, 'user_info_spider')