class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
判断是否达到了代理池限制
"""
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
print('获取器开始执行')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
# 获取代理
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
print('将获取到的代理添加到redis中去', proxy)
self.redis.add(proxy)
def get_stock_names():
""" Return list of stock keys """
r = RedisClient().conn
res = [key.decode('utf-8') for key in r.keys()]
print(res)
if len(res) == 0:
raise DataNotFound("Stock Data not available")
else:
return jsonify(res)
def return_all():
""" fetch all data to convert to csv format """
d = '}'
ticker_name = request.args.get('ticker_name')
r = RedisClient().conn
fetch_dat = r.get(ticker_name).decode('utf-8')
fetch_dat = [e + d for e in fetch_dat.split(d) if e]
obj = {}
obj['results'] = fetch_dat
if fetch_dat == None:
raise DataNotFound("Stock Data not available")
else:
return jsonify(obj)
def get_stock_data():
""" Fetch Stock data by date range provided """
d = '}'
ticker_name = request.args.get('ticker_name')
start = int(request.args.get('start'))
limit = 20
r = RedisClient().conn
fetch_dat = r.get(ticker_name).decode('utf-8')
fetch_dat = [e + d for e in fetch_dat.split(d) if e]
if fetch_dat == None:
raise DataNotFound("Stock Data not available")
else:
return jsonify(
paginated_results(fetch_dat, '/api/stockData', start, limit))
class Tester(object):
def __init__(self):
self.redis = RedisClient()
# 测试每一个协议
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:
:return:
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试', proxy)
async with session.get(TEST_URL,
proxy=real_proxy,
timeout=15,
allow_redirects=False) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print('代理可用', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法 ', response.status, 'IP', proxy)
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
asyncio.TimeoutError, AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
# 拿到所有的协议来一个一个测试
def run(self):
"""
测试主函数
:return:
"""
print('测试器开始运行')
try:
count = self.redis.count()
print('当前剩余', count, '个代理')
for i in range(0, count, BATCH_TEST_SIZE):
start = i
stop = min(i + BATCH_TEST_SIZE, count)
print('正在测试第', start + 1, '-', stop, '个代理')
# 拿到将要测试的所有协议
test_proxies = self.redis.batch(start, stop)
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
sys.stdout.flush()
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
def __init__(self, cluster_config_file, local_config_file,
local_slave_config_file):
# connect to the redis cluster as configured
self.redis_cluster = RedisClient(Config(file_path=cluster_config_file),
kind="JAVA").handle()
# connect to local redis client
try:
self.redis_local = RedisClient(
Config(file_path=local_config_file)).handle()
except:
print(
"Run 'sudo docker run --name redis-local -p 6380:6379 -d redis' to create redis-local"
)
self.redis_local = None
raise
# connect to redis client with slave as read-only
try:
self.redis_slave = RedisClient(
Config(file_path=local_slave_config_file)).handle()
except:
print(
"Run 'sudo docker run --name redis-slave -p 6381:6379 -d redis' to create redis-slave"
)
self.redis_slave = None
# raise
# migrate to redis_local if it is empty
if self.redis_local and len(self.redis_local.keys()) == 0:
self.migrate_data(self.redis_cluster, self.redis_local)
# migrate to redis_slave as well?
# if self.redis_slave and len(self.redis_slave.keys()) == 0:
# self.migrate_data(self.redis_local, self.redis_slave)
# # other slave settings
print("cluster=%s\nlocal=%s\nlocal_slave=%s" %
(sum(self.redis_cluster.dbsize().values()),
self.redis_local.dbsize(), self.redis_slave.dbsize()))
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def __init__(self):
self.redis = RedisClient()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from __future__ import unicode_literals
from scplib import *
from iplugin import Plugin
from icommand import *
from state import *
from redisclient import RedisClient
_redis = RedisClient()
class CommandRedisConnect(Command):
def add_args(self):
self.get_parser().add_argument('-i', '--ip', dest = 'ip', metavar = 'ip', default = "127.0.0.1", help='the server ip')
self.get_parser().add_argument('-p', '--port', metavar = 'port', action = "store", default = '6379', help='the server port')
def perform(self, argv):
ns = self.get_parser().parse_args(argv[1:])
_redis.connect(ns.ip, int(ns.port))
class CommandSend(Command):
def add_args(self):
self.get_parser().add_argument('cmd', nargs = '+')
def perform(self, argv):
ns = self.get_parser().parse_args(argv)
rsp = _redis.transmit(ns.cmd)
class RedisPerf(object):
def __init__(self, cluster_config_file, local_config_file,
local_slave_config_file):
# connect to the redis cluster as configured
self.redis_cluster = RedisClient(Config(file_path=cluster_config_file),
kind="JAVA").handle()
# connect to local redis client
try:
self.redis_local = RedisClient(
Config(file_path=local_config_file)).handle()
except:
print(
"Run 'sudo docker run --name redis-local -p 6380:6379 -d redis' to create redis-local"
)
self.redis_local = None
raise
# connect to redis client with slave as read-only
try:
self.redis_slave = RedisClient(
Config(file_path=local_slave_config_file)).handle()
except:
print(
"Run 'sudo docker run --name redis-slave -p 6381:6379 -d redis' to create redis-slave"
)
self.redis_slave = None
# raise
# migrate to redis_local if it is empty
if self.redis_local and len(self.redis_local.keys()) == 0:
self.migrate_data(self.redis_cluster, self.redis_local)
# migrate to redis_slave as well?
# if self.redis_slave and len(self.redis_slave.keys()) == 0:
# self.migrate_data(self.redis_local, self.redis_slave)
# # other slave settings
print("cluster=%s\nlocal=%s\nlocal_slave=%s" %
(sum(self.redis_cluster.dbsize().values()),
self.redis_local.dbsize(), self.redis_slave.dbsize()))
def smart_get(self, redis, key, key_type):
if key_type == "list":
return redis.lrange(key, 0, -1)
elif key_type == "hash":
return redis.hgetall(key)
elif key_type == "set":
return list(redis.sscan_iter(key))
elif key_type == "zset":
values = redis.zrange(key, 0, -1, withscores=True)
values_list = []
for k, v in values:
values_list.append(v)
values_list.append(k)
return values_list
elif key_type == "string":
return redis.get(key)
else:
raise Exception("unhandled type: %s" % key_type)
def smart_set(self, dst_redis, key, key_type, values):
if key_type == "list":
dst_redis.lpush(key, values)
elif key_type == "hash":
dst_redis.hmset(key, values)
elif key_type == "set":
dst_redis.sadd(key, values)
elif key_type == "zset":
dst_redis.zadd(key, *values)
elif key_type == "string":
dst_redis.set(key, values)
else:
raise Exception("unhandled type: %s" % key_type)
def migrate_data(self, src_redis, dst_redis):
count = 0
for key in src_redis.keys():
key_type = src_redis.type(key)
values = self.smart_get(src_redis, key_type)
self.smart_set(dst_redis, key_type, values)
def perf_read(self, string_list, step=100):
"""
- perf the redis read part
- cluster with single get
- local redis with single get
- redis master-slave mode, slave as READ-ONLY
:param string_list: the list of strings
:param step: the number of string per step should increment, if len(string_list) = 1000, and step = 100, then
the experiment will use number of strings from [100, 200, 300, ..., 1000] for perf. step can also be a list
of predefined numbers, and can be useful for log-based step.
:return:
"""
len_string_list = len(string_list)
if isinstance(step, int):
steps = range(0, len_string_list, step)
if steps[-1] != len_string_list:
steps.append(len_string_list)
elif isinstance(step, list):
steps = [int(i) for i in step]
else:
raise Exception("Unexpected type for step: %s has type %s" %
(step, type(step)))
# maps each perf type, to {size: time_elapsed_list}
perf_numbers = {
"redis_cluster_sget": {},
"redis_local_sget": {},
"redis_slave_sget": {}
}
for size in steps:
if size == 0:
# This doesn't make sense
for key, v in perf_numbers.items():
v[size] = []
continue
shuffle_num = len_string_list / size
tmp_string_list = list(string_list)
perf_numbers['redis_cluster_sget'].setdefault(size, [])
perf_numbers['redis_local_sget'].setdefault(size, [])
perf_numbers['redis_slave_sget'].setdefault(size, [])
for index in range(shuffle_num):
shuffle(tmp_string_list)
use_tmp_string_list = tmp_string_list[:size]
if self.redis_cluster:
# run cluster single get
t1 = time()
for s in use_tmp_string_list:
self.smart_get(self.redis_cluster, s,
self.redis_cluster.type(s))
t2 = time()
perf_numbers['redis_cluster_sget'][size].append(t2 - t1)
if self.redis_local:
# run local single get
t1 = time()
for s in use_tmp_string_list:
self.smart_get(self.redis_local, s,
self.redis_cluster.type(s))
t2 = time()
perf_numbers['redis_local_sget'][size].append(t2 - t1)
if self.redis_slave:
# run local slave single get
pass
# run local slave multiple get
pass
for key, value in perf_numbers.items():
print(key, {size: mean(t) for size, t in value.items() if t})
def perf_write(self):
"""
- perf the redis write part
- only do local write?
- pipeline all the requests
- worker nodes do the computation, the master node processes strings, and inserts them into the database
"""
pass
def perf_pipe(self, n=1000):
src_redis = self.redis_cluster
keys = src_redis.keys()[:n]
nonpipe_type_set = set()
t1 = time()
for key in keys:
key_type = src_redis.type(key)
nonpipe_type_set.add(key_type)
t2 = time()
pipe = src_redis.pipeline()
for key in keys:
pipe.type(key)
values = pipe.execute()
pipe_type_set = set(values)
t3 = time()
print("string count %d" % len(keys))
print("non-pipeline took: %f, with result: %s" %
(t2 - t1, nonpipe_type_set))
print("pipeline took: %f, with result: %s" %
(t3 - t2, nonpipe_type_set))
def get_conn():
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis