def stateMachine(config): numclients = 1 maxVal = 1000 cli = RedisClient() print 'Going to start failure scheduler' t = threading.Thread(target=failureScheduler, args=(10, config,)) t.start() spawnClients(numclients, maxVal, cli) (successes, fails) = collateLogFiles(numclients, maxVal) log = set(cli.getLog()) (ackedAndLost, ackedAndPresent, notAckedAndPresent, notAckedAndAbsent) = printStats (log, successes, fails) print 'STATISTICS' print 'ackedAndLost %s' %(len(ackedAndLost)) print 'ackedAndPresent %s'%(len(ackedAndPresent)) print 'notAckedAndPresent %s'%(len(notAckedAndPresent)) print 'notAckedAndAbsent %s' %(len(notAckedAndAbsent)) print ackedAndLost print sorted(successes) print sorted(log) t.join()
def extract_article(self):
r = RedisClient()
articles = r.get(self._id + ":articles")
selected = r.get(self._id + ":selected")
article = None
entities = self.outcome["entities"]
if "ordinal" in entities:
value = entities["ordinal"][0]["value"]
if value < len(articles):
article = articles[value - 1]
if not article and "topic" in entities:
topic = entities["topic"][0]["value"]
is_current_context = False
for word in ["it", "this", "article"]:
if word in topic:
article = selected
is_current_context = True
break
if not is_current_context:
# fuzzy string matching
result = process.extractOne(topic, map(lambda x: x["headline"], articles))
if (result[1] > 75):
for i, article_candidate in enumerate(articles):
if (article_candidate["headline"] == result[0]):
article = article_candidate
break
if not article and selected and selected != "None":
article = selected
return article
def get_media(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response(
"Sorry, what kind of news would you like to hear about?")
article = self.extract_article()
if article:
if article['multimedia']:
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
have_media = Media(self.name)
self.payload = {
"read": have_media.get_phrase() # usable url for html
}
self.pushlater = {
"type": "media",
"value": article['multimedia']
}
self.finish_response()
else:
self.error_response(
"Sorry I don't have images for that article")
else:
self.error_response(
"Sorry I don't know what article you are talking about")
class CmdLineApp(Cmd):
def do_list(self, arg, opts=None):
data = config.list_conns()
header = ["ID", "Name", "Host", "Port", "DB", "comments"]
# tabulate.WIDE_CHARS_MODE = True
# print_table(header,data)
print tabulate(data, headers=header)
@options(
[
make_option("-p", "--piglatin", action="store_true", help="atinLay"),
make_option("-s", "--shout", action="store_true", help="N00B EMULATION MODE"),
make_option("-r", "--repeat", type="int", help="output [n] times"),
]
)
def do_conn(self, arg, opts=None):
(host, port, db) = config.get_conn(int(arg))
self.prompt = "%s:%s > " % (host, str(db))
self.conn = RedisClient(host, port, db)
# @options([make_option('-p', '--piglatin', action="store_true", help="atinLay"),
# make_option('-s', '--shout', action="store_true", help="N00B EMULATION MODE"),
# make_option('-r', '--repeat', type="int", help="output [n] times")
# ])
def do_get(self, arg, opts=None):
data = self.conn.get(arg)
txt = highlight(unicode(data, "UTF-8"), JsonLexer(), pygments.formatters.TerminalFormatter(bg="dark"))
print txt.rstrip("\r\n").decode("raw_unicode_escape")
def do_keys(self, arg, opts=None):
print self.conn.keys(arg)
def __init__(self):
self.dbClient = MongoDbClient()
self.redisClient = RedisClient()
with open('/Users/vinusebastian/uci/quarter2/IR/project/WEBPAGES_RAW/bookkeeping.json') as json_data:
self.bookKeeper = json.load(json_data)
self.stopWords = []
self.documentFrequencyThreshold = 1000
def extract_article(self):
r = RedisClient()
articles = r.get(self._id + ":articles")
selected = r.get(self._id + ":selected")
article = None
entities = self.outcome["entities"]
if "ordinal" in entities:
value = entities["ordinal"][0]["value"]
if value < len(articles):
article = articles[value - 1]
if not article and "topic" in entities:
topic = entities["topic"][0]["value"]
is_current_context = False
for word in ["it", "this", "article"]:
if word in topic:
article = selected
is_current_context = True
break
if not is_current_context:
# fuzzy string matching
result = process.extractOne(
topic, map(lambda x: x["headline"], articles))
if (result[1] > 75):
for i, article_candidate in enumerate(articles):
if (article_candidate["headline"] == result[0]):
article = article_candidate
break
if not article and selected and selected != "None":
article = selected
return article
def confirm_action(self):
r = RedisClient()
current_state = r.get(self._id + ":state")
if (current_state != "start"):
self.error_response("Sorry, what are you saying okay for?")
else:
articles = r.get(self._id + ":articles")
self.respond_get_headlines(articles)
def start(self):
r = RedisClient()
# Clear the state for this new user
r.set(self._id + ":state", None)
r.set(self._id + ":articles", None)
r.set(self._id + ":selected", None)
NYTimes.get_headlines(self.respond_start)
def confirm_action(self):
r = RedisClient()
current_state = r.get(self._id + ":state")
if (current_state != "start"):
self.error_response("Sorry, what are you saying okay for?")
else:
articles = r.get(self._id + ":articles")
self.respond_get_headlines(articles)
def __init__(self, lowerBound, upperBound):
self.bigDictionary = {}
self.folders = glob.glob("/Users/vinusebastian/uci/quarter2/IR/project/WEBPAGES_RAW/*")
self.htmlParser = htmlParser()
self.lowerBound = lowerBound
self.upperBound = upperBound
self.dbClient = MongoDbClient()
self.redisClient = RedisClient()
self.documentLength = {}
self.textDictionary = {}
def post(self):
payload = {}
data = tornado.escape.json_decode(self.request.body)
user_id = data["id"]
r = RedisClient()
name = r.get(user_id + ":name")
if not name:
payload = {"status": 500}
else:
payload = {"id": user_id, "name": name, "status": 200}
self.write(tornado.escape.json_encode(payload))
def post(self):
payload = {}
data = tornado.escape.json_decode(self.request.body)
user_id = data["id"]
if ("name" not in data):
payload = {"status": 500}
else:
user_name = data["name"]
r = RedisClient()
r.set(user_id + ":name", user_name)
payload = {"id": user_id, "name": user_name, "status": 200}
self.write(tornado.escape.json_encode(payload))
def __init__(self,
host="localhost",
port='6379',
key='Proxy',
moni_port=9999):
self.__redis = RedisClient(host=host,
port=port,
password=None,
key=key)
self.__port = moni_port
self.__host = host
self.__db_port = port
self.__key = key
def respond_get_headlines(self, payload):
if not payload:
self.error_response("API error")
sentence_headlines, topic_headlines, article_order = NLPParser.parse_headlines(
map(lambda x: x["headline"], payload))
article_values = []
for article in payload:
article_values.append({
"headline": article["headline"],
"url": article["url"],
"snippet": article["snippet"]
})
self.pushlater = {"type": "article", "value": article_values}
found = FoundHeadlines(sentence_headlines, topic_headlines, self.name)
self.payload = {"read": found.get_phrase()}
r = RedisClient()
ordered_articles = map(lambda x: payload[x], article_order)
# State transition
r.set(self._id + ":selected", None)
r.set(self._id + ":articles", ordered_articles)
r.set(self._id + ":state", "headlines")
self.finish_response()
def __init__(self,
validation_addr="http://www.baidu.com",
host="www.baidu.com",
batch_size=100,
db_host='localhost',
port='6379',
key='Proxy'):
self.__redis = RedisClient(host=db_host,
port=port,
password=None,
key=key)
self.__host = host
self.__addr = validation_addr
self.__batch_size = batch_size
def post(self):
payload = {}
data = tornado.escape.json_decode(self.request.body)
user_id = data["id"]
r = RedisClient()
name = r.get(user_id + ":name")
if not name:
payload = {"status": 500}
else:
payload = {
"id": user_id,
"name": name,
"status": 200
}
self.write(tornado.escape.json_encode(payload))
class WeiboCookiesGenerator(object):
def __init__(self, website='weibo'):
"""
初始化一些对象
:param website: 名称
:param browser: 浏览器, 若不使用浏览器则可设置为 None
"""
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.accounts_db = RedisClient('accounts', self.website)
self.uid_db = RedisClient('uid', self.website)
def run(self):
"""
运行, 得到所有账户, 然后逐个模拟登录获取cookies,以及uid
:return:
"""
accounts_usernames = self.accounts_db.usernames()
cookies_usernames = self.cookies_db.usernames()
for username in accounts_usernames:
if not username in cookies_usernames:
password = self.accounts_db.get(username)
print('正在生成Cookies', '账号', username, '密码', password)
result = self.new_cookies(username, password)
# 成功获取
if result.get('status') == 1:
cookies = result.get('content') #cookies为dict类型
uid = result.get('uid')
print('成功获取到Cookies', cookies, '成功获取到uid', uid)
if self.cookies_db.set(username, json.dumps(cookies)):
print('成功保存Cookies')
if self.uid_db.set(username, uid):
print('成功保存uid')
# 登录出错
elif result.get('status') == 0:
print(result.get('content'))
else:
print('所有账号都已经成功获取Cookies')
def new_cookies(self, username, password):
"""
生成Cookies
:param username: 用户名
:param password: 密码
:return: 用户名和Cookies
"""
return WeiboCookies(username, password).main()
def post(self):
payload = {}
data = tornado.escape.json_decode(self.request.body)
user_id = data["id"]
if ("name" not in data):
payload = {"status": 500}
else:
user_name = data["name"]
r = RedisClient()
r.set(user_id + ":name", user_name)
payload = {
"id": user_id,
"name": user_name,
"status": 200
}
self.write(tornado.escape.json_encode(payload))
def save_headline(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response("Sorry, I didn't get that.")
article = self.extract_article()
if article:
confirmation = SaveConfirmation()
self.payload = {"read": confirmation.get_phrase()}
# Do some other stuff to actually save the article somewhere
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
self.finish_response()
else:
self.error_response(
"I'm sorry. I don't know which article you want me to save.")
class Getter(object):
# 爬取代理网站的免费代理IP，存入redis
def __init__(self):
self.redis_client = RedisClient()
self.crawler = Crawler()
def is_full(self):
# 判断代理池是否满了
return self.redis_client.get_proxy_count() >= FULL_COUNT
def run(self):
# 将爬取到的代理存入redis
if not self.is_full():
proxys = self.crawler.get_crawler_proxy()
for proxy in proxys:
self.redis_client.add(proxy)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy: 单个代理
:return: NOne
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = "http://" + proxy
print("正在测试", proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODES:
self.redis.max(proxy)
print("代理可用", proxy)
else:
self.redis.decrease(proxy)
print("请求接口相应不合法", proxy)
except:
pass
def run(self):
"""
测试主函数
:return: None
"""
print("测试器开始执行")
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print("测试器发生错误", e.args)
def save_headline(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response("Sorry, I didn't get that.");
article = self.extract_article()
if article:
confirmation = SaveConfirmation()
self.payload = {
"read": confirmation.get_phrase()
}
# Do some other stuff to actually save the article somewhere
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
self.finish_response()
else:
self.error_response("I'm sorry. I don't know which article you want me to save.")
def respond_get_headlines(self, payload):
if not payload:
self.error_response("API error")
sentence_headlines, topic_headlines, article_order = NLPParser.parse_headlines(map(lambda x: x["headline"], payload))
article_values = []
for article in payload:
article_values.append({"headline": article["headline"], "url": article["url"], "snippet": article["snippet"]})
self.pushlater = {
"type": "article",
"value": article_values
}
found = FoundHeadlines(sentence_headlines, topic_headlines, self.name)
self.payload = {
"read": found.get_phrase()
}
r = RedisClient()
ordered_articles = map(lambda x: payload[x], article_order)
# State transition
r.set(self._id + ":selected", None)
r.set(self._id + ":articles", ordered_articles)
r.set(self._id + ":state", "headlines")
self.finish_response()
def store_proxies(proxies):
conn = RedisClient(name='certificate_proxies')
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
conn_check = connect_check(proxies)
if conn_check:
proxies = json.dumps(proxies)
duplicate_check = conn.exist(proxies)
if not duplicate_check:
conn.set(proxies, 1)
info_logger.info(str(now) + ' New proxies: ' + str(proxies))
print(str(now) + ' New proxies: ' + str(proxies))
else:
info_logger.info(
str(now) + ' Already exist proxies: ' + str(proxies))
else:
err_logger.error(
str(now) + ' Can not connect baidu.com -- proxies: ' +
str(proxies))
def get_summary(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response("Sorry, what kind of news would you like to hear about?")
article = self.extract_article()
if article:
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
print "**************"
print type(article)
print article
print article is None
self.payload = {
"read": "%s. Would you like me to send the full article to your kindle?" % article["snippet"]
}
self.finish_response()
else:
self.error_response("Sorry I don't know what article you are talking about")
class API(object):
def __init__(self,
host="localhost",
port='6379',
key='Proxy',
moni_port=9999):
self.__redis = RedisClient(host=host,
port=port,
password=None,
key=key)
self.__port = moni_port
self.__host = host
self.__db_port = port
self.__key = key
def __message(self, method):
if method == "random":
return self.__redis.random()
elif method == "random_max":
return self.__redis.random_max()
elif method == "all":
return self.__redis.all()
elif method == "all_max":
return self.__redis.all_max()
elif method == "count":
return self.__redis.count()
else:
return "Sorry,wrong name or wrong password,try again..."
def run(self):
self.__set_api()
app.run(host="0.0.0.0", port=self.__port)
def __set_api(self):
global HOST
global PORT
global KEY
global MONI_PORT
HOST = self.__host
PORT = self.__db_port
KEY = self.__key
MONI_PORT = self.__port
def get_uniform_proxy(name, url):
proxy_url = 'http://166.188.20.55:3000/api/proxy/web'
headers = {
'Content-Type':
'application/json',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/'
'537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
}
data = {'name': name, 'url': url}
while True:
resp = requests.post(url=proxy_url,
headers=headers,
data=json.dumps(data)).json()
region = resp['proxy']['region']
if region == 'R1' or region == 'R2':
# print(region, 'err')
continue
else:
conn = RedisClient(name='certificate_proxies')
ip = resp['proxy']['ip']
port = '30000'
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port),
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url,
proxies=proxies,
timeout=(3.1, 15)).status_code
if status_code == 200:
p = json.dumps(proxies)
check = conn.exist(p)
if not check:
conn.set(p, 1)
# conn.lpush(p)
print('New proxies: ', proxies)
break
else:
print('already exist proxies: ', p)
else:
print('connection error...')
def get_media(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response("Sorry, what kind of news would you like to hear about?")
article = self.extract_article()
if article:
if article['multimedia']:
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
have_media = Media(self.name)
self.payload = {
"read": have_media.get_phrase() # usable url for html
}
self.pushlater = {"type": "media", "value": article['multimedia']}
self.finish_response()
else:
self.error_response("Sorry I don't have images for that article")
else:
self.error_response("Sorry I don't know what article you are talking about")
def __init__(self, redsmin_prod):
#creating both sockets
self.redisClient = RedisClient(config["uri"])
self.endpoint = Endpoint(redsmin_prod)
#binding data's transfer to the sockets
self.redisClient.handle_read = self.sendDataFromRedisToEndPoint
self.endpoint.handle_read = self.sendDataFromEndPointToRedis
#calling the handshake
self.endpoint.handshake()
def do_response():
redis = RedisClient(host=HOST, port=PORT, key=KEY)
RANDOM = str(redis.random())
RANDOM_MAX = str(redis.random_max())
ALL = redis.all()
all = ''
for ip in ALL:
all += str(ip) + "<br />"
ALL = all
ALL_MAX = redis.all_max()
all = ''
for ip in ALL_MAX:
all += str(ip) + "<br />"
ALL_MAX = all
COUNT = str(redis.count())
WRONG = 'Sorry,wrong name or wrong password,try again...'
if request.method == "GET":
name = request.args.get("name", "")
password = request.args.get("password", "")
method = request.args.get("method", "")
if not name:
return app.send_static_file("index.html")
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
if request.method == "POST":
name = request.form['name']
password = request.form['password']
method = request.form['method']
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
def post(self):
self.pushlater = {}
self.payload = {}
self.outcome = tornado.escape.json_decode(self.request.body)
intent = self.outcome["intent"]
if ("id" in self.outcome):
self._id = self.outcome["id"]
else:
self.error_response("No id passed in outcome")
print "DEBUGGING RESPONSE"
print self.outcome
print self._id
f = FirebaseDB()
if ("user" in self.outcome):
f.post(self._id, {"type": "user", "value": self.outcome["user"]})
if ("reporta" in self.outcome):
f.post(self._id, {
"type": "reporta",
"value": self.outcome["reporta"]
})
r = RedisClient()
self.name = r.get(self._id + ":name")
if (intent == "start"):
self.start()
elif (intent == "confirm_action"):
self.confirm_action()
elif (intent == "get_headlines"):
self.get_headlines()
elif (intent == "get_summary"):
self.get_summary()
elif (intent == "get_media"):
self.get_media()
elif (intent == "save_headline"):
self.save_headline()
else:
error_messages = Error(self.name)
self.error_response(error_messages.get_phrase())
class Validation(object):
def __init__(self,
validation_addr="http://www.baidu.com",
host="www.baidu.com",
batch_size=100,
db_host='localhost',
port='6379',
key='Proxy'):
self.__redis = RedisClient(host=db_host,
port=port,
password=None,
key=key)
self.__host = host
self.__addr = validation_addr
self.__batch_size = batch_size
async def validation_single(self, proxy):
'''validate single proxy'''
# print("validation proxy: ", proxy)
headers = UserAgent(host=self.__host).headers()
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
async with session.get(self.__addr,
proxy=real_proxy,
headers=headers,
timeout=15) as response:
if response.status == 200:
self.__redis.max(proxy)
# print("proxy {} is good.".format(proxy))
else:
self.__redis.decrease(proxy)
# print("proxy {} is not good".format(proxy))
except Exception:
# print("proxy {} is not good".format(proxy))
self.__redis.decrease(proxy)
def run(self):
try:
proxies = self.__redis.all()
for i in range(0, len(proxies), self.__batch_size):
# print("processing i: {}\tprogress:{}/{} {:.2f}%".format(i + 1, i + 1, len(proxies),
# (i + 1) / len(proxies) * 100))
task_proxies = proxies[i:self.__batch_size + i]
loop = asyncio.get_event_loop()
tasks = [
self.validation_single(proxy) for proxy in task_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
# print("After validation,total {} in database".format(self.__redis.count()))
except Exception as e:
pass
class WeiboValidTester(object):
def __init__(self, website='weibo'):
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.uid_db = RedisClient('uid', self.website)
def test(self, username, cookies):
print('正在测试Cookies', '用户名', username)
try:
cookies = json.loads(cookies)
except TypeError:
print('Cookies不合法', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
return
try:
test_url = TEST_URL_MAP[self.website]
response = requests.get(test_url % self.uid_db.get(username),
cookies=cookies,
timeout=5,
allow_redirects=False)
if response.status_code == 200:
print('Cookies有效', username)
else:
print(response.status_code, response.headers)
print('Cookies失效', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
except ConnectionError as e:
print('发生异常', e.args)
def run(self):
cookies_groups = self.cookies_db.all()
for username, cookies in cookies_groups.items():
self.test(username, cookies)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('测试代理: {}'.format(real_proxy))
async with session.get(TEST_URL,
proxy=real_proxy,
allow_redirects=False,
timeout=10) as resp:
if resp.status in VALID_STATUS_CODE:
self.redis.max(proxy)
print('代理{} 可用'.format(real_proxy))
else:
self.redis.decrase(proxy)
print('代理{}的返回状态错误'.format(real_proxy))
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
asyncio.TimeoutError, AttributeError):
self.redis.decrase(proxy)
print("代理{}请求异常".format(real_proxy))
def run(self):
'''
测试主函数
'''
print('开始测试')
try:
count = self.redis.count()
print('当前剩余{}个代理'.format(count))
for i in range(0, count, BATCH_TEST_SIZE):
start = i
stop = min(i + BATCH_TEST_SIZE, count)
print('测试测范围{}-{}'.format(start, stop))
test_proxies = self.redis.batch(start, stop)
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
sys.stdout.flush()
time.sleep(2)
except Exception as e:
print('测试异常{}'.format(e.args))
class Tester(object):
"""
从redis中取出代理，测试代理是否可用，并调整代理IP的优先级
"""
def __init__(self, test_url):
self.redisdb = RedisClient()
# 用来测试代理是否可用的地址
self.test_url = test_url
def test_proxy(self, proxy):
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy
}
print('正在检测:{}'.format(proxy))
res = requests.get(self.test_url, proxies=proxies, timeout=10)
if res.status_code == 200:
return True, proxy
else:
return False, proxy
# 代理不可用，就降低其优先级
except Exception as e:
return False, proxy
# print('代理检测异常:{} {}'.format(proxy, e))
self.redisdb.decrease(proxy)
print('代理不可用:{}'.format(proxy))
def run(self):
print('启动检测模块......')
try:
# 获取redis中所有爬取到的代理
proxies = self.redisdb.get_all_proxy()
for i in range(0, len(proxies), 50):
test_proxies = proxies[i:i+50]
workers = len(test_proxies)
with futures.ThreadPoolExecutor(workers) as executor:
tasks_res = executor.map(self.test_proxy, test_proxies)
for res, proxy in tasks_res:
if not res:
# 代理不可用，就降低其优先级
self.redisdb.decrease(proxy)
print('代理不可用:{}'.format(proxy))
else:
# 代理可用,将其优先级置为最大
self.redisdb.max(proxy)
print('代理可用:{}'.format(proxy))
except Exception as e:
print(traceback.format_exc())
print('检测模块出错！！！')
class Getter(object):
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
'''
判断是否超出了抓取的限制
'''
if self.redis.count() >= settings.POOL_UPPER_THRESHOLD:
return True
return False
def run(self):
print('开始抓取')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
self.redis.add(proxy)
class Getter(object):
def __init__(self):
self.redis_client = RedisClient()
self.crawler = Crawler()
def is_full(self):
"""
判断代理池是否满了
"""
if self.redis_client.get_proxy_count() >= FULL_COUNT:
return True
else:
return False
def run(self):
"""
将爬取到的代理存入redis
"""
if not self.is_full():
proxy_list = self.crawler.get_crawler_proxy()
for proxy in proxy_list:
self.redis_client.add(proxy)
def get_summary(self):
r = RedisClient()
state = r.get(self._id + ":state")
if state == "start":
self.error_response(
"Sorry, what kind of news would you like to hear about?")
article = self.extract_article()
if article:
r.set(self._id + ":selected", article)
r.set(self._id + ":state", "selected")
print "**************"
print type(article)
print article
print article is None
self.payload = {
"read":
"%s. Would you like me to send the full article to your kindle?"
% article["snippet"]
}
self.finish_response()
else:
self.error_response(
"Sorry I don't know what article you are talking about")
class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
判断是否达到代理池限制
:return:
"""
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
print("获取器开始执行")
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
for proxy in proxies:
self.redis.add(proxy)
def post(self):
self.pushlater = {}
self.payload = {}
self.outcome = tornado.escape.json_decode(self.request.body)
intent = self.outcome["intent"]
if ("id" in self.outcome):
self._id = self.outcome["id"]
else:
self.error_response("No id passed in outcome")
print "DEBUGGING RESPONSE"
print self.outcome
print self._id
f = FirebaseDB()
if ("user" in self.outcome):
f.post(self._id, {"type": "user", "value": self.outcome["user"] })
if ("reporta" in self.outcome):
f.post(self._id, {"type": "reporta", "value": self.outcome["reporta"] })
r = RedisClient()
self.name = r.get(self._id + ":name")
if (intent == "start"):
self.start()
elif (intent == "confirm_action"):
self.confirm_action()
elif (intent == "get_headlines"):
self.get_headlines()
elif (intent == "get_summary"):
self.get_summary()
elif (intent == "get_media"):
self.get_media()
elif (intent == "save_headline"):
self.save_headline()
else:
error_messages = Error(self.name)
self.error_response(error_messages.get_phrase())
def start(self):
r = RedisClient()
# Clear the state for this new user
r.set(self._id + ":state", None)
r.set(self._id + ":articles", None)
r.set(self._id + ":selected", None)
NYTimes.get_headlines(self.respond_start)
class Proxy:
def __init__(self, redsmin_prod):
#creating both sockets
self.redisClient = RedisClient(config["uri"])
self.endpoint = Endpoint(redsmin_prod)
#binding data's transfer to the sockets
self.redisClient.handle_read = self.sendDataFromRedisToEndPoint
self.endpoint.handle_read = self.sendDataFromEndPointToRedis
#calling the handshake
self.endpoint.handshake()
#transfer data from Redsmin to Redis (called when the sockets have something to read)
def sendDataFromEndPointToRedis(self):
data = self.endpoint.recv(8192)
if(not self.endpoint.handshaken):
print data
self.redisClient.send(data)
#transfer data from Redis to Redsmin (called when the sockets have something to read)
def sendDataFromRedisToEndPoint(self):
data = self.redisClient.recv(8192)
self.endpoint.send(data)
def __init__(self, website='weibo'):
"""
初始化一些对象
:param website: 名称
:param browser: 浏览器, 若不使用浏览器则可设置为 None
"""
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.accounts_db = RedisClient('accounts', self.website)
self.uid_db = RedisClient('uid', self.website)
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
"""
测试单个代理
:param proxy:
:return:
"""
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('正在测试：', proxy)
async with session.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STSTUS_CODES:
self.redis.max(proxy)
print('代理可用：', proxy)
else:
self.redis.decrease(proxy)
print('请求响应码不合法', proxy)
except (ClientError, ClientConnectorError, TimeoutError,
AttributeError):
self.redis.decrease(proxy)
print('代理请求失败', proxy)
def run(self):
"""
测试主函数
:return:
"""
print('测试器开始运行')
try:
proxies = self.redis.all()
loop = asyncio.get_event_loop()
#批量测试
for i in range(0, len(proxies), BATCH_TEST_SIZE):
test_proxies = proxies[i:i + BATCH_TEST_SIZE]
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
time.sleep(5)
except Exception as e:
print('测试器发生错误', e.args)
def respond_start(self, articles):
if not article:
self.error_response("API error")
r = RedisClient()
keywords = r.get(self._id + ":keywords")
if keywords:
keyword_articles = NYTimes.check_keywords(articles, keywords)
hasUpdate = len(keywords) > 0
articles = keyword_articles
else:
hasUpdate = False
start_state = UpdatesOrNoUpdates(hasUpdate, self.name)
self.payload = {
"read": start_state.get_phrase()
}
r.set(self._id + ":articles", articles)
r.set(self._id + ":state", "start")
self.finish_response()
import sys
from RedisClient import RedisClient
import json
def usage():
print "usage: python test_redis.py command{add|update|get} winid params"
print "\ttest_redis.py add winid dev"
print "\ttest_redis.py update winid app state"
print "\ttest_redis.py get winid"
exit(-1)
if len(sys.argv) < 3:
usage()
winid = int(sys.argv[2])
rcli = RedisClient()
if sys.argv[1] == 'add':
dev = sys.argv[3]
rcli.add_window(winid,dev)
print "add window [%x | %s] succeed!" %(winid,dev)
elif sys.argv[1] == 'update':
app = sys.argv[3]
state = sys.argv[4]
rcli.update(winid,app,state)
print "update window [%x] app=[%s]; state=[%s] succeed!" %(winid,app,state)
elif sys.argv[1] == 'get':
val = rcli.get_window_status(winid)
txt = json.dumps(val).decode('unicode-escape')
print "window id=%x; %s" %(winid,txt)
else:
def do_conn(self, arg, opts=None):
(host, port, db) = config.get_conn(int(arg))
self.prompt = "%s:%s > " % (host, str(db))
self.conn = RedisClient(host, port, db)
