class MonitorClient(object):
def __init__(self):
self.configs = {}
self.redis = RedisHelper()
def handle(self):
self.redis.public('xxxxxxxxxxxxxxxxxxxxxx')
class ErrorCounter():
def __init__(self):
self.key_prefix = "err_cnt:%s" % common.config.get("global", "error_counter_prefix")
self.handler = RedisHelper()
def incr(self):
dt = datetime.now()
key = "%s:%s" % (self.key_prefix, dt.strftime("%Y%m%d"))
field = dt.strftime("%H%M")
res = self.handler.hincrby(key, field, 1)
self.handler.expire(key, 86400)
return res
def __init__(self):
# 线程数量
self.pro_num = 20
# 记录失败的code的处理次数，达到N次将不会再次进队
self.fail_id = {}
# 连接MongoDB
self.conn = MongoClient(BaseConfig.mongo_cfg)
self.db = self.conn.movies
# 获取RedisHelper
self.redis_helper = RedisHelper()
class MonitorClient(object):
def __init__(self):
self.redis = RedisHelper()
self.configs = {}
@staticmethod
def format_msg(key, value):
msg = {key: value}
return pickle.dumps(msg)
def get_configs(self):
configs = self.redis.get('HostConfig:%s' % client_ip)
if configs:
self.configs = pickle.loads(configs)
return True
def handle(self):
if self.get_configs():
while True:
for service_name, config_value in self.configs.items():
interval, plugin, last_time = config_value
if time.time() - last_time > interval:
a = threading.Thread(target=self.task,
args=[service_name, plugin])
a.start()
self.configs[service_name][2] = time.time()
else:
wait_time = interval - (time.time() - last_time)
print '%s will run in %s seconds' % (service_name,
wait_time)
time.sleep(1)
else:
print '--- could not find the monitor configure ---'
def task(self, service_name, plugin_name):
print '--- Going to handle the %s task ---' % service_name
func = getattr(plugins_api, plugin_name)
res = func()
print "\033[1;32m%s\033[0m" % res
# 拼接出传递的数据
msg = self.format_msg('report', {
'ip': client_ip,
'service_name': service_name,
'info_data': res
})
self.redis.publish(msg)
def run(self):
print '--- start Monitor Client ---'
self.handle()
def main():
load_dotenv()
global REDIS_DB
REDIS_DB = RedisHelper().connection
updater = Updater(os.getenv("TG_TOKEN_QUIZ_BOT"))
dp = updater.dispatcher
conv_handler = ConversationHandler(
entry_points=[
CommandHandler('start', start),
RegexHandler('^Новый вопрос$', handle_new_question_request)
],
states={
States.QUESTION: [RegexHandler('^Новый вопрос$', handle_new_question_request)],
States.ANSWER: [
RegexHandler('^Сдаться$', handle_kapitulation),
MessageHandler(Filters.text, handle_solution_attempt)
],
},
fallbacks=[RegexHandler('^Сдаться$', handle_kapitulation)]
)
dp.add_handler(conv_handler)
updater.start_polling()
updater.idle()
class PubSubHelper(object):
MSG_TYPE_TB_UPDATE = 1
MSG_TYPE_FILE_EXPORT = 2
def __init__(self):
conf = common.config
self.channel = "fe:ws:%s" % conf.get("global", "subpub_channel")
self.handler = RedisHelper()
def pub(self, msg):
return self.handler.pub(self.channel, dumps(msg))
def listen(self):
c = self.handler.get_client(self.handler.MODE_READ)
ps = c.pubsub()
ps.subscribe(self.channel)
for msg in ps.listen():
if msg['type'] == "message":
yield loads(msg['data'])
def __init__(self):
self.headers = {'User-Agent': Faker().user_agent()}
self.db = RedisHelper()
self.save_to_redis = self.db.save_proxy_ip
self.pattern_tags = re.compile(r'<[^>]+>', re.S)
self.pattern_blank = re.compile(r'\s+', re.S)
self.pattern_colon = re.compile(r' ', re.S)
self.pattern_ip = re.compile(r'(?:\d+\.){3}\d+:\d+')
# 搜索深度(页数)
self.search_depth = 5
def __init__(self, config, loop=None):
self.loop = loop or asyncio.get_event_loop()
redis_url = config.pop('KAFKA_REDIS_URL', None)
assert redis_url is not None, 'kafka server need redis config: KAFKA_REDIS_URL'
self.kafka_redis = RedisHelper(redis_url, loop=loop)
self._waiting_publish_message_queue = config.pop(
'WAITING_PUBLISH_MESSAGE_QUEUE',
None) or 'kafka_waiting_publish_message'
self._consumers_name_set = set()
self._consumers = []
self._producer_count = config.pop('PRODUCER_COUNT', None) or 5
self._producers = []
self.config = config
# 'INIT' -> 'RUNNING' -> 'STOP'
self.status = 'INIT'
import datetime
from config import Config
from redis_helper import RedisHelper
redis_helper = RedisHelper()
def update_prices():
print(f'{datetime.datetime.now()}\tUpdate started')
conn = Config.conn
redis_helper.update_date()
redis_helper.load_prices(conn)
redis_helper.add_product(conn)
conn.close()
print(f'{datetime.datetime.now()}\tUpdate ended')
update_prices()
from redis_helper import RedisHelper redis_con = RedisHelper()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from redis_helper import RedisHelper
obj = RedisHelper()
obj.public('hello')
import redis
from flask import Flask, render_template, flash
from flask_bootstrap import Bootstrap
from flask_wtf import FlaskForm
from flask_wtf.csrf import CSRFProtect
from wtforms import SubmitField, SelectMultipleField, widgets
from config import Config
from order_table import OrderTable
from redis_helper import RedisHelper
app = Flask(__name__)
Bootstrap(app)
app.config.from_object(Config)
csrf = CSRFProtect(app)
rh = RedisHelper()
class MultiCheckboxField(SelectMultipleField):
widget = widgets.ListWidget(prefix_label=False)
option_widget = widgets.CheckboxInput()
class CalcForm(FlaskForm):
conn = redis.Redis(db=1)
products_dict = rh.get_products_dict(conn)
conn.close()
proc = SelectMultipleField(
'Processors (CPUs)',
choices=list((x, x) for x in products_dict.get('Processors (CPUs)')))
mobo = SelectMultipleField('Motherboards',
class CatchMovieInfo():
def __init__(self):
# 线程数量
self.pro_num = 20
# 记录失败的code的处理次数，达到N次将不会再次进队
self.fail_id = {}
# 连接MongoDB
self.conn = MongoClient(BaseConfig.mongo_cfg)
self.db = self.conn.movies
# 获取RedisHelper
self.redis_helper = RedisHelper()
def send_request(self, url):
try:
headers = {'User-Agent': Faker().user_agent()}
proxy_ip = self.redis_helper.get_proxy_ip()
proxies = {
'http': 'http://' + proxy_ip,
'https': 'https://' + proxy_ip
}
res = requests.get(url,
headers=headers,
proxies=proxies,
timeout=5,
verify=False)
status_code = res.status_code
if status_code < 200 or status_code > 300:
print(url, 'Request error')
raise Exception('Request error')
return res
except Exception as e:
# print_exc()
pass
def catch_movie_info(self, movie_id):
try:
data = {}
data['_id'] = movie_id
url = 'https://movie.douban.com/subject/{movie_id}/'.format(
movie_id=movie_id)
data['url'] = url
res = self.send_request(url)
if not res:
raise Exception('send_request error')
obj = BeautifulSoup(res.content, 'lxml').select('#content')[0]
data['watch'] = [{
'source': i.a.string.strip(),
'url': i.a['href'].strip(),
'charge': i.span.span.string.strip()
} for i in obj.select('.gray_ad li')]
data['name'] = obj.select('h1 span')[0].string
data['year'] = re.findall(r'\d+',
obj.select('h1 span')[1].string)[0]
data['actor'] = [item.string for item in obj.select('.actor a')]
data['type'] = [
item.string for item in obj.select("[property='v:genre']")
]
duration = obj.select("[property='v:runtime']")
data['duration'] = duration[
0].string if duration else '' # 2018-5-24 部分影片无片长
data['votes'] = obj.select("[property='v:votes']")[0].string
data['score'] = obj.select("[property='v:average']")[0].string
data['releaseDate'] = [
item['content']
for item in obj.select("[property='v:initialReleaseDate']")
]
desc = obj.select("[property='v:summary']")
if desc:
data['desc'] = '\n'.join([
i.strip() for i in desc[0].get_text().split('\n')
if i.strip()
]) # desc
else:
data['desc'] = '' # 2018-5-24 部分影片无描述信息
data['img_url'] = obj.select('#mainpic img')[0]['src'] # img_url
data['douban_recommend'] = list(
set(link['href'].split('/')[4]
for link in obj.select('.recommendations-bd a')))
# 存在则更新，不存在则插入
self.db.raw.update({'_id': data['_id']}, data, upsert=True)
# pprint(data)
# TODO IMDB、 国家地区、语言、alias、
# TODO IMDB 评分 推荐
except Exception as e:
# 异常 返回movie id
# traceback.print_exc()
self.fail_id.setdefault(movie_id, 0)
self.fail_id[movie_id] += 1
return movie_id
def extract_movie_id(self, url):
try:
res = self.send_request(url)
if not res:
print(url, 'continue')
data = json.loads(res.text)
movie_ids = [i['id'] for i in data['data']]
cur_sum = self.redis_helper.save_movie_id(movie_ids)
print(url, 'success')
print('movie_id:', cur_sum)
except Exception as e:
pass
def worker(self, pid):
"""
从redis获movie id，去采movie info
:return:
"""
while True:
_id, _len = self.redis_helper.get_movie_id()
print(f'[worker_id:{pid}]剩余movie id数量:{_len}')
# 判断movie id失败次数是否超过3次，如果失败次数超过3次则不再处理
if self.fail_id.get(_id, 0) > 3:
self.fail_id[_id] = 0 # 清零
continue
movie_id = self.catch_movie_info(_id)
if movie_id:
self.redis_helper.save_movie_id(movie_id)
def catch_movie_id(self):
"""
catch movie id and push to redis queue
:return:
"""
# https://movie.douban.com/tag/#/?sort=U&range=9,10&tags=电影
movie_count = 20000
data = {'sort': 'U', 'range': '0,10', 'tags': '电影', 'limit': 200}
data['tags'] = parse.quote(data['tags'])
url = 'https://movie.douban.com/j/new_search_subjects?sort={sort}&range={range}&tags={tags}&limit={limit}'
url = url.format(**data)
url += '&start={start}'
urls = [
url.format(start=i)
for i in range(0, movie_count, data.get('limit', 20))
]
p = Pool(10)
p.map(self.extract_movie_id, urls)
def start_worker(self):
"""
启动worker，开始采集电影信息
:return:
"""
p = Pool((self.pro_num))
p.map(self.worker, list(range(1, self.pro_num + 1)))
#!/usr/bin/env python3 # coding:utf-8 ''' Created on: 2016年3月22日 @author: 张晓宇 Email: [email protected] Version: V1.0 Description: Redis订阅和发布演示程序，发布方 Help: ''' from redis_helper import RedisHelper # 导入刚才定义的Redis公共类 if __name__ == '__main__': obj = RedisHelper() # 创建redis公共类对象 while True: obj.public(input('>> ')) # 获取输入，并把输入的内容发布出去
# -*- coding: UTF-8 -*-
from redis_helper import RedisHelper
obj = RedisHelper()
while True:
msg = input('>>:')
obj.public(msg)
user_answer = event.text
if check_is_right_answer(user_answer, right_answer):
vk_api.messages.send(user_id=event.user_id,
message="Правильный ответ!",
keyboard=keyboard.get_keyboard(),
random_id=random.randint(1, 1000))
else:
vk_api.messages.send(user_id=event.user_id,
message="Неравильный ответ!",
keyboard=keyboard.get_keyboard(),
random_id=random.randint(1, 1000))
if __name__ == "__main__":
load_dotenv()
redis_db = RedisHelper().connection
quiz_data = get_quiz_data()
vk_session = vk_api.VkApi(token=os.getenv("VK_TOKEN_QUIZ_BOT"))
vk_api = vk_session.get_api()
keyboard = VkKeyboard()
keyboard.add_button('Новый вопрос', color=VkKeyboardColor.SECONDARY)
keyboard.add_button('Сдаться', color=VkKeyboardColor.NEGATIVE)
keyboard.add_line()
keyboard.add_button('Мой счет', color=VkKeyboardColor.POSITIVE)
longpoll = VkLongPoll(vk_session)
def __init__(self):
conf = common.config
self.channel = "fe:ws:%s" % conf.get("global", "subpub_channel")
self.handler = RedisHelper()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from redis_helper import RedisHelper
obj = RedisHelper()
redis_sub = obj.subscribe()
while True:
msg= redis_sub.parse_response() #听
print(msg)
def __init__(self):
self.key_prefix = "err_cnt:%s" % common.config.get("global", "error_counter_prefix")
self.handler = RedisHelper()
def __init__(self):
self.configs = {}
self.redis = RedisHelper()
item['city'] = 'xiamenshi'
item['base_info'] = base_info_data
item['detail'] = response_detail_data
kafka = Kafka()
kafka.process_item(item)
print('切换token的值')
MysqlHelper.update_params_count(count, shop_list_url)
param = get_new_token(redis_helper)
if param != None:
count = 0
get_shop_detail(redis_helper, page_index, eval(param),
location_point, count, flag)
else:
keys = 'lgt_lat_page'
values = (wm_latitude_before, wm_latitude_before, page_index)
redis_helper.lpush_data(keys, values)
return
if __name__ == '__main__':
redis_helper = RedisHelper()
param = redis_helper.spop_data("token_url_header")
if param != None:
location_point = redis_helper.spop_data('lgt_lat_page')
location_point = eval(location_point)
page_index = location_point[2].encode('utf-8')
count = 0
flag = 0
get_shop_detail(redis_helper, page_index, eval(param), location_point,
count, flag)
def __init__(self):
self.redis = RedisHelper()
self.configs = {}
from redis_helper import RedisHelper
obj = RedisHelper()
obj.publish('hello world')
print 1
from redis_helper import RedisHelper
import datetime
import time
import time_utils
helper = RedisHelper()
redis_sub = helper.subscribe()
while True:
# 这是使用base64编码，和下面使用字符直接编码，性能差距很小
# msg = redis_sub.parse_response()
# d = datetime.datetime.now()
# base64_bytes = msg[2].encode('ascii')
# image_bytes = base64.b64decode(base64_bytes)
# time_utils.millis(datetime.datetime.now(), d)
# with open('received-image-{}.{}'.format(datetime.datetime.now(), 'jpg'), 'wb') as image_file:
# image_file.write(image_bytes)
for item in redis_sub.listen():
msg = bytes(item['data'], encoding='ISO-8859-1')
d = datetime.datetime.now()
time_utils.millis(datetime.datetime.now(), d)
with open('received-image-{}.{}'.format(time.time(), 'jpg'),
'wb') as image_file:
image_file.write(msg)
# -*- coding:utf-8 -*-
__author__ = 'liulin'
from redis_helper import RedisHelper
obj = RedisHelper()
redis_sub = obj.subscribe()
while True:
msg = redis_sub.parse_response()
print(msg)
#发布者：
#!/usr/bin/env python
#-*- coding:utf-8 -*-
__author__ = 'luotianshuai'
from redis_helper import RedisHelper
obj = RedisHelper() #实例化方法
obj.publish('hello') #执行发布
#!/usr/bin/env python3 # coding:utf-8 """ Created on: 2016年3月22日 @author: 张晓宇 Email: [email protected] Version: V1.0 Description: Redis订阅和发布演示程序，订阅方 Help: """ from redis_helper import RedisHelper # 导入redis公共类 if __name__ == "__main__": obj = RedisHelper() # 创建公共类对象 redis_sub = obj.subscribe() # 执行收听方法 while True: msg = redis_sub.parse_response() # 第二次调用parse_response()方法，开始阻塞知道收到一条消息 print(msg) # 打印消息信息，parse_response()方法返回的其实是一个列表类似[b'message', b'fm104.5', b'abc']
from redis_helper import RedisHelper
import datetime
import base64
import time_utils
publish = RedisHelper()
# publish.publish('hello world.')
with open('robots.jpg', 'rb') as image_file:
image_bytes = image_file.read()
d = datetime.datetime.now()
# encoded = base64.b64encode(image_bytes).decode('ascii')
# publish.publish(encoded)
# 读取的文件的编码居然是ISO-8859-1, 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte
publish.publish(str(image_bytes, encoding='ISO-8859-1')) # 字节数组转str？时间上跟b64encode差不多
time_utils.millis(datetime.datetime.now(), d)
