def node_kill_event(redis, sport, hash):
#redis komanda za ubijanje singla
node_kill_e_message = "kill:{}:{}".format(sport, hash)
node_cmd_ch = get_curr_node_channels(rdb=redis,
ev_hash=hash,
key='commands')
redis.lpush(node_cmd_ch, node_kill_e_message)
def on_status(self, data):
global count
count += 1
if count % 100 == 0:
logging.debug('%s tweets gathered.' % str(count))
tweetID = data.id_str
tweet = {}
tweet['id'] = encode(data.id_str)
tweet['created_at'] = dumps(data.created_at, default=json_serial)
tweet['text'] = encode(data.text)
tweet['username'] = encode(data.user.screen_name)
tweet['url'] = encode(data.user.url)
tweet['location'] = encode(data.user.location)
json_tweet = json.dumps(tweet)
redis.lpush(args.query, json_tweet)
i = redis.llen(args.query)
if args.mode == 'f':
if i % 50 == 0:
os.system('python writer.py -q %s' % args.query)
if args.mode == 'd':
if i % 50 == 0:
os.system('python upload.py -q %s' % args.query)
def parse(self, response):
redis = self.redis
re_url = re.compile(r'https://www.tianyancha.com/company/\d+')
code = response.meta['code']
url = re_url.findall(response.text)
if len(url) > 0:
url = url[0]
redis.lpush('lawcourt', code)
province = response.xpath(
'/html/body/div[2]/div/div[1]/div[2]/div[2]/div[1]/div/span/text()'
).extract_first()
score = response.xpath(
'/html/body/div[2]/div/div[1]/div[2]/div[2]/div[1]/div/div[4]/span[1]/text()'
).extract_first()
yield scrapy.Request(url,
callback=self.parse_second,
meta={
'url_link': url,
'code': code,
'province': province,
'score': score
})
else:
redis.rpush('base', code)
code = redis.lpop('base')
print(self.redis.llen('base'))
url = 'https://www.tianyancha.com/search?key={}'.format(code)
yield scrapy.Request(url,
callback=self.parse,
meta={'code': code},
dont_filter=True)
def get_page_url(date, num, page):
print(date, num, page)
result = requests.get(
'https://pacaio.match.qq.com/openapi/json?key=news:%d&num=%d&page=%d&expIds=&callback=__jp0'
% (date, num, page))
# 得到的返回值被故意加了一层,把这一层切掉才是json的标准格式
j = json.loads(result.text[6:-1])
# 页数超过最后一页后,没有数据,直接跳出返回False停止下一页请求
if j['data'] == None:
return False
# 得到正常返回值后遍历所有'data'节点
for jj in j['data']:
title = jj['title']
url = jj['url']
print(title, url)
# 判断redis是否已经存在该文章标题,如果不存在,将它加进Redis中
if not redis.hexists(hashKey, title):
# 使用Hash是为了防止加入重复数据
# List是起到消息队列的作用,消费者从队列右侧pop即可
redis.hset(hashKey, title, url)
redis.lpush(listKey, title)
# 如果标题已经存在,有可能是以下情况:
# 1. 在之前已经爬过这一天数据了,数据没有更新
# 2. 在之前爬过这一天数据了,数据更新了,但是之前的循环已经把更新的数据搞定了
# 这两种情况中,无论剩下的还是后面页的所有数据都已经存入Redis了,所以直接跳出并且终止下一页的请求
else:
return False
# 本页数据都存成功了,请求下一页
return True
def serverside_task_loop(self):
"""
The server side task loop. Fetches new tasks and executes them, as appropriate.
"""
while True:
try:
msg_data = yield from self.get_msg()
except RuntimeError as e:
if e.args[0] == "Event loop is closed":
return
else:
raise
logger.debug("Got new task: {}".format(msg_data))
# On the server side, try and get the function.
func_id = msg_data['id']
if func_id not in self.tasks:
logger.critical(
"Could not find task `{}` on the server-side! Try reloading Kohlrabi."
.format(func_id))
continue
task = self.tasks[func_id]
# ACK that we got the func.
with (yield from self.redis_conn) as redis:
redis.lpush("{}-ACK".format(msg_data["ack"]), 1)
# Run it on the server-side.
assert isinstance(
task, ServerTaskBase
), "Task invocation should be happening on the server side"
self._loop.create_task(
task.invoke_func(msg_data["ack"], *msg_data["args"],
**msg_data["kwargs"]))
def main():
config = ConfigParser.ConfigParser()
if len(config.read(['/home/pi/lol/conf.ini'])):
api_key = config.get("user", "api_key")
redis_host = config.get("database", "redis_host")
redis_port = config.get("database", "redis_port")
redis = setRedisConn(host=redis_host, port=redis_port)
last_match = redis.lindex("last100", 0)
if last_match is None: # for brand-new database
match_id = getLastMatch(region, seed_pid="4460427", api_key=api_key)
else:
match_id = increaseId(last_match)
region = "kr"
while True:
# crawl the match data
match_details = getMatchDetails(region=region, match_id=match_id, api_key=api_key)
if match_details is not None:
# insert match data into redis
mid = match_details.pop("matchId")
redis.hmset(mid, match_details)
# update list of last-inserted
redis.lpush("last100", mid)
redis.ltrim("last100", 0, 99)
match_id = increaseId(match_id)
time.sleep(1) # to avoid api overshooting (max 500 queries per 10 min)
else:
print "File conf.ini not found. Program aborted."
exit(1)
def serverside_task_loop(self):
"""
The server side task loop. Fetches new tasks and executes them, as appropriate.
"""
while True:
try:
msg_data = yield from self.get_msg()
except RuntimeError as e:
if e.args[0] == "Event loop is closed":
return
else:
raise
logger.debug("Got new task: {}".format(msg_data))
# On the server side, try and get the function.
func_id = msg_data['id']
if func_id not in self.tasks:
logger.critical("Could not find task `{}` on the server-side! Try reloading Kohlrabi.".format(func_id))
continue
task = self.tasks[func_id]
# ACK that we got the func.
with (yield from self.redis_conn) as redis:
redis.lpush("{}-ACK".format(msg_data["ack"]), 1)
# Run it on the server-side.
assert isinstance(task, ServerTaskBase), "Task invocation should be happening on the server side"
self._loop.create_task(task.invoke_func(msg_data["ack"], *msg_data["args"], **msg_data["kwargs"]))
def poll_queue(redis):
while 1:
item = redis.zrange('delayed:queue:', 0, 0, withscores=True)
# 任务集合没有任务，或者任务的执行时间未到。
if not item or item[0][1] > time.time():
time.sleep(0.01)
continue
# 解码要被执行的任务，弄清除它应该被推入哪个任务队列。
item = item[0][0]
identifier, queue, name, args = json.loads(item)
# 为了对任务进行移动尝试获取锁
# 细粒度锁
locked = acquire_lock(redis, identifier)
if not locked:
continue
# 将任务推入适当的任务队列里面。
if redis.zrem('delayed:queue:', item):
redis.lpush('queue:' + queue, item)
release_lock(redis, identifier, locked)
#####################################################################################################################
# 延迟的优先级任务队列
# ['high-delayed', 'medium-delayed', 'low-delayed']
def create(cls, data, listeners):
if not 'id' in data:
data['id'] = redis.incr('global:%s' % cls._namespace())
_id = cls._id(data['id'])
redis.set(_id, data)
redis.lpush(cls._namespace(), _id)
return data, listeners
def yihaodian_order_listener(db, redis):
"""
定时从一号店获得新生成的订单, 筛选未处理订单，并放入订单处理队列
:type db:torndb.Connection
:type redis:redis.StrictRedis
"""
# 生产环境运行
# if not options.app_mode == 'prod':
# logging.info("yihaodian order listener quit: not in prod mode")
# return
end_time = datetime.now() + timedelta(minutes=10) # 当前时间往后10分钟
start_time = end_time - timedelta(days=14) # 初始时间推前14天,因为一号店api支持的最大跨度是15天
end = end_time.strftime('%Y-%m-%d %H:%M:%S')
start = start_time.strftime('%Y-%m-%d %H:%M:%S')
# 获得一号店订单列表
orders = fetch_yihaodian_order(start, end)
if orders:
# 查询已经存在的订单
result = db.query('select order_no from distributor_order where distributor_shop_id=%s and order_no in ('
+ ','.join(['%s']*len(orders)) + ')',
options.shop_id_yihaodian, *orders)
exist_orders = [row['order_no'] for row in result]
# 插入不存在的订单
for order in orders:
if not exist_orders or exist_orders and order not in exist_orders:
distributor_order_id = db.execute('insert into distributor_order (order_no, created_at, message, '
'distributor_shop_id) values (%s, now(), "", %s)',
order, options.shop_id_yihaodian)
logging.info('insert new distributor order_no: %s', order)
# 推进 redis 处理队列
redis.lpush(options.queue_distributor_order,
json_dumps({'distributor_order_id': distributor_order_id, 'distributor': 'YHD', 'retry': 0}))
def doWelfareById(redis, uid, account, id):
"""
福利
"""
today = datetime.now().strftime("%Y-%m-%d")
if id == '2':
playerCoin = redis.hget(FORMAT_USER_TABLE % uid, 'gold')
playerCoin = playerCoin if playerCoin else 0
if int(playerCoin) >= SIGN_LINE:
return {'code': 1, 'msg': u'未达到低保线无法领取'}
key = WELFARE_USER_INSURANCE % (account, today)
if redis.llen(key) >= SIGN_MAX:
return {'code': 1, 'msg': u'已经领取了 {0} 次'.format(SIGN_MAX)}
redis.lpush(key, SIGN_COINNUM)
player_add_gold(redis, account, SIGN_COINNUM)
elif id == '1':
# 新手礼包
if redis.hget(GOLD_REWARD_NEW_PRESENT_HASH, account) == MESSION_STATUS_OVER:
return {'code': 1, 'msg': u'您已领取，无法再次领取'}
redis.hset(GOLD_REWARD_NEW_PRESENT_HASH, account, MESSION_STATUS_OVER)
elif id == '0':
# 每日首冲奖励
if not redis.sismember(DAILY_USER_GOLD2_SET % today, account):
return {'code': 1, 'msg': u'您今日还未完成首冲'}
elif redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account) == MESSION_STATUS_OVER:
return {'code': 1, 'msg': u'您已领取，无法再次领取'}
redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account, MESSION_STATUS_OVER)
return {'code': 0, 'msg': u'领取成功'}
def send():
email = request.form.get("email", None)
address = request.form.get("address", None)
address = re.sub("/answer/\d+", '', address)
if email and address:
redis.lpush("kindle", address + ";" + email)
return jsonify({"code": 0})
return jsonify({"code": 412})
def episode_callback(episode):
# Compute total reward.
rewards = [t[2] for t in episode]
episode_reward = np.sum(rewards)
#if np.random.random_sample() < 0.05:
print('### episode_reward: {}.'.format(episode_reward))
redis.lpush(REWARDS_KEY, struct.pack('d', episode_reward))
redis.ltrim(REWARDS_KEY, 0, 20)
def send_policy(policy, data=None):
# Wrangle arguments into correct format
args = None
if len(data) == 1:
args = data[0]
elif len(data) > 1:
args = data
redis.lpush('cmds', json.dumps({'Policy': {policy: args}}))
def on_msg(client, userdata, message):
print('message received %s, searching for %s' %
(str(message.payload.decode('utf-8')), sensor_key))
data = json.loads(message.payload.decode('utf-8'))
data['value'] = data[sensor_key]
data['when'] = str(datetime.now())
redis.lpush(sys.argv[2], json.dumps(data))
redis.ltrim(sys.argv[2], 0, 50)
def chat_message(g):
if g.get('msg') and current_user.is_authenticated:
message = {
'user': current_user.name,
'msg': escape_html(g.get('msg')[:250])
}
redis.lpush('chathistory', json.dumps(message))
redis.ltrim('chathistory', 0, 20)
socketio.emit('msg', message, namespace='/snt', room='chat')
def re_map(msg_id, dbName):
start_id = int(msg_id) - 10
end_id = int(msg_id) + 10
sql = "select * from main.message WHERE type =3 and talker='9990246678@chatroom' and msgId >=" + str(
start_id) + " and msgId<=" + str(end_id) + ";"
data = query(dbName, sql)
for i in range(0, len(data)):
tailorImg(decodeDbName, data[i][0], data[i][1])
redis.lpush(has_done_key, msg_id)
def post(self, id):
redis = datastore.get_datastore()
log = request.get_json(force=True)
akey = akey = "activity:" + str(id) + ":logs"
redis.lpush (akey, pickle.dumps(log) )
# get a dict from the response
log = request.get_json(force=True)
return log, 201
def send_message(self, key='messages', value=''):
"""
Create or append redis list value with a given key
:param key: str, redis key
:param value: str, value
:return: None
"""
if not value:
value = self.generate_message()
redis.lpush(key, value)
def add_player():
if request.method == "POST":
name = request.form['name']
redis.lpush('players', name)
return f"{head}{css}Summoner <b>{name}</b> added! {foot}"
return head+css+"""
<form accept-charset=\"UTF-8\" action=\"\" autocomplete=\"off\" method=\"POST\">
<label for=\"name\">Player Name</label><br />
<input name=\"name\" type=\"text\" value=\"\" /> <br />
<button type=\"submit\" value=\"Submit\">Add</button>
</form>"""+foot
def on_click(self, widget):
if self.circles:
print(self.selected.number)
for c in self.circles:
if c == self.selected:
c.main = True
else:
c.main = False
c.cc = '#204a87'
redis.lpush("beat_queue",
"gui:bodies:setmain:{}".format(self.selected.number))
def checkRate(ip):
if redis.llen(ip) == 0:
print("First encounter")
i = 1
redis.lpush(ip,1)
redis.expire(ip,60)
else:
if redis.llen(ip) == 5:
return False
else:
redis.lpush(ip,1)
return True
def test_get_by_category(redis):
def get_by_category(client, category):
ids = client.lrange('category:' + category, 0, -1)
items = []
for id in ids:
items.append(client.hget(id, 'name'))
return items
redis.hmset('item:1', {'name': 'Foo', 'category': 'Bar'})
redis.lpush('category:Bar', 'item:1')
assert get_by_category(redis, 'Bar') == ['Foo']
def on_mouse_press(self, *args):
print("clicked")
if self.circles:
#self.text = "x:{} y:{}".format(self.selected.rect.left, self.selected.rect.bottom)
#redis.lpush("beat_queue", "gui:bodies:setlocation:{}:{},{}".format(self.selected.number, round(self.selected.rect.left)*4, round(self.selected.rect.bottom)*4))
# send locations of circles to redis
msg_list = []
for c in self.circles:
msg_list.append(
(round(c.rect.left * 4), round(c.rect.bottom * 4)))
msg = "|".join(["{},{}".format(a[0], a[1]) for a in msg_list])
print(msg)
redis.lpush("beat_queue", "gui:bodies:setlocations:{}".format(msg))
def node_open_event(redis, sport, hash):
#redis komanda za otvaranje singla
# todo: handle this !!!
try:
node_open_e_message = "open:{}:{}".format(sport, hash)
node_cmd_ch = get_curr_node_channels(rdb=redis,
ev_hash=hash,
key='commands')
redis.lpush(node_cmd_ch, node_open_e_message)
except Exception as e:
logger_filepath = '/var/log/sbp/flashscore/catched_errors.log'
error_logger = parserLog(logger_filepath, 'tipbet-live-errors')
error_logger.critical(e)
def outbox(ws):
handle = unicode(redis.get("handle"), "utf-8")
roomnum = unicode(redis.get("roomnum"), "utf-8")
print("regist:", handle, roomnum)
chats.register(ws, handle, roomnum)
redis.lpush(roomnum, handle)
app.logger.info(u"regist: {}".format(ws))
while not ws.closed:
gevent.sleep(0.1)
if ws.closed:
chats.delete_client[ws]
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def send_email_prd(redis, seller, item, price, buyer):
""" 先到先服务
lpush brpop
:param redis:
:param seller:
:param item:
:param price:
:param buyer:
:return:
"""
data = {
"seller": seller,
"item": item,
"price": price,
"buyer": buyer,
}
redis.lpush('queue:email', json.dumps(data))
def list_redis():
import redis
redis = redis.Redis(host='127.0.0.1',
port=6379,
db=0,
charset="utf-8",
decode_responses=True)
print("-------------")
print("LIST")
print("-------------")
#lpush(name, *values) - in initial position
redis.lpush("names", "pedro" " ana" " mara")
print("names: ", redis.lrange("names", 0, 2))
#delete all keys
for key in redis.scan_iter("prefix:*"):
redis.delete(key)
def send():
email = request.form.get("email", None)
address = request.form.get("address", None)
address = re.sub("/answer/\d+", '', address)
if email and address:
mobi = db.kindle.find({"address": address}).count()
if mobi:
print "already exists"
question_id = address.split("/")[-1]
mobi_path = os.path.join(os.path.join(os.getcwd(), "mobis"), question_id+".mobi")
mk = MailToKindle(mobi_path, email)
mk.send_mail()
else:
redis.lpush("kindle", address+";"+email)
db.kindle.insert({"address": address, "email": email})
return jsonify({"code": 0})
return jsonify({"code": 412})
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error':
"The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error':
"I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info':
"Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" %
data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url, ),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def send():
email = request.form.get("email", None)
address = request.form.get("address", None)
address = re.sub("/answer/\d+", '', address)
if email and address:
mobi = db.kindle.find({"address": address}).count()
if mobi:
print "already exists"
question_id = address.split("/")[-1]
mobi_path = os.path.join(os.path.join(os.getcwd(), "mobis"),
question_id + ".mobi")
mk = MailToKindle(mobi_path, email)
mk.send_mail()
else:
redis.lpush("kindle", address + ";" + email)
db.kindle.insert({"address": address, "email": email})
return jsonify({"code": 0})
return jsonify({"code": 412})
def test_list(redis, keys, length, retry):
print "generating data..."
for i in range(length):
redis.lpush("key", ''.join([random.choice(string.ascii_letters + string.digits) for i in range(8)]))
for i in range(keys - 1):
redis.lpush(''.join([random.choice(string.ascii_letters + string.digits) for i in range(8)]),
''.join([random.choice(string.ascii_letters + string.digits) for i in range(8)]))
delays = []
print "testing..."
for i in range(retry):
t1 = datetime.datetime.now()
redis.lrange("key", 0, -1)
t2 = datetime.datetime.now()
td = t2 - t1
delays.append(td.days * 24 * 3600 * 1000 + td.seconds * 1000 + td.microseconds / 1000.0)
result = pd.Series(delays)
result.to_csv("list_%d_%d.csv" % (length, retry))
print result.describe()
def get_items_ids(self, response):
soup = BeautifulSoup(response.text, 'lxml')
ids = None
try:
ids = re.search(r'ordered_service_ids = (.*)?]', soup.text, re.M).group(0).replace('ordered_service_ids = ',
'')
ids = eval(ids)
except Exception as e:
print(e)
print(response.url + ' cannot get app\'s ids, and had skipped.')
return ids
redis = redis_resource()
for _id in ids:
if redis.sismember(consumedKey(response.persist['category']), _id):
print('id: %s under %s had already crawlered, auto skip done' % (_id, response.persist['category']))
continue
else:
redis.lpush(response.persist['category'], _id)
return {response.persist['category']: ids}
def yihaodian_order_listener(db, redis):
"""
定时从一号店获得新生成的订单, 筛选未处理订单，并放入订单处理队列
:type db:torndb.Connection
:type redis:redis.StrictRedis
"""
# 生产环境运行
# if not options.app_mode == 'prod':
# logging.info("yihaodian order listener quit: not in prod mode")
# return
end_time = datetime.now() + timedelta(minutes=10) # 当前时间往后10分钟
start_time = end_time - timedelta(days=14) # 初始时间推前14天,因为一号店api支持的最大跨度是15天
end = end_time.strftime('%Y-%m-%d %H:%M:%S')
start = start_time.strftime('%Y-%m-%d %H:%M:%S')
# 获得一号店订单列表
orders = fetch_yihaodian_order(start, end)
if orders:
# 查询已经存在的订单
result = db.query(
'select order_no from distributor_order where distributor_shop_id=%s and order_no in ('
+ ','.join(['%s'] * len(orders)) + ')', options.shop_id_yihaodian,
*orders)
exist_orders = [row['order_no'] for row in result]
# 插入不存在的订单
for order in orders:
if not exist_orders or exist_orders and order not in exist_orders:
distributor_order_id = db.execute(
'insert into distributor_order (order_no, created_at, message, '
'distributor_shop_id) values (%s, now(), "", %s)', order,
options.shop_id_yihaodian)
logging.info('insert new distributor order_no: %s', order)
# 推进 redis 处理队列
redis.lpush(
options.queue_distributor_order,
json_dumps({
'distributor_order_id': distributor_order_id,
'distributor': 'YHD',
'retry': 0
}))
def on_data(self, data):
jsonData = json.loads(data)
tweet = {}
tweet['id'] = jsonData['id_str']
tweet['text'] = jsonData['text']
tweet['username'] = jsonData['user']['screen_name']
json_tweet = json.dumps(tweet)
self.queue.put(redis.lpush('event', json_tweet))
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error': "The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error': "I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info': "Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" % data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url,),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def send_msg(self, to_send, queue="kohlrabi-tasks"):
"""
Put a message onto the queue.
This will automatically msgpack-encode the data, including a 'use_bin_type' value so the resulting data can
be decoded in UTF-8 properly.
:param to_send: The data to send.
:param queue: The queue to send the data to.
"""
data = msgpack.packb(to_send, use_bin_type=True)
with (yield from self.redis_conn) as redis:
assert isinstance(redis, aioredis.Redis)
yield from redis.lpush(queue, data)
def task_status():
cursor.execute("SELECT id, iphone_token FROM users WHERE iphone_token <> '' GROUP BY iphone_token")
rows = cursor.fetchall()
for row in rows:
sql = "SELECT count(*) AS status_count FROM tasks WHERE user_id='%s' AND status='%s' AND is_del='%s'" % (
row["id"],
0,
0,
)
cursor.execute(sql)
result = cursor.fetchone()
if result and result["status_count"] > 3:
message = "云秘书提醒您：您还有%s个未完成的工作等待您的处理！" % str(result["status_count"])
value = {"iphone_token": row["iphone_token"], "message": message}
res = redis.lpush("notify", json.dumps(value))
def twodays():
cursor.execute("SELECT id, iphone_token FROM users WHERE iphone_token <> '' GROUP BY iphone_token")
rows = cursor.fetchall()
today = datetime.datetime.now()
week = int(today.strftime("%w"))
if week == 1 or week == 0 or week == 6:
start_time = today - datetime.timedelta(days=4)
else:
start_time = today - datetime.timedelta(days=2)
for row in rows:
sql = "SELECT id FROM tasks WHERE user_id='%s' AND created_at > '%s'" % (row["id"], start_time)
cursor.execute(sql)
result = cursor.fetchone()
if result is None:
value = {"iphone_token": row["iphone_token"], "message": "云秘书提醒您：您有两天没有写工作日志了！"}
res = redis.lpush("notify", json.dumps(value))
def _send_item_with_size(key, data, size):
redis.lpush(key, data)
redis.ltrim(key, 0, size - 1)
def store_most_recently_added_certificates(cert):
redis = _get_redis()
if redis:
redis.lpush('last-certificates', cert.get_pem())
redis.ltrim('last-certificates', 0, 10)
def access_domain(domain, port):
redis = _get_redis()
if redis:
redis.lpush('last-domains', '%s:%d' % (domain, port))
redis.ltrim('last-domains', 0, 10)
redis.setex('%s:%d' % (domain, port), '', 60 * 10)
# Data Type : String Value
redis.set("name", "zedo")
print(redis.get("name"))
# Data Type : Integer Value
redis.set("counter", 1)
print(redis.get("counter")) # 1
redis.incr("counter")
print(redis.get("counter")) # 2
redis.decr("counter")
print(redis.get("counter")) #1
# List : possible to duplicate values
redis.rpush("members", "r1")
redis.rpush("members", "r2")
redis.lpush("members", "l1")
redis.lpush("members", "l2")
print(redis.lrange("members", 0, 0))
print(redis.lrange("members", 0, 1))
print(redis.lrange("members", 0, 2))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
print(redis.lindex("members",3))
print(redis.rpop("members"))
print(redis.lpop("members"))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
redis.delete("members")
#Sets : impossible to duplicate values
redis.sadd("members", "s1")
def add(email, json):
note = json['note']
redis.lpush(redis_book_key_from_email(email), note)
return {'result': 'added'}
def monday():
cursor.execute("SELECT id, iphone_token FROM users WHERE iphone_token <> '' GROUP BY iphone_token")
rows = cursor.fetchall()
for row in rows:
value = {"iphone_token": row["iphone_token"], "message": "云秘书提醒您：新的一周开始了，写下本周的工作计划吧！"}
res = redis.lpush("notify", json.dumps(value))
def push(self, element):
"""Push an element"""
redis.lpush(self.key, element)
log.debug('Pushed ** %s ** for key ** %s **' % (element, self.key))
def push_back(self, element):
"""Push an element to the back of the deque"""
redis.lpush(self.key, element)
log.debug('Pushed ** %s ** for key ** %s **' % (element, self.key))
import redis
from kazoo.client import KazooClient
task = {
"start_url": "http://www.hhu.edu.cn",
"allowed_domain": "hhu.edu.cn"
}
redis = redis.Redis()
redis.lpush("task:spider", {
"url": "http://www.hhu.edu.cn",
"life": 5
})
zk = KazooClient(hosts="127.0.0.1")
zk.start()
zk.create("/jetsearch/job", str(task))
zk.stop()
def evening():
cursor.execute("SELECT id, iphone_token FROM users WHERE iphone_token <> '' GROUP BY iphone_token")
rows = cursor.fetchall()
for row in rows:
value = {"iphone_token": row["iphone_token"], "message": "云秘书提醒您：快下班了哦，总结下今天的工作吧！"}
res = redis.lpush("notify", json.dumps(value))
channel = settings.channel
stream_host = settings.stream_host
def bad (term):
bad = len(term) <= 1
bad = bad or re.search("\D", term) is None
bad = bad or re.search(settings.exclude_regex, term) is not None
return bad
if __name__ == '__main__':
stream = redis.StrictRedis(host=stream_host, port=6379, db=0)
redis = redis.StrictRedis(host="localhost", port=6379, db=0)
ps = stream.pubsub()
ps.subscribe([channel])
print "Subcribed to ", channel
for item in ps.listen():
if item['type'] == 'message':
i = item['data'].strip()
# print "Received ", item['data']
if not bad(i):
items = redis.lrange(channel, 0, -1)
if not i in items:
print "Added ", i
redis.ltrim(channel, 0, 20)
redis.lpush(channel, i)
else :
print "Ignored", i
def morning():
cursor.execute("SELECT id, iphone_token FROM users WHERE iphone_token <> '' GROUP BY iphone_token")
rows = cursor.fetchall()
for row in rows:
value = {"iphone_token": row["iphone_token"], "message": "云秘书提醒您：一天工作开始了，先写点你将要做什么吧！"}
res = redis.lpush("notify", json.dumps(value))
