def get_page_url(date, num, page):
print(date, num, page)
result = requests.get(
'https://pacaio.match.qq.com/openapi/json?key=news:%d&num=%d&page=%d&expIds=&callback=__jp0'
% (date, num, page))
# 得到的返回值被故意加了一层,把这一层切掉才是json的标准格式
j = json.loads(result.text[6:-1])
# 页数超过最后一页后,没有数据,直接跳出返回False停止下一页请求
if j['data'] == None:
return False
# 得到正常返回值后遍历所有'data'节点
for jj in j['data']:
title = jj['title']
url = jj['url']
print(title, url)
# 判断redis是否已经存在该文章标题,如果不存在,将它加进Redis中
if not redis.hexists(hashKey, title):
# 使用Hash是为了防止加入重复数据
# List是起到消息队列的作用,消费者从队列右侧pop即可
redis.hset(hashKey, title, url)
redis.lpush(listKey, title)
# 如果标题已经存在,有可能是以下情况:
# 1. 在之前已经爬过这一天数据了,数据没有更新
# 2. 在之前爬过这一天数据了,数据更新了,但是之前的循环已经把更新的数据搞定了
# 这两种情况中,无论剩下的还是后面页的所有数据都已经存入Redis了,所以直接跳出并且终止下一页的请求
else:
return False
# 本页数据都存成功了,请求下一页
return True
def learn_speaker():
check_post_param('id')
user_id = request.form['id']
filename = user_id + next(tempfile._get_candidate_names()) + '.wav'
request.files['wav_sample'].save(filename)
user_training = None
if redis.hexists(hm_data(user_id), USER_TRAINING):
user_training = pickle.loads(redis.hget(hm_data(user_id), USER_TRAINING))
gmm, mfccs = predict.learn(filename, user_training)
redis.hmset(hm_data(user_id), {USER_TRAINING: pickle.dumps(mfccs), USER_MODEL: pickle.dumps(gmm)})
os.remove(filename)
return str(len(mfccs))
def learn_speaker():
check_post_param('id')
user_id = request.form['id']
filename = user_id + next(tempfile._get_candidate_names()) + '.wav'
request.files['wav_sample'].save(filename)
user_training = None
if redis.hexists(hm_data(user_id), USER_TRAINING):
user_training = pickle.loads(
redis.hget(hm_data(user_id), USER_TRAINING))
gmm, mfccs = predict.learn(filename, user_training)
redis.hmset(hm_data(user_id), {
USER_TRAINING: pickle.dumps(mfccs),
USER_MODEL: pickle.dumps(gmm)
})
os.remove(filename)
return str(len(mfccs))
def writerRedis(self):
self.initialRedisPool()
redis = self.redis
for index in range(len(self.urls)):
name = 'html'
print(self.urls[index])
# 先判断key 是否存在如果存在则跳过
if redis.hexists(name, self.urls[index]):
continue
else:
component = {
'titles': self.titles[index],
'urls': self.urls[index],
'auths': self.auths[index],
'dates': self.dates[index],
'digs': self.digs[index],
'comments': self.comments[index],
'browse': self.browse[index],
'content': self.content[index],
'imgs': self.imgs[index]
}
# component=self.titles[index]+'@'+self.urls[index]+'@'+self.auths[index]+'@'+self.dates[index]+'@'+self.digs[index]+'@'+self.comments[index]+'@'+self.browse[index]+'@'+self.content[index]+'@'+self.imgs[index]
redis.hset(name, self.urls[index], str(component))
print("\n")
# +-------------------------------------------------------------------+
# IS THERE ENTRIES IN REDIS WHICH NO MATCHES WITH BLOCKCHAIN ENTRIES? |
# +-------------------------------------------------------------------+
print("*********************************************************************")
print("Check all entries in Redis to be sure there is no bad entries")
print("*********************************************************************")
#Metablock
for key in all_decoded_metablock_metadata:
validation_blockchain = redis.hget("blocks:" + key, "flag")
if (validation_blockchain.decode('utf-8') == "0"):
continue
else:
already_checked = redis.hexists("blocks:" + key, "audit")
if (already_checked == False):
print("-- blocks:" + key +
" : Not associated with an entry in the blockchain!")
redis.delete("blocks:" + key)
print("---- blocks:" + key + " deleted.")
bad_entries = bad_entries + 1
else:
# This metablock was already checked in the first way blockchain ==> redis
# Delete audit field
redis.hdel("blocks:" + key, "audit")
print("-- blocks:" + key +
" : already checked (Audit field deleted in Redis)")
#Metadoc
for key in all_decoded_metadoc_metadata:
####
file_data = redis.hget("FILES:APP", "eto_site_setup.json")
temp = base64.b64decode(file_data)
eto_site_data = json.loads(temp)
redis.delete("ETO_RESOURCE_A")
for j in eto_site_data:
redis.hset("ETO_RESOURCE_A", j["controller"] + "|" + str(j["pin"]), 0)
keys = redis.hkeys("ETO_RESOURCE")
print "keys", keys
for i in keys:
print "i", i
value = redis.hget("ETO_RESOURCE", i)
if redis.hexists("ETO_RESOURCE_A", i):
redis.hset("ETO_RESOURCE_A", i, value)
redis.delete("ETO_RESOURCE")
redis.rename("ETO_RESOURCE_A", "ETO_RESOURCE")
if redis.hget("CONTROL_VARIABLES", "ETO_RESOURCE_UPDATED") != "TRUE":
redis.hset("CONTROL_VARIABLES", "ETO_RESOURCE_UPDATED", "FALSE")
#
# delete process keys
#
keys = redis.hkeys("WD_DIRECTORY")
for i in keys:
print "i", i
redis.hdel("WD_DIRECTORY", i)
redis.hset("SYS_DICT", "CONTROL_VARIABLES",
####
file_data = redis.hget("FILES:APP", "eto_site_setup.json")
temp = base64.b64decode(file_data)
eto_site_data = json.loads(temp)
redis.delete("ETO_RESOURCE_A")
for j in eto_site_data:
redis.hset("ETO_RESOURCE_A", j["controller"] + "|" + str(j["pin"]), 0)
keys = redis.hkeys("ETO_RESOURCE")
print "keys", keys
for i in keys:
print "i", i
value = redis.hget("ETO_RESOURCE", i)
if redis.hexists("ETO_RESOURCE_A", i):
redis.hset("ETO_RESOURCE_A", i, value)
redis.delete("ETO_RESOURCE")
redis.rename("ETO_RESOURCE_A", "ETO_RESOURCE")
if redis.hget("CONTROL_VARIABLES", "ETO_RESOURCE_UPDATED") != "TRUE":
redis.hset("CONTROL_VARIABLES", "ETO_RESOURCE_UPDATED", "FALSE")
#
# delete process keys
#
keys = redis.hkeys("WD_DIRECTORY")
for i in keys:
print "i", i
redis.hdel("WD_DIRECTORY", i)
redis.hset("SYS_DICT", "CONTROL_VARIABLES", "system control and status variables")
def is_game_already_has(game_seq):
return redis.hexists('_user_game', game_seq)
def Update(key,item,val,redis):
d,lst = Query()
if key in d.keys() and redis.hexists(key,item):
old_val = redis.hget(key,item)
redis.hset(key,item,val)
r.publish(src[0],src[1])
bot = telegram.Bot(token=TELEGRAM_TOKEN)
redis = redis.StrictRedis(host='localhost', port=6379)
last_update = redis.get('bot:last_update')
while True:
if last_update == None:
updates = bot.getUpdates()
else:
updates = bot.getUpdates(offset=last_update)
for u in updates:
user = u.message.from_user.first_name
message = u.message.text
chat_id = u.message.from_user.id
if not redis.hexists('bot:session:'+str(chat_id), 'user'):
#start session
print 'new session!!!'
redis.hset('bot:session:'+str(chat_id), 'user', user)
wall_id = None
#check for commands
words = message.split(' ')
if words[0] == '/start':
print 'COMMAND START'
if len(words) > 1:
wall_name = words[1]
#find wall name
wall_id = wall_name
redis.hset('bot:session:'+str(chat_id), 'wall_id', wall_id)
bot.sendMessage(chat_id=chat_id, text="Welcome to our LED Wall! You can start sending text or images right now to wall " + str(wall_id))