def cleanDeletedItemsByPrefix(prefix):
items = rs.keys(prefix+"*")
for item in items:
title = item.partition(prefix)[2]
if item_deleted(title.decode('utf-8')):
logger.info('check deleted: %s, %s' % (prefix, title))
rs.delete(item)
def cleanDeletedItemsByPrefix(prefix):
items = rs.keys(prefix + "*")
for item in items:
title = item.partition(prefix)[2]
if item_deleted(title.decode('utf-8')):
logger.info('check deleted: %s, %s' % (prefix, title))
rs.delete(item)
def verifyingKeyExpired(verifyingKey):
expire_time = rs.zscore(VERIFYING_SET, verifyingKey)
if not expire_time:
logger.info("verifyingKey not in verifying keys set")
return False
if expire_time < time.time():
return True
return False
def __init__(self):
super(SendItem, self).__init__()
prefix, self.ItemTobeSendKey = getItemTobeSend()
if not self.ItemTobeSendKey:
setattr(self, 'ItemTobeSend', None)
logger.info("No item to be send")
return None
self.ItemTobeSend = self.ItemTobeSendKey.partition(prefix)[2]
self.getWeiboApi()
def updateRoutine(self):
try:
self.fetchNewItemGenerated(self.lastmins)
self.filter_valid()
self.storeValidItems()
autoVerifyExpiredItems()
cleanDeletedItems()
except:
logger.info(traceback.format_exc())
def __init__(self):
super(SendItem, self).__init__()
prefix, self.ItemTobeSendKey = getItemTobeSend()
if not self.ItemTobeSendKey:
setattr(self, 'ItemTobeSend', None)
logger.info("No item to be send")
return None
self.ItemTobeSend = self.ItemTobeSendKey.partition(prefix)[2]
self.getWeiboApi()
def updateRoutine(self):
try:
self.fetchNewItemGenerated(self.lastmins)
self.filter_valid()
self.storeValidItems()
autoVerifyExpiredItems()
cleanDeletedItems()
except:
logger.info(traceback.format_exc())
def filterRedirectedItems(that):
url = queryRedirectUrl % that.encode("utf-8")
r = requests.post(url)
rjson = json.loads(r.text)
try:
logger.info("redirected item: %s" % rjson['query']['redirects'][0]['from'].encode('utf-8'))
return False
except KeyError:
return True
def verifyingKeyExpired(verifyingKey):
expire_time = rs.zscore(VERIFYING_SET, verifyingKey)
if not expire_time:
logger.info("verifyingKey not in verifying keys set")
return False
if expire_time < time.time():
return True
return False
def filterRedirectedItems(that):
url = queryRedirectUrl % that.encode("utf-8")
r = requests.post(url)
rjson = json.loads(r.text)
try:
logger.info("redirected item: %s" %
rjson['query']['redirects'][0]['from'].encode('utf-8'))
return False
except KeyError:
return True
def storeValidItems(self):
for item in self.edited_items:
itemKey = EDITED + item
rs.set(EDITED+item, item, EDITED_EXPIRE)
msg = "插入旧词条更新：%s" % item.encode('utf-8')
logger.info(msg)
for item in self.new_items:
rs.zadd(VERIFYING_SET, item, time.time()+VERIFYING_ZSET_SCORE)
msg = "插入新创建词条：%s" % item.encode('utf-8')
logger.info(msg)
def sendRoutine(self):
if not self.ItemTobeSend:
return None
try:
self.prepare()
self.send()
logger.info("Sending: "+self.ItemTobeSend+" Succ")
except:
logger.info(traceback.format_exc())
finally:
self.postsend()
def storeValidItems(self):
for item in self.edited_items:
itemKey = EDITED + item
rs.set(EDITED + item, item, EDITED_EXPIRE)
msg = "插入旧词条更新：%s" % item.encode('utf-8')
logger.info(msg)
for item in self.new_items:
rs.zadd(VERIFYING_SET, item, time.time() + VERIFYING_ZSET_SCORE)
msg = "插入新创建词条：%s" % item.encode('utf-8')
logger.info(msg)
def sendRoutine(self):
if not self.ItemTobeSend:
return None
try:
self.prepare()
self.send()
logger.info("Sending: " + self.ItemTobeSend + " Succ")
except:
logger.info(traceback.format_exc())
finally:
self.postsend()
def fetchNewItemGenerated(self, mins):
url_t = "http://zh.moegirl.org/api.php?format=json&action=query&list=recentchanges&rcstart=%s&rcend=%s&rcdir=newer&rcnamespace=0&rctoponly&rctype=edit|new"
rcstart = calendar.timegm((utcnow() - datetime.timedelta(
minutes=mins)).utctimetuple())
rcend = calendar.timegm(utcnow().utctimetuple())
url = url_t % (rcstart, rcend)
print url
r = requests.post(url)
rjson = json.loads(r.text)
self.rch = rjson['query']['recentchanges']
self.new_items = [item['title'] for item in self.rch if item['type']
== 'new']
self.edited_items = [item['title'] for item in self.rch if item['type']
!= 'new']
msg = '，'.join([item['title'].encode('utf-8') for item in self.rch])
if msg:
msg = ('过去%d分钟更新的条目有：' % mins)+msg
else:
msg = "过去%d分钟没有条目更新" % mins
logger.info(msg)
def fetchNewItemGenerated(self, mins):
url_t = "http://zh.moegirl.org/api.php?format=json&action=query&list=recentchanges&rcstart=%s&rcend=%s&rcdir=newer&rcnamespace=0&rctoponly&rctype=edit|new"
rcstart = calendar.timegm(
(utcnow() - datetime.timedelta(minutes=mins)).utctimetuple())
rcend = calendar.timegm(utcnow().utctimetuple())
url = url_t % (rcstart, rcend)
print url
r = requests.post(url)
rjson = json.loads(r.text)
self.rch = rjson['query']['recentchanges']
self.new_items = [
item['title'] for item in self.rch if item['type'] == 'new'
]
self.edited_items = [
item['title'] for item in self.rch if item['type'] != 'new'
]
msg = '，'.join([item['title'].encode('utf-8') for item in self.rch])
if msg:
msg = ('过去%d分钟更新的条目有：' % mins) + msg
else:
msg = "过去%d分钟没有条目更新" % mins
logger.info(msg)
def nighttest():
logger.info("nighttest")
def justtest():
logger.info("justtest")
def send(self):
try:
self.weiboApi.send(self.weibo_content, self.weibo_link, self.img)
except APIError as e:
if 'expired_token' in e:
logger.info("Sina Weibo Token已到期，请重新授权")
def send(self):
try:
self.weiboApi.send(self.weibo_content, self.weibo_link, self.img)
except APIError as e:
if 'expired_token' in e:
logger.info("Sina Weibo Token已到期，请重新授权")
