def cleanDeletedItemsByPrefix(prefix): items = rs.keys(prefix+"*") for item in items: title = item.partition(prefix)[2] if item_deleted(title.decode('utf-8')): logger.info('check deleted: %s, %s' % (prefix, title)) rs.delete(item)
def cleanDeletedItemsByPrefix(prefix): items = rs.keys(prefix + "*") for item in items: title = item.partition(prefix)[2] if item_deleted(title.decode('utf-8')): logger.info('check deleted: %s, %s' % (prefix, title)) rs.delete(item)
def verifyingKeyExpired(verifyingKey): expire_time = rs.zscore(VERIFYING_SET, verifyingKey) if not expire_time: logger.info("verifyingKey not in verifying keys set") return False if expire_time < time.time(): return True return False
def __init__(self): super(SendItem, self).__init__() prefix, self.ItemTobeSendKey = getItemTobeSend() if not self.ItemTobeSendKey: setattr(self, 'ItemTobeSend', None) logger.info("No item to be send") return None self.ItemTobeSend = self.ItemTobeSendKey.partition(prefix)[2] self.getWeiboApi()
def updateRoutine(self): try: self.fetchNewItemGenerated(self.lastmins) self.filter_valid() self.storeValidItems() autoVerifyExpiredItems() cleanDeletedItems() except: logger.info(traceback.format_exc())
def filterRedirectedItems(that): url = queryRedirectUrl % that.encode("utf-8") r = requests.post(url) rjson = json.loads(r.text) try: logger.info("redirected item: %s" % rjson['query']['redirects'][0]['from'].encode('utf-8')) return False except KeyError: return True
def storeValidItems(self): for item in self.edited_items: itemKey = EDITED + item rs.set(EDITED+item, item, EDITED_EXPIRE) msg = "插入旧词条更新:%s" % item.encode('utf-8') logger.info(msg) for item in self.new_items: rs.zadd(VERIFYING_SET, item, time.time()+VERIFYING_ZSET_SCORE) msg = "插入新创建词条:%s" % item.encode('utf-8') logger.info(msg)
def sendRoutine(self): if not self.ItemTobeSend: return None try: self.prepare() self.send() logger.info("Sending: "+self.ItemTobeSend+" Succ") except: logger.info(traceback.format_exc()) finally: self.postsend()
def storeValidItems(self): for item in self.edited_items: itemKey = EDITED + item rs.set(EDITED + item, item, EDITED_EXPIRE) msg = "插入旧词条更新:%s" % item.encode('utf-8') logger.info(msg) for item in self.new_items: rs.zadd(VERIFYING_SET, item, time.time() + VERIFYING_ZSET_SCORE) msg = "插入新创建词条:%s" % item.encode('utf-8') logger.info(msg)
def sendRoutine(self): if not self.ItemTobeSend: return None try: self.prepare() self.send() logger.info("Sending: " + self.ItemTobeSend + " Succ") except: logger.info(traceback.format_exc()) finally: self.postsend()
def fetchNewItemGenerated(self, mins): url_t = "http://zh.moegirl.org/api.php?format=json&action=query&list=recentchanges&rcstart=%s&rcend=%s&rcdir=newer&rcnamespace=0&rctoponly&rctype=edit|new" rcstart = calendar.timegm((utcnow() - datetime.timedelta( minutes=mins)).utctimetuple()) rcend = calendar.timegm(utcnow().utctimetuple()) url = url_t % (rcstart, rcend) print url r = requests.post(url) rjson = json.loads(r.text) self.rch = rjson['query']['recentchanges'] self.new_items = [item['title'] for item in self.rch if item['type'] == 'new'] self.edited_items = [item['title'] for item in self.rch if item['type'] != 'new'] msg = ','.join([item['title'].encode('utf-8') for item in self.rch]) if msg: msg = ('过去%d分钟更新的条目有:' % mins)+msg else: msg = "过去%d分钟没有条目更新" % mins logger.info(msg)
def fetchNewItemGenerated(self, mins): url_t = "http://zh.moegirl.org/api.php?format=json&action=query&list=recentchanges&rcstart=%s&rcend=%s&rcdir=newer&rcnamespace=0&rctoponly&rctype=edit|new" rcstart = calendar.timegm( (utcnow() - datetime.timedelta(minutes=mins)).utctimetuple()) rcend = calendar.timegm(utcnow().utctimetuple()) url = url_t % (rcstart, rcend) print url r = requests.post(url) rjson = json.loads(r.text) self.rch = rjson['query']['recentchanges'] self.new_items = [ item['title'] for item in self.rch if item['type'] == 'new' ] self.edited_items = [ item['title'] for item in self.rch if item['type'] != 'new' ] msg = ','.join([item['title'].encode('utf-8') for item in self.rch]) if msg: msg = ('过去%d分钟更新的条目有:' % mins) + msg else: msg = "过去%d分钟没有条目更新" % mins logger.info(msg)
def nighttest(): logger.info("nighttest")
def justtest(): logger.info("justtest")
def send(self): try: self.weiboApi.send(self.weibo_content, self.weibo_link, self.img) except APIError as e: if 'expired_token' in e: logger.info("Sina Weibo Token已到期,请重新授权")