def main(): for i in ormiter(Follow): print i.from_id, i.to_id for i in ormiter(SiteRecHistory): print i.zsite_id, i.user_id, i.state
def zsite_show_update(): for i in ormiter(ZsiteList, 'owner_id=0'): i.rank = zsite_rank_get(i.zsite_id) i.save() for cid in (CID_SITE, CID_USER): mc_flush(OWNER_ID, cid)
def rss_tag(): from model.rss import RSS_PO_ID_STATE_NOTAG, CID_USER, RssPoId, RSS_PO_ID_STATE_AUTOTAG from zweb.orm import ormiter from zdata.idf.tfidf import tag_id_rank_list_by_txt, ID2NAME from model.po import Po from operator import itemgetter from model.po_tag_user import tag2idlist_po_user, rss_po_new from zkit.algorithm.unique import unique for rss_po_id in ormiter( RssPoId, 'user_cid=%s and state=%s' % (CID_USER, RSS_PO_ID_STATE_NOTAG)): po = Po.mc_get(rss_po_id.po_id) if not po: continue #print po.name_ txt = '%s\n%s' % (po.name_, po.txt) tag_id_rank_list = tag_id_rank_list_by_txt(txt)[:7] tag_id_list = map(itemgetter(0), tag_id_rank_list) user_tag_id_list = map( int, tag2idlist_po_user.tag_id_list_by_id(po.user_id)) id_list = user_tag_id_list[:] id_list.extend(tag_id_list) rss_po_id.tag_id_list = ' '.join(map(str, unique(id_list))) #for i in tag_id_list: # i.append_id_tag_id_list #print i.tag_id_list #raise rss_po_id.state = RSS_PO_ID_STATE_AUTOTAG rss_po_id.save() rss_po_new(po, user_tag_id_list)
def main(): from zweb.orm import ormiter exist = set() for i in ormiter(DoubanFeedOwner): topic_id = None user_id = None feed = DoubanFeed.get(i.id) if feed.cid == CID_DOUBAN_FEED_TOPIC: group_url = feed.topic_id or i.topic group = DoubanGroup.by_url(group_url) if not group: if not group_url in exist: exist.add(group_url) yield parse_group_htm, "http://www.douban.com/group/%s/" % group_url else: topic_id = group user_id = feed.user_id or i.owner if not (user_id and str(user_id).isdigit()): user_id = DoubanUser.by_url(user_id) if topic_id is not None and user_id: feed.topic_id = topic_id feed.user_id = user_id feed.save() i.delete()
def main(): from zweb.orm import ormiter exist = set() for i in ormiter(DoubanFeedOwner): topic_id = None user_id = None feed = DoubanFeed.get(i.id) if feed.cid == CID_DOUBAN_FEED_TOPIC: group_url = feed.topic_id or i.topic group = DoubanGroup.by_url(group_url) if not group: if not group_url in exist: exist.add(group_url) yield parse_group_htm, 'http://www.douban.com/group/%s/' % group_url else: topic_id = group user_id = feed.user_id or i.owner if not (user_id and str(user_id).isdigit()): user_id = DoubanUser.by_url(user_id) if topic_id is not None and user_id: feed.topic_id = topic_id feed.user_id = user_id feed.save() i.delete()
def audio_compress(): id = kv_int.get(KV_PO_AUDIO) for i in ormiter(Po, 'cid=%s and id>%s'%( CID_AUDIO, id )): id = i.id input_filename = fs_file_audio(id) if not exists(input_filename): continue output_filename = '/tmp/po.audio.%s'%SITE_DOMAIN #if not isdir(output_filename): # os.mkdir(output_filename) subprocess.call([ 'lame', '--quiet', '--mp3input', '--abr', '64', input_filename, output_filename ]) if not exists(output_filename): continue shutil.move(output_filename, input_filename) kv_int.set(KV_PO_AUDIO, id)
def can_rec_site_id_list(): result = [] for i in ormiter(Zsite, 'cid=%s' % CID_SITE): zsite_id = i.id count = po_cid_count_by_zsite_id(zsite_id, CID_NOTE) if count > 5: result.append(zsite_id) return set(result)
def zsite_rank_rebase(): n = kv_int.get(KV_ZSITE_RANK_POWER) or 100 if n > 5000: kv_int.set(KV_ZSITE_RANK_POWER, 100) ZsiteRank.raw_sql('update zsite_rank set value=value*100/%s', n) for i in ormiter(ZsiteRank): zsite_rank.mc_flush(i.id) mc_zsite_rank_max.delete('')
def url_list(): for i in ormiter(DoubanUser): id = i.id if DoubanFetched.get(id): continue yield douban_recommendation, URL_REC % id, 1 f = DoubanFetched.get_or_create(id=id) f.save()
def zsite_rank_update(days): n = kv_int.get(KV_ZSITE_RANK_POWER) or 100 for i in ormiter(ZsiteUvDaily, 'days=%s' % days): zsite_id = i.zsite_id zsite_rank_update_by_zsite_id(zsite_id, i.uv * n) mc_zsite_rank_max.delete('') kv_int.set(KV_ZSITE_RANK_POWER, n * 1.1)
def ping_po(begin): if begin < 10075567: begin = 10075567 po = None for po in ormiter(Po, 'id>%s and state=%s and cid=%s' % (begin, STATE_ACTIVE, CID_NOTE)): ping_all(po) if po: return po.id
def _sync_recommend(begin_id): for i in ormiter(Feed, 'id>%s and cid=%s' % (begin_id, CID_REC)): begin_id = i.id try: sync_recommend(i.zsite_id, i.id) except: print_exc() return begin_id
def zsite_rank_update(days): n = kv_int.get(KV_ZSITE_RANK_POWER) or 100 for i in ormiter(ZsiteUvDaily, 'days=%s' % days): zsite_id = i.zsite_id zsite_rank_update_by_zsite_id(zsite_id, i.uv*n) mc_zsite_rank_max.delete('') kv_int.set(KV_ZSITE_RANK_POWER, n*1.1)
def can_rec_site_id_list(): result = [] for i in ormiter(Zsite, 'cid=%s'%CID_SITE): zsite_id = i.id count = po_cid_count_by_zsite_id(zsite_id, CID_NOTE) if count > 5: result.append(zsite_id) return set(result)
def url_list(): for i in ormiter(DoubanUser): id = i.id if DoubanFetched.get(id): continue yield douban_recommendation, URL_REC%id, 1 f = DoubanFetched.get_or_create(id=id) f.save()
def buzz_follow_new(from_id, to_id): if not Buzz.where( from_id=from_id, to_id=to_id, cid=CID_BUZZ_FOLLOW, rid=to_id).count(): buzz_new(from_id, to_id, CID_BUZZ_FOLLOW, to_id) for i in ormiter(Follow, 'to_id=%s and from_id!=%s' % (from_id, to_id)): buzz_new(from_id, i.from_id, CID_BUZZ_FOLLOW, to_id)
def search_zsite_keyword_iter(): for i in ormiter(SearchZsite): id = i.id i.delete() zsite = Zsite.mc_get(id) if zsite and zsite.state: kw = zsite2keyword(zsite) if kw: yield zsite_keyword(zsite, kw)
def duplicator_set_by_user_id(user_id): if not user_id: return for i in ormiter(Po, 'user_id=%s and cid=%s'%(user_id, CID_NOTE)): title_txt = '%s\n%s'%(i.name_, i.txt) if duplicator_rss.txt_is_duplicate(title_txt): #print i.name_ continue duplicator_rss.set_record(title_txt, i.id)
def _sync_join_event(begin_id): for i in ormiter(EventJoiner, 'id>%s and state>%s' % (begin_id, EVENT_JOIN_STATE_NO)): begin_id = i.id try: sync_join_event(i.user_id, i.event_id) except: print_exc() return begin_id
def duplicator_set_by_user_id(user_id): if not user_id: return for i in ormiter(Po, 'user_id=%s and cid=%s' % (user_id, CID_NOTE)): title_txt = '%s\n%s' % (i.name_, i.txt) if duplicator_rss.txt_is_duplicate(title_txt): #print i.name_ continue duplicator_rss.set_record(title_txt, i.id)
def douban_feed_to_review_iter(): from zweb.orm import ormiter for i in ormiter( DoubanFeed, 'state = %s'%DOUBAN_FEED_STATE_TO_REIVEW ): i.state = DOUBAN_FEED_STATE_REVIEWED i.save() yield i
def ping_po(begin): if begin < 10075567: begin = 10075567 po = None for po in ormiter( Po, 'id>%s and state=%s and cid=%s' % (begin, STATE_ACTIVE, CID_NOTE)): ping_all(po) if po: return po.id
def feed2po_new(): from zweb.orm import ormiter for feed in ormiter( FeedImport, 'state>%s and state<%s'%( FEED_IMPORT_STATE_INIT, FEED_IMPORT_STATE_POED ) ): feed_new(feed)
def _sync_po(begin_id): for po in ormiter( Po, 'id>%s and state=%s and cid in (%s,%s,%s, %s,%s,%s)' % (begin_id, STATE_ACTIVE, CID_WORD, CID_NOTE, CID_EVENT, CID_AUDIO, CID_VIDEO, CID_PHOTO)): begin_id = po.id try: sync_po(po) except: print_exc() return begin_id
def event_weekly(begin): mail = weekly_mail_pop() event_list = Event.where(state=EVENT_STATE_BEGIN).where('id>%s', begin) if event_list: last_id = event_list[-1].id event_li = event_city_list(event_list) for i in ormiter(Zsite, 'cid=%s and state>=%s' % (CID_USER, ZSITE_STATE_ACTIVE)): event_weekly_mail(i, event_li, mail) #print i.id #sys.stdout.flush() time.sleep(0.01) return last_id
def _sync_recommend(begin_id): for i in ormiter(Feed, 'id>%s and cid=%s'%( begin_id,CID_REC ) ): begin_id = i.id try: sync_recommend(i.zsite_id, i.id) except: print_exc() return begin_id
def _sync_join_event(begin_id): for i in ormiter( EventJoiner, 'id>%s and state>%s'%( begin_id, EVENT_JOIN_STATE_NO ) ): begin_id = i.id try: sync_join_event(i.user_id, i.event_id) except: print_exc() return begin_id
def _sync_po(begin_id): for po in ormiter( Po, 'id>%s and state=%s and cid in (%s,%s,%s, %s,%s,%s)' % ( begin_id, STATE_ACTIVE, CID_WORD, CID_NOTE, CID_EVENT, CID_AUDIO, CID_VIDEO, CID_PHOTO ) ): begin_id = po.id try: sync_po(po) except: print_exc() return begin_id
def main(): for vote in ormiter(Feed, 'rid !=0'): recommend = po_new(CID_REC, vote.zsite_id, '', state=STATE_ACTIVE, rid=vote.rid, id=vote.id) if recommend: mc_po_recommend_id_by_rid_user_id.set( '%s_%s' % (vote.rid, vote.zsite_id), recommend.id) vote.cid = CID_REC vote.rid = 0 vote.save()
def zsite_tag_id_mv(zsite_id, from_tag_id, to_tag_id=1): tag = ZsiteTag.get(zsite_id=zsite_id, tag_id=from_tag_id) for i in ormiter(ZsiteTagPo, 'zsite_tag_id=%s' % tag.id): i.zsite_tag_id = to_tag_id i.save() po_id = i.po_id mc_tag_by_po_id.delete('%s_%s' % (zsite_id, po_id)) #print "delete zsite", zsite_id, from_tag_id ZsiteTag.where(zsite_id=zsite_id, tag_id=from_tag_id).delete() mc_flush_zsite_id(zsite_id) mc_flush_zsite_tag_id(from_tag_id) mc_flush_zsite_tag_id(to_tag_id)
def event_weekly(begin): mail = weekly_mail_pop() event_list = Event.where(state=EVENT_STATE_BEGIN).where('id>%s', begin) if event_list: last_id = event_list[-1].id event_li = event_city_list(event_list) for i in ormiter( Zsite, 'cid=%s and state>=%s' % (CID_USER, ZSITE_STATE_ACTIVE)): event_weekly_mail(i, event_li, mail) #print i.id #sys.stdout.flush() time.sleep(0.01) return last_id
def zsite_tag_id_mv(zsite_id, from_tag_id, to_tag_id=1): tag = ZsiteTag.get(zsite_id=zsite_id, tag_id=from_tag_id) for i in ormiter(ZsiteTagPo, 'zsite_tag_id=%s'%tag.id): i.zsite_tag_id = to_tag_id i.save() po_id = i.po_id mc_tag_by_po_id.delete('%s_%s'%(zsite_id, po_id)) #print "delete zsite", zsite_id, from_tag_id ZsiteTag.where(zsite_id=zsite_id, tag_id=from_tag_id).delete() mc_flush_zsite_id(zsite_id) mc_flush_zsite_tag_id(from_tag_id) mc_flush_zsite_tag_id(to_tag_id)
def main(): for vote in ormiter(Feed,'rid !=0'): recommend = po_new( CID_REC, vote.zsite_id, '', state=STATE_ACTIVE, rid=vote.rid, id=vote.id ) if recommend: mc_po_recommend_id_by_rid_user_id.set( '%s_%s'%(vote.rid, vote.zsite_id), recommend.id ) vote.cid =CID_REC vote.rid = 0 vote.save()
def rss_tag(): from model.rss import RSS_PO_ID_STATE_NOTAG, CID_USER, RssPoId, RSS_PO_ID_STATE_AUTOTAG from zweb.orm import ormiter from zdata.idf.tfidf import tag_id_rank_list_by_txt, ID2NAME from model.po import Po from operator import itemgetter from model.po_tag_user import tag2idlist_po_user, rss_po_new from zkit.algorithm.unique import unique for rss_po_id in ormiter( RssPoId, 'user_cid=%s and state=%s'%(CID_USER, RSS_PO_ID_STATE_NOTAG) ): po = Po.mc_get(rss_po_id.po_id) if not po: continue #print po.name_ txt = '%s\n%s'%(po.name_, po.txt) tag_id_rank_list = tag_id_rank_list_by_txt(txt)[:7] tag_id_list = map(itemgetter(0), tag_id_rank_list) user_tag_id_list = map( int, tag2idlist_po_user.tag_id_list_by_id( po.user_id ) ) id_list = user_tag_id_list[:] id_list.extend(tag_id_list) rss_po_id.tag_id_list = ' '.join( map(str, unique(id_list)) ) #for i in tag_id_list: # i.append_id_tag_id_list #print i.tag_id_list #raise rss_po_id.state = RSS_PO_ID_STATE_AUTOTAG rss_po_id.save() rss_po_new(po, user_tag_id_list)
def user_id_site_can_rec(): id_list = can_rec_site_id_list() for i in ormiter(Zsite, 'cid=%s'%CID_USER): user_id = i.id if TOP_REC_CID_SITE_REC&top_rec(user_id) and SiteRec.get(user_id): continue fav_list = list(zsite_id_list_by_user_id(user_id)) fav_list.extend( SiteRecHistory.where(user_id=user_id).col_list(col='zsite_id') ) fav_site_id_set = set(fav_list) can_rec_id_list = id_list - fav_site_id_set if not can_rec_id_list: continue yield user_id, can_rec_id_list
def main(): for i in ormiter(FeedImport, "state=%s" % FEED_IMPORT_STATE_WITHOUT_TAG): txt = "%s\n%s" % (i.title, i.txt) tag_id_rank_list = tag_id_rank_list_by_txt(txt)[:7] tag_id_list = map(itemgetter(0), tag_id_rank_list) i.tag_id_list = " ".join(map(str, tag_id_list)) i.state = FEED_IMPORT_STATE_INIT i.save() tag_admin_new(i.id, tag_id_list, i.rank) print i.id, i.title, i.url for k, v in tag_id_rank_list: print k, v, for j in ID2NAME[k]: print j, print "" print ">>>" * 7
def main(): for i in ormiter(FeedImport, "state=%s"%FEED_IMPORT_STATE_WITHOUT_TAG): txt = "%s\n%s"%( i.title, i.txt ) tag_id_rank_list = tag_id_rank_list_by_txt(txt)[:7] tag_id_list = map(itemgetter(0), tag_id_rank_list) i.tag_id_list = " ".join(map(str,tag_id_list)) i.state = FEED_IMPORT_STATE_INIT i.save() tag_admin_new(i.id, tag_id_list, i.rank) print i.id, i.title, i.url for k, v in tag_id_rank_list: print k, v, for j in ID2NAME[k]: print j, print "" print ">>>"*7
def zsite_verify_ajust(zsite): if zsite.cid != CID_USER: return need = 0 id = zsite.id name = zsite.name if name.isdigit() or not name: need |= ZSITE_VERIFY_NEED_NAME if not ico.get(id): need |= ZSITE_VERIFY_NEED_ICO if not any(career_current(id)): need |= ZSITE_VERIFY_NEED_CAREER if need: zsite_verify_rm(zsite) return need else: zsite_verify_new(zsite) if __name__ == '__main__': from zweb.orm import ormiter for zsite in ormiter( Zsite, 'state>=%s and state<%s and cid=%s' % (ZSITE_STATE_ACTIVE, ZSITE_STATE_VERIFY, CID_USER)): print zsite.id, zsite_verify_ajust(zsite)
return id, access_token @mc_oauth_access_token_verify('{access_token}') def oauth_access_token_verify(access_token): id, token = id_binary_decode(access_token) o = OauthAccessToken.get(id) if o and o.token == token: return o.user_id return 0 def oauth_authorize_code_rm(id): oauth_authorize_code.delete(id) def oauth_authorization_code_verify(authorization_code): id, code = id_binary_decode(authorization_code) t = oauth_authorize_code.get(id) if t and t == code: return id return 0 if __name__ == '__main__': from zweb.orm import ormiter for i in ormiter(OauthClient): print i if oauth_client_uri.get(i.id): print i i.cid = 1 i.save()
#!/usr/bin/env python # -*- coding: utf-8 -*- import _env from zweb.orm import ormiter from model.buzz import Buzz from model.kv_misc import kv_int from model.cid import CID_BUZZ_FOLLOW, CID_BUZZ_EVENT_JOIN, CID_BUZZ_SITE_NEW, CID_BUZZ_SITE_FAV, CID_BUZZ_SYS, CID_BUZZ_EVENT_FEEDBACK_JOINER, CID_BUZZ_EVENT_FEEDBACK_OWNER cid_set = ( CID_BUZZ_FOLLOW, CID_BUZZ_EVENT_JOIN, CID_BUZZ_SITE_NEW, CID_BUZZ_SITE_FAV, CID_BUZZ_SYS, CID_BUZZ_EVENT_FEEDBACK_JOINER, CID_BUZZ_EVENT_FEEDBACK_OWNER, ) for i in ormiter(Buzz): if i.cid not in cid_set: i.delete() if i.id % 1009 == 1: print i.id KV_SHOW_BUZZ_POS = 1 # 加入Show的非重要通知 kv_int.delete(KV_SHOW_BUZZ_POS)
need = 0 id = zsite.id name = zsite.name if name.isdigit() or not name: need |= ZSITE_VERIFY_NEED_NAME if not ico.get(id): need |= ZSITE_VERIFY_NEED_ICO if not any(career_current(id)): need |= ZSITE_VERIFY_NEED_CAREER if need: zsite_verify_rm(zsite) return need else: zsite_verify_new(zsite) if __name__ == '__main__': from zweb.orm import ormiter for zsite in ormiter( Zsite, 'state>=%s and state<%s and cid=%s'%( ZSITE_STATE_ACTIVE, ZSITE_STATE_VERIFY, CID_USER ) ): print zsite.id, zsite_verify_ajust(zsite)
if hostname != 'e1': raise 'hostname is not e1' import _env import config import config.host.e1 config.host.e1.prepare(config) config.default.finish(config) if config.MYSQL_HOST != "127.0.0.1": raise from zweb.orm import ormiter from model.vps import Vps for i in ormiter(Vps): #print i.passwd i.passwd = '' i.save() from model.user_info import UserInfo for i in ormiter(UserInfo): i.birthday = 0 i.save() from model.namecard import Namecard for i in ormiter(Namecard): i.mail = "" i.phone = "" i
rech[0].state = state rech[0].save() else: SiteRecHistory( user_id=user_id, zsite_id=zsite_id, state=state ).save() id_list = SiteRecNew.get(user_id).split() if zsite_id in id_list: id_list.remove(zsite_id) SiteRecNew.set(user_id, ' '.join(map(str,id_list))) top_rec_unmark(user_id, TOP_REC_CID_SITE_REC) def site_rec_set(user_id, site_id): SiteRecNew.set(user_id, ' '.join([str(i) for i in site_id])) top_rec_mark(user_id, TOP_REC_CID_SITE_REC) def site_rec_dump(user_id): zsite_list = site_rec(user_id) return zsite_json(user_id, zsite_list) if __name__ == '__main__': from model.zsite_url import id_by_url jid = id_by_url('jandan') from zweb.orm import ormiter for i in ormiter(SiteRecHistory): print i.id
def main(): for i in ormiter(Spammer): print "spaming",i.id spammer_new(i.id)
def sitemap_po(): from model.po import Po for i in ormiter(Po): yield 'http:%s' % (i.link)
rech[0].state = state rech[0].save() else: SiteRecHistory(user_id=user_id, zsite_id=zsite_id, state=state).save() id_list = SiteRecNew.get(user_id).split() if zsite_id in id_list: id_list.remove(zsite_id) SiteRecNew.set(user_id, ' '.join(map(str, id_list))) top_rec_unmark(user_id, TOP_REC_CID_SITE_REC) def site_rec_set(user_id, site_id): SiteRecNew.set(user_id, ' '.join([str(i) for i in site_id])) top_rec_mark(user_id, TOP_REC_CID_SITE_REC) def site_rec_dump(user_id): zsite_list = site_rec(user_id) return zsite_json(user_id, zsite_list) if __name__ == '__main__': from model.zsite_url import id_by_url jid = id_by_url('jandan') from zweb.orm import ormiter for i in ormiter(SiteRecHistory): print i.id
def sitemap_po(): from model.po import Po for i in ormiter(Po): yield 'http:%s'%( i.link )
for i in po_list: pos = po_pos_get_last_reply_id(user_id, i.id) new_reply_id_list = [] for reply_id in i.reply_id_list(): if reply_id > pos: new_reply_id_list.append(reply_id) user_id_list = [] for reply in Reply.mc_get_list(reversed(new_reply_id_list)): user_id_list.append(reply.user_id) po_user_id.append(user_id_list) return buzz_po_bind_user(po_list, po_user_id, user_id) if __name__ == '__main__': pass user_id = 10031395 from zweb.orm import ormiter from model.po import Po from model.po_pos import PoPos from po_pos import PoPos, STATE_BUZZ for po in ormiter(Po): po_id = po.id print po.id reply_id_last = po.reply_id_last for user_id in PoPos.where(po_id=po_id, state=STATE_BUZZ).where('pos>=%s', reply_id_last).col_list(col='user_id'): BuzzReply.where(po_id=po_id, user_id=user_id, state=BUZZ_REPLY_STATE_SHOW).update(state=BUZZ_REPLY_STATE_HIDE) mc_flush(user_id)
def sitemap_zsite(): from model.zsite import Zsite for i in ormiter(Zsite): yield 'http:%s' % (i.link)
def sitemap_zsite(): from model.zsite import Zsite for i in ormiter(Zsite): yield 'http:%s'%( i.link )
@mc_oauth_access_token_verify('{access_token}') def oauth_access_token_verify(access_token): id, token = id_binary_decode(access_token) o = OauthAccessToken.get(id) if o and o.token == token: return o.user_id return 0 def oauth_authorize_code_rm(id): oauth_authorize_code.delete(id) def oauth_authorization_code_verify(authorization_code): id, code = id_binary_decode(authorization_code) t = oauth_authorize_code.get(id) if t and t == code: return id return 0 if __name__ == '__main__': from zweb.orm import ormiter for i in ormiter(OauthClient): print i if oauth_client_uri.get(i.id): print i i.cid = 1 i.save()
def _event_end(end): end = end + 8*60 for i in ormiter(Event, 'state=%s and end_time<=%s' % (EVENT_STATE_NOW, end)): event_end(i)