TODAY=date.today() DAY=TODAY.day MONTH=TODAY.month YEAR=TODAY.year HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) RSET=db.logSystem.find_one({'table':'populate_image'}) send_list=[] TRENDING_LIST_QUERY=db.prodtrend.find({u'd':DAY,u'm':MONTH,u'y':YEAR}).sort('Hits',-1).limit(150) syslog.syslog('memcache-hour-trending: count: '+str(TRENDING_LIST_QUERY.count())) print TRENDING_LIST_QUERY.count() COUNTER=0 for p in TRENDING_LIST_QUERY: rec={'title':p['title'],'place':p['place'],'Hits':p['Hits'],'linktitle':p['linktitle'],'id':p['id']} send_list.append(rec) COUNTER+=1 wikilib.GenInfoPage(p['id']) syslog.syslog('memcache-hour-trending: '+str(COUNTER)+' pages appended succesfully.') syslog.syslog('memcache-hour-trending: '+str(len(send_list))+' items in send_list.') wikilib.fnSetMemcache('HOURKEY',send_list,60*60*3) wikilib.fnLaunchNextJob('set_trending')
thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) DAILYPAGERESULTS=db.command({'distinct':thCN,'key':'d','query':{'m':int(MONTH)}}) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) RSET=db.logSystem.find_one({'table':'populate_image'}) send_list=[] syslog.syslog('calling daily pages image script....') for d in DAILYPAGERESULTS['values']: rec={'d':d,'m':MONTH,'y':YEAR,'stry':str(YEAR),'strm':str(MONTH),'strd':str(d)} send_list.append(rec) wikilib.fnSetMemcache('mcdpDaysList'+str(MONTH)+str(YEAR),send_list,60*60*24) QUERY={'d':DAY,'m':MONTH,'y':YEAR} send_list=[] RESULTSET=db[thCN].find(QUERY).sort('place',1).limit(100) syslog.syslog('memcache-daily: '+str(QUERY)+' count: '+str(RESULTSET.count())) for row in RESULTSET: wikilib.GenInfoPage(row['id']) title,utitle=wikilib.fnFormatName(row['title']) rec={'place':row['place'],'Hits':row['Hits'],'title':utitle ,'id':str(row['id']),'linktitle':title.encode('utf-8')} send_list.append(rec) wikilib.fnSetMemcache('DAYKEY',send_list,7200) notedate='' notes=''
DAY=TODAY.day MONTH=TODAY.month YEAR=TODAY.year HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) print 'debuts query...' send_list=[] title='' utitle='' QUERY=db[dbCN].find({u'd':DAY,u'm':MONTH,u'y':YEAR}).sort('place',1).limit(100) syslog.syslog('memcache-debuts: count: '+str(QUERY.count())) for item in QUERY: COUNT=0 TITLE='' wikilib.GenInfoPage(item['id']) title,utitle=wikilib.fnFormatName(item['title']) rec={'title':utitle,'place':item['place'],'Hits':item['Hits'],'linktitle':item['linktitle'],'id':item['id']} send_list.append(rec) wikilib.fnSetMemcache('DEBUTS_ARTICLES',send_list,60*60) wikilib.fnLaunchNextJob('set_debuts')
HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) RSET=db.logSystem.find_one({'table':'populate_image'}) print 'random query...' send_list=[] for a in range(1,50): place=random.randint(1,250000) FQUERY={u'd':DAY,u'm':MONTH,u'y':YEAR,'place':place} RANDOM_LIST_QUERY=db[thCN].find(FQUERY) for item in RANDOM_LIST_QUERY: id=item['id'] wikilib.GenInfoPage(item['id']) title,utitle=wikilib.fnFormatName(item['title']) rec={'title':utitle,'place':item['place'],'Hits':item['Hits'],'linktitle':title.encode('utf-8'),'id':item['id']} send_list.append(rec) wikilib.fnSetMemcache('RANDOM_ARTICLES',send_list,60*60) wikilib.fnLaunchNextJob('set_random')
import datetime import syslog conn=Connection('10.164.95.114') db=conn.wc mc=memcache.Client(['127.0.0.1:11211'],debug=0) HOUR=wikilib.fnGetHour() DAY,MONTH,YEAR=wikilib.fnGetDate() MONTHNAME=wikilib.fnGetMonthName() thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=wikilib.fnMinusHour(int(HOUR)) wikilib.fnOpenSitemap() syslog.syslog('starting memcache_set_cold query') send_list=[] COLD_LIST_QUERY=db.prodcold.find().sort('Hits',1).limit(100) syslog.syslog('memcache-cold: COLD_LIST_QUERY count: '+str(COLD_LIST_QUERY.count())) print COLD_LIST_QUERY.count() a=0 for p in COLD_LIST_QUERY: a+=1 rec={'title':p['title'],'place':p['place'],'Hits':p['Hits'],'linktitle':p['linktitle'],'id':p['id']} send_list.append(rec) wikilib.GenInfoPage(p['id']) syslog.syslog('memcache-cold: total of '+str(a)+' records processed') wikilib.fnSetMemcache('COLD_LIST_QUERY',send_list,60*60*2) syslog.syslog('memcache-cold: done!') wikilib.fnLaunchNextJob('set_cold')
send_list=[] thCN='tophits'+str(YEAR)+MONTHNAME PASTTABLES=['tophits2013February'] for COLLECTION in PASTTABLES: RESULTSET=db.command({'distinct':COLLECTION,'key':'d'}) if COLLECTION=='tophits': MONTH=1 YEAR=2013 elif COLLECTION=='tophits2013February': MONTH=2 YEAR=2013 for d in RESULTSET['values']: rec={'d':d,'m':MONTH,'y':YEAR,'stry':str(YEAR),'strm':str(MONTH),'strd':str(d)} QUERY={'d':int(d),'m':int(MONTH),'y':int(YEAR)} DAYKEY='toplist'+str(YEAR)+str(MONTH)+str(d) page_list=[] PAGERESULTSET=db[COLLECTION].find(QUERY).sort('place',1).limit(100) syslog.syslog('mc-archives: '+str(DAYKEY)+' '+str(QUERY)+' count: '+str(PAGERESULTSET.count())) for row in PAGERESULTSET: title, utitle=wikilib.fnFindName(row['id']) prec={'place':row['place'],'Hits':row['Hits'],'title':title ,'id':str(row['id']),'linktitle':utitle} page_list.append(prec) wikilib.GenInfoPage(row['id']) syslog.syslog('mc-archives: Now setting mc key '+str(DAYKEY)+' of length '+str(len(page_list))) wikilib.fnSetMemcache(DAYKEY,page_list,60*60) send_list.append(rec) mc.set('mcdpDaysList'+str(MONTH)+str(YEAR),send_list,60*60*24*60)