TODAY=date.today() DAY=TODAY.day MONTH=TODAY.month YEAR=TODAY.year HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) RSET=db.logSystem.find_one({'table':'populate_image'}) send_list=[] TRENDING_LIST_QUERY=db.prodtrend.find({u'd':DAY,u'm':MONTH,u'y':YEAR}).sort('Hits',-1).limit(150) syslog.syslog('memcache-hour-trending: count: '+str(TRENDING_LIST_QUERY.count())) print TRENDING_LIST_QUERY.count() COUNTER=0 for p in TRENDING_LIST_QUERY: rec={'title':p['title'],'place':p['place'],'Hits':p['Hits'],'linktitle':p['linktitle'],'id':p['id']} send_list.append(rec) COUNTER+=1 wikilib.GenInfoPage(p['id']) syslog.syslog('memcache-hour-trending: '+str(COUNTER)+' pages appended succesfully.') syslog.syslog('memcache-hour-trending: '+str(len(send_list))+' items in send_list.') wikilib.fnSetMemcache('HOURKEY',send_list,60*60*3) wikilib.fnLaunchNextJob('set_trending')
syslog.syslog('calling daily pages image script....') for d in DAILYPAGERESULTS['values']: rec={'d':d,'m':MONTH,'y':YEAR,'stry':str(YEAR),'strm':str(MONTH),'strd':str(d)} send_list.append(rec) wikilib.fnSetMemcache('mcdpDaysList'+str(MONTH)+str(YEAR),send_list,60*60*24) QUERY={'d':DAY,'m':MONTH,'y':YEAR} send_list=[] RESULTSET=db[thCN].find(QUERY).sort('place',1).limit(100) syslog.syslog('memcache-daily: '+str(QUERY)+' count: '+str(RESULTSET.count())) for row in RESULTSET: wikilib.GenInfoPage(row['id']) title,utitle=wikilib.fnFormatName(row['title']) rec={'place':row['place'],'Hits':row['Hits'],'title':utitle ,'id':str(row['id']),'linktitle':title.encode('utf-8')} send_list.append(rec) wikilib.fnSetMemcache('DAYKEY',send_list,7200) notedate='' notes='' latest_hits_list = db[thCN].find(QUERY).sort('place',1).limit(100) syslog.syslog('memcache-latest: '+str(QUERY)+' count: '+str(latest_hits_list.count())) for p in latest_hits_list: wikilib.GenInfoPage(p['id']) title,utitle=wikilib.fnFormatName(p['title']) rec={'title':utitle,'place':p['place'],'Hits':p['Hits'],'linktitle':title.encode('utf-8'),'notedate':notedate,'notes':notes,'id':p['id']} send_list.append(rec) wikilib.fnSetMemcache('send_list',send_list,3600) wikilib.fnLaunchNextJob('set_day')
YEAR=TODAY.year HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) SEARCH_HOUR='%02d' % (int(HOUR),) print SEARCH_HOUR HOURQUERY=db.hitshourlydaily.find({str(SEARCH_HOUR):{'$gt':1}}).sort(str(SEARCH_HOUR),-1).limit(50) send_list=[] place=1 syslog.syslog('memcache-hourly: '+' count: '+str(HOURQUERY.count())) for row in HOURQUERY: title,utitle=wikilib.fnFindName(row['_id']) wikilib.GenInfoPage(row['_id']) rec={'place':place,'Hits':row[str(SEARCH_HOUR)],'title':title ,'id':str(row['_id']),'linktitle':title} place+=1 send_list.append(rec) print len(send_list) wikilib.fnSetMemcache('HOURKEY',send_list,60*60*3) wikilib.fnLaunchNextJob('set_lasthour')
DAY=TODAY.day MONTH=TODAY.month YEAR=TODAY.year HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) print 'debuts query...' send_list=[] title='' utitle='' QUERY=db[dbCN].find({u'd':DAY,u'm':MONTH,u'y':YEAR}).sort('place',1).limit(100) syslog.syslog('memcache-debuts: count: '+str(QUERY.count())) for item in QUERY: COUNT=0 TITLE='' wikilib.GenInfoPage(item['id']) title,utitle=wikilib.fnFormatName(item['title']) rec={'title':utitle,'place':item['place'],'Hits':item['Hits'],'linktitle':item['linktitle'],'id':item['id']} send_list.append(rec) wikilib.fnSetMemcache('DEBUTS_ARTICLES',send_list,60*60) wikilib.fnLaunchNextJob('set_debuts')
HOUR=wikilib.fnGetHour() MONTHNAME=datetime.datetime.now().strftime("%B") thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=datetime.datetime.now().strftime('%H') HOUR=wikilib.fnMinusHour(int(HOUR)) RSET=db.logSystem.find_one({'table':'populate_image'}) print 'random query...' send_list=[] for a in range(1,50): place=random.randint(1,250000) FQUERY={u'd':DAY,u'm':MONTH,u'y':YEAR,'place':place} RANDOM_LIST_QUERY=db[thCN].find(FQUERY) for item in RANDOM_LIST_QUERY: id=item['id'] wikilib.GenInfoPage(item['id']) title,utitle=wikilib.fnFormatName(item['title']) rec={'title':utitle,'place':item['place'],'Hits':item['Hits'],'linktitle':title.encode('utf-8'),'id':item['id']} send_list.append(rec) wikilib.fnSetMemcache('RANDOM_ARTICLES',send_list,60*60) wikilib.fnLaunchNextJob('set_random')
import datetime import syslog conn=Connection('10.164.95.114') db=conn.wc mc=memcache.Client(['127.0.0.1:11211'],debug=0) HOUR=wikilib.fnGetHour() DAY,MONTH,YEAR=wikilib.fnGetDate() MONTHNAME=wikilib.fnGetMonthName() thCN='tophits'+str(YEAR)+MONTHNAME dbCN='proddebuts'+str(YEAR)+str(MONTHNAME) HOUR=wikilib.fnMinusHour(int(HOUR)) wikilib.fnOpenSitemap() syslog.syslog('starting memcache_set_cold query') send_list=[] COLD_LIST_QUERY=db.prodcold.find().sort('Hits',1).limit(100) syslog.syslog('memcache-cold: COLD_LIST_QUERY count: '+str(COLD_LIST_QUERY.count())) print COLD_LIST_QUERY.count() a=0 for p in COLD_LIST_QUERY: a+=1 rec={'title':p['title'],'place':p['place'],'Hits':p['Hits'],'linktitle':p['linktitle'],'id':p['id']} send_list.append(rec) wikilib.GenInfoPage(p['id']) syslog.syslog('memcache-cold: total of '+str(a)+' records processed') wikilib.fnSetMemcache('COLD_LIST_QUERY',send_list,60*60*2) syslog.syslog('memcache-cold: done!') wikilib.fnLaunchNextJob('set_cold')