def p98_todaysmovers():
DAY,MONTH,YEAR,HOUR,expiretime= wikicount.fnReturnTimes()
YDAY=DAY
YMONTH=MONTH
if DAY==1:
YMONTH-=1
YDAY=30
else:
YDAY-=1
STARTTIME= wikicount.fnStartTimer()
STRINGDATE=str(YEAR)+"_"+"%02d" % (MONTH,)+"_"+"%02d" % (DAY,)
YSTRINGDATE=str(YEAR)+"_"+"%02d" % (YMONTH,)+"_"+"%02d" % (YDAY,)
print STRINGDATE, YSTRINGDATE
conn=Connection()
db=conn.wc
LANGUAGES= wikicount.getLanguageList()
for lang in LANGUAGES:
RECEVALS=0
RECEXCEPTS=0
DBOPS=0
MOVERS=[]
hdTABLE=str(lang)+"_hitsdaily"
outTABLE=str(lang)+"_threehour"
print hdTABLE
DBOPS+=1
RECSET=db[hdTABLE].find({STRINGDATE:{"$gt":10}})
print RECSET.count()
for RS in RECSET:
try:
rec=(RS[STRINGDATE]-RS[YSTRINGDATE],RS['_id'],RS['title'])
MOVERS.append(rec)
RECEVALS+=1
except KeyError:
rec=(RS[STRINGDATE],RS['_id'],RS['title'])
MOVERS.append(rec)
RECEVALS+=1
except:
RECEXCEPTS+=1
continue
SMOVERS=sorting.QuickSortListArray(MOVERS)
db[outTABLE].remove()
for a in range(1,100):
try:
rec={'place':a,'title':SMOVERS[-a][2],'rollavg':SMOVERS[-a][0],'id':SMOVERS[-a][1]}
db[outTABLE].insert(rec)
DBOPS+=1
except:
continue
syslog.syslog("[movers.py][main] "+str(lang)+" language done. DBOPS: "+str(DBOPS)+" Records sorted: "+str(len(SMOVERS))+" out of "+str(RECEVALS)+" initial recs. Exceptions encountered: "+str(RECEXCEPTS))
return
from pymongo import Connection
from lib import wikicount
DAY, MONTH, YEAR,HOUR, expiretime= wikicount.fnReturnTimes()
conn=Connection()
db=conn.wc
LANGUAGES= wikicount.getLanguageList()
for lang in LANGUAGES:
HHNAME=str(lang)+'_hitsdaily'
QUERYA=db[HHNAME].update({'2013_09_12':{'$exists':True}},{'$set':{'2013_09_12':1}},False,True)
# p.start()
# print 'main proc started'
# lyst=pconn.recv()
# print 'starting out write'
# print 'joining child procs'
# p.join()
# OFILE=open("/home/ec2-user/mongo.csv.sorted","w")
# for a in lyst:
# OFILE.write(str(a[0])+','+a[1])
# OFILE.close()
# print 'all done!'
# RUNTIME=wikicount.fnEndTimerCalcRuntime(STARTTIME)
# syslog.syslog('sortMongoHD.csv: runtime '+str(RUNTIME)+' seconds.')
LANGLIST= wikicount.getLanguageList()
for lang in LANGLIST:
IFILE=open("/home/ec2-user/"+str(lang)+"_mongo.csv","r")
SORTME=[]
for line in IFILE:
line=line.strip('"').split(',')
HASH=line[1].replace("\"","")
rec=(line[0],HASH)
SORTME.append(rec)
IFILE.close()
pconn,cconn=Pipe()
lyst=[]
p=Process(target=sorting.QuickSortMPListArray,args=(SORTME,cconn,n))
p.start()
lyst=pconn.recv()
p.join()
import os from lib import wikicount LANG= wikicount.getLanguageList() FOLDERS=['action','category','image','ondeck','staging'] for lang in LANG: for folder in FOLDERS: NEWPATH="/tmp/"+str(lang)+"_"+str(folder) if not os.path.exists(NEWPATH): os.makedirs(NEWPATH)