コード例 #1
0
ファイル: movers.py プロジェクト: rob-berkes/wikicount-dbproc
def p98_todaysmovers():
	DAY,MONTH,YEAR,HOUR,expiretime= wikicount.fnReturnTimes()
	YDAY=DAY
	YMONTH=MONTH
	if DAY==1:
		YMONTH-=1
		YDAY=30
	else:
		YDAY-=1
	STARTTIME= wikicount.fnStartTimer()
	STRINGDATE=str(YEAR)+"_"+"%02d" % (MONTH,)+"_"+"%02d" % (DAY,)
	YSTRINGDATE=str(YEAR)+"_"+"%02d" % (YMONTH,)+"_"+"%02d" % (YDAY,)
	print STRINGDATE, YSTRINGDATE
	conn=Connection()
	db=conn.wc
	LANGUAGES= wikicount.getLanguageList()
	for lang in LANGUAGES:
		RECEVALS=0
		RECEXCEPTS=0
		DBOPS=0	
		MOVERS=[]
		hdTABLE=str(lang)+"_hitsdaily"	
		outTABLE=str(lang)+"_threehour"
		print hdTABLE
		DBOPS+=1
		RECSET=db[hdTABLE].find({STRINGDATE:{"$gt":10}})
		print RECSET.count()
		for RS in RECSET:
			try:
				rec=(RS[STRINGDATE]-RS[YSTRINGDATE],RS['_id'],RS['title'])
				MOVERS.append(rec)
				RECEVALS+=1
			except KeyError:
				rec=(RS[STRINGDATE],RS['_id'],RS['title'])
				MOVERS.append(rec)
				RECEVALS+=1
			except:
				RECEXCEPTS+=1
				continue
		SMOVERS=sorting.QuickSortListArray(MOVERS)
		db[outTABLE].remove()
		for a in range(1,100):
			try:
		                rec={'place':a,'title':SMOVERS[-a][2],'rollavg':SMOVERS[-a][0],'id':SMOVERS[-a][1]}
				db[outTABLE].insert(rec)
				DBOPS+=1
			except:
				continue

		syslog.syslog("[movers.py][main] "+str(lang)+" language done. DBOPS: "+str(DBOPS)+" Records sorted: "+str(len(SMOVERS))+" out of "+str(RECEVALS)+" initial recs.  Exceptions encountered: "+str(RECEXCEPTS))
	return
コード例 #2
0
from pymongo import Connection
from lib import wikicount

DAY, MONTH, YEAR,HOUR, expiretime= wikicount.fnReturnTimes()
conn=Connection()
db=conn.wc
LANGUAGES= wikicount.getLanguageList()
for lang in LANGUAGES:
	HHNAME=str(lang)+'_hitsdaily'
	QUERYA=db[HHNAME].update({'2013_09_12':{'$exists':True}},{'$set':{'2013_09_12':1}},False,True)
コード例 #3
0
#	p.start()
#	print 'main proc started'

#	lyst=pconn.recv()
#	print 'starting out write'
#	print 'joining child procs'
#	p.join()
#	OFILE=open("/home/ec2-user/mongo.csv.sorted","w")
#	for a in lyst:
#		OFILE.write(str(a[0])+','+a[1])
#	OFILE.close()
#	print 'all done!'
#	RUNTIME=wikicount.fnEndTimerCalcRuntime(STARTTIME)
#	syslog.syslog('sortMongoHD.csv: runtime '+str(RUNTIME)+' seconds.')

	LANGLIST= wikicount.getLanguageList()
	for lang in LANGLIST:
		IFILE=open("/home/ec2-user/"+str(lang)+"_mongo.csv","r")
		SORTME=[]
		for line in IFILE:
			line=line.strip('"').split(',')
			HASH=line[1].replace("\"","")
			rec=(line[0],HASH)
			SORTME.append(rec)
		IFILE.close()
		pconn,cconn=Pipe()
		lyst=[]
		p=Process(target=sorting.QuickSortMPListArray,args=(SORTME,cconn,n))
		p.start()
		lyst=pconn.recv()
		p.join()
コード例 #4
0
import os

from lib import wikicount

LANG= wikicount.getLanguageList()
FOLDERS=['action','category','image','ondeck','staging']
for lang in LANG:
	for folder in FOLDERS:
		NEWPATH="/tmp/"+str(lang)+"_"+str(folder)
		if not os.path.exists(NEWPATH):
			os.makedirs(NEWPATH)