LANGLIST= wikicount.getLanguageList() for lang in LANGLIST: PLACEMAP=str(lang)+"_mapPlace" HITSMAP=str(lang)+"_mapHits" try: IFILE=open("/home/ec2-user/"+str(lang)+"_mongo.csv.sorted","r") except IOError: syslog.syslog("Error opening file for "+str(lang)) continue RESULT=[] RECCOUNT=0 for line in IFILE: if RECCOUNT < 1000: line=line.strip().split(",") RESULT.append((line[0],line[1])) RECCOUNT+=1 try: db[PLACEMAP].update({'_id':str(line[1])},{"$set":{DAYKEY:RECCOUNT}},upsert=True) db[HITSMAP].update({'_id':str(line[1])},{"$set":{DAYKEY:int(line[0])}},upsert=True) except TypeError: pass else: break IFILE.close() RUNTIME= wikicount.fnEndTimerCalcRuntime(STARTTIME) syslog.syslog('tophits.py: runtime is '+str(RUNTIME)+' seconds.') wikicount.fnSetStatusMsg('tophits',3) wikicount.fnLaunchNextJob('tophits')
line=line.strip('"').split(',') HASH=line[1].replace("\"","") rec=(line[0],HASH) SORTME.append(rec) IFILE.close() pconn,cconn=Pipe() lyst=[] p=Process(target=sorting.QuickSortMPListArray,args=(SORTME,cconn,n)) p.start() lyst=pconn.recv() p.join() OFILE=open("/home/ec2-user/"+str(lang)+"_mongo.csv.sorted","w") for a in lyst: OFILE.write(str(a[0])+','+a[1]) OFILE.close() wikicount.fnSetStatusMsg('sortMongoHD',3) wikicount.fnLaunchNextJob('sortMongoHD')
delta=item['Hits']-YHits print 'hello!' nameq=db.hitsdaily.find({'_id':item['_id']}) NEWPOST={'id':item['_id'],'delta':int(delta),'orPlace':item['place'],'title':nameq['title']} db.tmpHot.insert(NEWPOST) return STARTTIME= wikicount.fnStartTimer() wikicount.toSyslog('filltmpHot.py : starting...') DAY,MONTH,YEAR,HOUR,expiretime= wikicount.fnReturnTimes() DAYKEY=str(YEAR)+'_'+str(MONTH)+'_'+str(DAY) COLLECTIONNAME=str('tophits')+DAYKEY conn=Connection() db=conn.wc RECCOUNT=1 NUMRECS=250 wikicount.fnSetStatusMsg('fillTmpHot',0) print COLLECTIONNAME db.tmpHot.remove() RESULT=db[COLLECTIONNAME].find() #RESULT1=db[COLLECTIONNAME].find().limit(NUMRECS).skip(0) #RESULT2=db[COLLECTIONNAME].find().limit(NUMRECS).skip(NUMRECS) #RESULT3=db[COLLECTIONNAME].find().limit(NUMRECS).skip(NUMRECS*2) #RESULT4=db[COLLECTIONNAME].find().limit(NUMRECS).skip(NUMRECS*3) FillTmpHot(RESULT) RUNTIME= wikicount.fnEndTimerCalcRuntime(STARTTIME) wikicount.toSyslog('prepop_filltmpHot.py: runtime is '+str(RUNTIME)+' seconds.') wikicount.fnSetStatusMsg('fillTmpHot',3) wikicount.fnLaunchNextJob('fillTmpHot')