def __init__(self,type,number,tablename,offset): mycache = cache() if mycache.get("mapoffset") == None : mycache.put("mapoffset",0) logging.info("inside ljlkjljljjljjklk") else: logging.info("inside dkashdkashdkashd") mapoffset = mycache.get("mapoffset") mapoffset = int(mapoffset)+10 mycache.replace("mapoffset",mapoffset) self.offset = mapoffset logging.info("this is called"+str(self.offset)) self.number_items = number self.tablename = tablename self.type = type query ="select * from "+self.tablename+" where dataset_name = '"+self.type+"'" logging.info(query) self.query = query self.maps=db.GqlQuery(self.query) logging.info("fetching "+ str(self.number_items) + " starting from "+ str(self.offset)) results=self.maps.fetch(self.number_items,self.offset) retval = self.populate(results) self.maplist = retval if not mycache.get("mapcount"): mycache.put("mapcount",len(self.maplist)) else: mycache.replace("mapcount",len(self.maplist))
def getStatus(self): cc = cache() admin_object = cc.get("admin_data") if admin_object == None: return None else: return admin_object.currentStatus
def post(self): resultType = self.request.get('resultType') clientCookie = self.request.cookies.get('BigKahuna', '') result = self.request.get('result') id= self.request.get('id') reduceQueue = reducequeue() reduceQueue.put(id,result) if resultType=="map": #remove from maptoken manager mtm = maptokenmanager() mtm.removeMapToken(clientCookie) logging.info("Map result received" + resultType + ":" + result) else: mycache = cache() curresult =mycache.get("reduceresult") if curresult == None: if result.isdigit()==True: mycache.put("reduceresult",result) else: try: logging.info(" Reduceresult is not null") #if result.isdigit()==True and int(result)!=0: curresult = int(curresult) curresult = int(result) + curresult mycache.replace("reduceresult",curresult) except: logging.info("exception in ResultManager.py ") logging.info("Reduce result received:" + result)
def resultcount(self): self.mycache = cache() resultlist =self. mycache.get("resultlist") if resultlist== None: return 0 else: return len(resultlist)
def resultobject(self): #returns latest result self.mycache = cache() reslist = self.mycache.get("reduceresult") if reslist == None: return "0" else: return reslist
def removeMapToken(self, inToken): mycache = cache() dic = mycache.get("maptokenmanager") if not dic: logging.error("maptokenmanager not found in removeMapToken") if dic.has_key(inToken): dic.pop(inToken) mycache.replace("maptokenmanager", dic)
def getWorkflowType(self): mycache = cache() if not mycache.get("settings"): settingsInCache = SettingsInCache() else: settingsInCache = mycache.get("settings") workflow_type = settingsInCache.type_of_workflow return workflow_type
def _setRequestSession(self): ''' creates or sets the Brain requests Session object ''' if isinstance(bconfig.request_session, type(None)): if not cache: bconfig.request_session = requests.Session() else: bconfig.request_session = cache(requests.Session()) self.session = bconfig.request_session else: self.session = bconfig.request_session
def get(self): self.resource_cache = cache() self.response.out.write("In Resource Manager") self.resourcelist = self.resource_cache.get("clientlist") if self.resourcelist == None: self.response.out.write("Non clients yet") else: path = os.path.join(os.path.dirname(__file__), 'templates/resource.html') dictSettings = {} dictSettings['resources']=self.resourcelist self.response.out.write(template.render(path, dictSettings))
def __init__(self): mycache = cache() dic = mycache.get("maptokenmanager") if dic == None: mapTokenMapping = {} mycache.put("maptokenmanager", mapTokenMapping) logging.info("added maptoken manager to cache") if not mycache.get("settings"): settingsInCache = SettingsInCache() else: settingsInCache = mycache.get("settings") workflow_type = settingsInCache.type_of_workflow
def putMapToken(self, inMap, inToken): mycache = cache() dic = mycache.get("maptokenmanager") if dic == None: logging.error("maptokenmanager not found in putMapToken") if dic.has_key(inToken): existingClient = dic.pop(inToken) self.requeueForToken(existingClient.map) dic[inToken] = inMap else: newClient = clientTokenMap() newClient.token = inToken newClient.map = inMap dic[inToken] = newClient mycache.replace("maptokenmanager", dic)
def get(self): mycache = cache() if not mycache.get("settings"): settingsInCache = SettingsInCache() settingsInCache = self.createSettings(settingsInCache) mycache.put("settings",settingsInCache) self.redirect("/settings") else: settingsInCache = mycache.get("settings") settingsInCache = self.createSettings(settingsInCache) mycache.replace("settings",settingsInCache) path = os.path.join(os.path.dirname(__file__), 'templates/settings.html') dictSettings = {} dictSettings['dataset_name'] = settingsInCache.dataset_name dictSettings['no_items'] = settingsInCache.no_of_items_in_cache dictSettings['workflow_type'] = settingsInCache.type_of_workflow logging.error(dictSettings['dataset_name']) self.response.out.write(template.render(path, dictSettings))
def __init__(self,type,number,tablename,offset): self.offset = offset self.number_items = number self.tablename = tablename self.type = type query ="select * from "self.tablename+" where dataset_name = '"+self.type+"'" self.query = query self.maps=db.GqlQuery(self.query) results=self.maps.fetch(self.number_items,self.offset) retval = self.populate(results) self.maplist = maps mycache = cache() mycache.put("mapcount",len(maplist)) if not mycache.get("mapoffset"): mycache.put("mapoffset",0) else: mapoffset = mycache.get("mapoffset") mapoffset = mapoffset+1000 mycache.put("mapoffset",mapoffset)
def post(self): admin = AdminData() mycache = cache() if not mycache.get("settings"): settingsInCache = SettingsInCache() else: settingsInCache = mycache.get("settings") workflow_type = settingsInCache.type_of_workflow startapp = self.request.get('startapp') if startapp!= None: if workflow_type=="gears": admin.changeStatus("dataload") else: admin.changeStatus("map") self.response.out.write("<div style='background-color:yellow'> Application Started</div>") path = os.path.join(os.path.dirname(__file__), 'templates/admin.html') adminSettings ={} adminSettings["status"]=admin.get() self.response.out.write(template.render(path, adminSettings))
def post(self): clientCookie = self.request.cookies.get('BigKahuna', '') gearsenabled = self.request.get('gearsenabled') gears = cache() templist = gears.get("clientlist") if templist == None: self.response.out.write("No clients yet") else: logging.info("Inside else") for obj in templist: logging.info("Inside for") if obj.cookie == int(clientCookie): logging.info("Inside else for if") obj.gearsenabled = gearsenabled templist.remove(obj) templist.append(obj) gears.replace("clientlist",templist) self.response.out.write("Replaced list") self.response.out.write("gears enabled:" + gearsenabled)
def get(self): self.response.out.write("<h4>Test page for memcache</h4>") cc = cache() value1 = 100 key1 = "number" cc.put(key1, value1) returnValue = cc.get(key1) if returnValue == None: self.response.out.write("return value is null") else: self.response.out.write(returnValue) value2 = [] value2.append("january") value2.append("february") value2.append("march") key2 = "array" cc.put(key2, value2) returnValue = cc.get(key2) if returnValue == None: self.response.out.write("return value is null") else: self.response.out.write(returnValue)
def currentmapcount(self): self.mycache = cache() offset = self.mycache.get("mapoffset") pos = self.mycache.get("mappos") return pos+offset
def __init__(self, number): self.mycache = cache()
def __init__(self): self.reduce_cache = cache() self.reduce_cache.put("resultlist", self.reducelist)
def __init__(self,type): self.mycache = cache() self.offset = 0 self.type = type start()
def __init__(self): self.mycache = cache()
def __init__(self): self.reduce_cache = cache() if not self.reduce_cache.get("resultlist"): self.reduce_cache.put("resultlist",self.reducelist)
def checkCookie(self,request,response): clientlist = [] cc = cache() templist = cc.get("clientlist") if templist == None: cc.put("clientlist",clientlist) logging.info("created client list") clientCookie = request.cookies.get('BigKahuna', '') """ create a unique key value pair to track max value of cookie set """ cookieKey = "cookieKey" cookieValue = cc.get(cookieKey) """ If that key value pair doesnt exist, create one and set it to zero """ if cookieValue == None: cc.put(cookieKey,0) cookieValue = 0 logging.info("created cookie counter") """ If client cookie isnt set, increment max value by one and set it as the cookie in the response object """ if clientCookie == "": resource = Resource() resource.ip = request.remote_addr resource.clienttype = os.environ['HTTP_USER_AGENT'] resource.cookie = cc.incr(cookieKey) logging.info("created resource object") clientlist = cc.get("clientlist") clientlist.append(resource) cc.replace("clientlist",clientlist) logging.info("replaced client list") response.headers.add_header('Set-Cookie', 'BigKahuna='+str(resource.cookie)+'; expires=Fri, 31-Dec-2020 23:59:59 GMT') response.out.write('Hello, webapp World!') response.out.write("Cookie:" + str(resource.cookie)) response.out.write("IP:" + resource.ip) response.out.write("IP:" + resource.clienttype) else: response.out.write("You have already visited us once") tlist = cc.get("clientlist") response.out.write("Cookie:" + clientCookie) response.out.write("IP:" + request.remote_addr) response.out.write("IP:" + os.environ['HTTP_USER_AGENT']) resource = Resource() resource.ip = request.remote_addr resource.clienttype = os.environ['HTTP_USER_AGENT'] resource.cookie = clientCookie tlist.append(resource) cc.replace("clientlist",tlist) for obj in tlist: if obj == "": if obj.cookie == int(clientCookie): logging,info("Details already in cache") else: logging.info("Inside for in client") obj = resource tlist.remove(obj) tlist.append(obj) cc.replace("clientlist",tlist) logging.info("replaced client list")
def datstoremapcount(self): self.mycache = cache() count = self.mycache.get("mapcount") return count
def initialize(self): if self.getStatus() != "inactive" : currentStatus = "initialized" cc = cache() cc.put("admin_data",self)
def changeStatus(self,newStatus): self.currentStatus = newStatus cc = cache() cc.replace("admin_data",self)