def __init__(self,type,number,tablename,offset):
mycache = cache()
if mycache.get("mapoffset") == None :
mycache.put("mapoffset",0)
logging.info("inside ljlkjljljjljjklk")
else:
logging.info("inside dkashdkashdkashd")
mapoffset = mycache.get("mapoffset")
mapoffset = int(mapoffset)+10
mycache.replace("mapoffset",mapoffset)
self.offset = mapoffset
logging.info("this is called"+str(self.offset))
self.number_items = number
self.tablename = tablename
self.type = type
query ="select * from "+self.tablename+" where dataset_name = '"+self.type+"'"
logging.info(query)
self.query = query
self.maps=db.GqlQuery(self.query)
logging.info("fetching "+ str(self.number_items) + " starting from "+ str(self.offset))
results=self.maps.fetch(self.number_items,self.offset)
retval = self.populate(results)
self.maplist = retval
if not mycache.get("mapcount"):
mycache.put("mapcount",len(self.maplist))
else:
mycache.replace("mapcount",len(self.maplist))
def getStatus(self):
cc = cache()
admin_object = cc.get("admin_data")
if admin_object == None:
return None
else:
return admin_object.currentStatus
def post(self):
resultType = self.request.get('resultType')
clientCookie = self.request.cookies.get('BigKahuna', '')
result = self.request.get('result')
id= self.request.get('id')
reduceQueue = reducequeue()
reduceQueue.put(id,result)
if resultType=="map":
#remove from maptoken manager
mtm = maptokenmanager()
mtm.removeMapToken(clientCookie)
logging.info("Map result received" + resultType + ":" + result)
else:
mycache = cache()
curresult =mycache.get("reduceresult")
if curresult == None:
if result.isdigit()==True:
mycache.put("reduceresult",result)
else:
try:
logging.info(" Reduceresult is not null")
#if result.isdigit()==True and int(result)!=0:
curresult = int(curresult)
curresult = int(result) + curresult
mycache.replace("reduceresult",curresult)
except:
logging.info("exception in ResultManager.py ")
logging.info("Reduce result received:" + result)
def resultcount(self):
self.mycache = cache()
resultlist =self. mycache.get("resultlist")
if resultlist== None:
return 0
else:
return len(resultlist)
def resultobject(self):
#returns latest result
self.mycache = cache()
reslist = self.mycache.get("reduceresult")
if reslist == None:
return "0"
else:
return reslist
def removeMapToken(self, inToken):
mycache = cache()
dic = mycache.get("maptokenmanager")
if not dic:
logging.error("maptokenmanager not found in removeMapToken")
if dic.has_key(inToken):
dic.pop(inToken)
mycache.replace("maptokenmanager", dic)
def getWorkflowType(self):
mycache = cache()
if not mycache.get("settings"):
settingsInCache = SettingsInCache()
else:
settingsInCache = mycache.get("settings")
workflow_type = settingsInCache.type_of_workflow
return workflow_type
def _setRequestSession(self):
''' creates or sets the Brain requests Session object '''
if isinstance(bconfig.request_session, type(None)):
if not cache:
bconfig.request_session = requests.Session()
else:
bconfig.request_session = cache(requests.Session())
self.session = bconfig.request_session
else:
self.session = bconfig.request_session
def get(self):
self.resource_cache = cache()
self.response.out.write("In Resource Manager")
self.resourcelist = self.resource_cache.get("clientlist")
if self.resourcelist == None:
self.response.out.write("Non clients yet")
else:
path = os.path.join(os.path.dirname(__file__), 'templates/resource.html')
dictSettings = {}
dictSettings['resources']=self.resourcelist
self.response.out.write(template.render(path, dictSettings))
def __init__(self):
mycache = cache()
dic = mycache.get("maptokenmanager")
if dic == None:
mapTokenMapping = {}
mycache.put("maptokenmanager", mapTokenMapping)
logging.info("added maptoken manager to cache")
if not mycache.get("settings"):
settingsInCache = SettingsInCache()
else:
settingsInCache = mycache.get("settings")
workflow_type = settingsInCache.type_of_workflow
def putMapToken(self, inMap, inToken):
mycache = cache()
dic = mycache.get("maptokenmanager")
if dic == None:
logging.error("maptokenmanager not found in putMapToken")
if dic.has_key(inToken):
existingClient = dic.pop(inToken)
self.requeueForToken(existingClient.map)
dic[inToken] = inMap
else:
newClient = clientTokenMap()
newClient.token = inToken
newClient.map = inMap
dic[inToken] = newClient
mycache.replace("maptokenmanager", dic)
def get(self):
mycache = cache()
if not mycache.get("settings"):
settingsInCache = SettingsInCache()
settingsInCache = self.createSettings(settingsInCache)
mycache.put("settings",settingsInCache)
self.redirect("/settings")
else:
settingsInCache = mycache.get("settings")
settingsInCache = self.createSettings(settingsInCache)
mycache.replace("settings",settingsInCache)
path = os.path.join(os.path.dirname(__file__), 'templates/settings.html')
dictSettings = {}
dictSettings['dataset_name'] = settingsInCache.dataset_name
dictSettings['no_items'] = settingsInCache.no_of_items_in_cache
dictSettings['workflow_type'] = settingsInCache.type_of_workflow
logging.error(dictSettings['dataset_name'])
self.response.out.write(template.render(path, dictSettings))
def __init__(self,type,number,tablename,offset):
self.offset = offset
self.number_items = number
self.tablename = tablename
self.type = type
query ="select * from "self.tablename+" where dataset_name = '"+self.type+"'"
self.query = query
self.maps=db.GqlQuery(self.query)
results=self.maps.fetch(self.number_items,self.offset)
retval = self.populate(results)
self.maplist = maps
mycache = cache()
mycache.put("mapcount",len(maplist))
if not mycache.get("mapoffset"):
mycache.put("mapoffset",0)
else:
mapoffset = mycache.get("mapoffset")
mapoffset = mapoffset+1000
mycache.put("mapoffset",mapoffset)
def post(self):
admin = AdminData()
mycache = cache()
if not mycache.get("settings"):
settingsInCache = SettingsInCache()
else:
settingsInCache = mycache.get("settings")
workflow_type = settingsInCache.type_of_workflow
startapp = self.request.get('startapp')
if startapp!= None:
if workflow_type=="gears":
admin.changeStatus("dataload")
else:
admin.changeStatus("map")
self.response.out.write("<div style='background-color:yellow'> Application Started</div>")
path = os.path.join(os.path.dirname(__file__), 'templates/admin.html')
adminSettings ={}
adminSettings["status"]=admin.get()
self.response.out.write(template.render(path, adminSettings))
def post(self):
clientCookie = self.request.cookies.get('BigKahuna', '')
gearsenabled = self.request.get('gearsenabled')
gears = cache()
templist = gears.get("clientlist")
if templist == None:
self.response.out.write("No clients yet")
else:
logging.info("Inside else")
for obj in templist:
logging.info("Inside for")
if obj.cookie == int(clientCookie):
logging.info("Inside else for if")
obj.gearsenabled = gearsenabled
templist.remove(obj)
templist.append(obj)
gears.replace("clientlist",templist)
self.response.out.write("Replaced list")
self.response.out.write("gears enabled:" + gearsenabled)
def get(self):
self.response.out.write("<h4>Test page for memcache</h4>")
cc = cache()
value1 = 100
key1 = "number"
cc.put(key1, value1)
returnValue = cc.get(key1)
if returnValue == None:
self.response.out.write("return value is null")
else:
self.response.out.write(returnValue)
value2 = []
value2.append("january")
value2.append("february")
value2.append("march")
key2 = "array"
cc.put(key2, value2)
returnValue = cc.get(key2)
if returnValue == None:
self.response.out.write("return value is null")
else:
self.response.out.write(returnValue)
def currentmapcount(self):
self.mycache = cache()
offset = self.mycache.get("mapoffset")
pos = self.mycache.get("mappos")
return pos+offset
def __init__(self, number):
self.mycache = cache()
def __init__(self):
self.reduce_cache = cache()
self.reduce_cache.put("resultlist", self.reducelist)
def __init__(self,type):
self.mycache = cache()
self.offset = 0
self.type = type
start()
def __init__(self):
self.mycache = cache()
def __init__(self):
self.reduce_cache = cache()
if not self.reduce_cache.get("resultlist"):
self.reduce_cache.put("resultlist",self.reducelist)
def checkCookie(self,request,response):
clientlist = []
cc = cache()
templist = cc.get("clientlist")
if templist == None:
cc.put("clientlist",clientlist)
logging.info("created client list")
clientCookie = request.cookies.get('BigKahuna', '')
""" create a unique key value pair to track max value of cookie set """
cookieKey = "cookieKey"
cookieValue = cc.get(cookieKey)
""" If that key value pair doesnt exist, create one and set it to zero """
if cookieValue == None:
cc.put(cookieKey,0)
cookieValue = 0
logging.info("created cookie counter")
""" If client cookie isnt set, increment max value by one and set it as the cookie in the response object """
if clientCookie == "":
resource = Resource()
resource.ip = request.remote_addr
resource.clienttype = os.environ['HTTP_USER_AGENT']
resource.cookie = cc.incr(cookieKey)
logging.info("created resource object")
clientlist = cc.get("clientlist")
clientlist.append(resource)
cc.replace("clientlist",clientlist)
logging.info("replaced client list")
response.headers.add_header('Set-Cookie', 'BigKahuna='+str(resource.cookie)+'; expires=Fri, 31-Dec-2020 23:59:59 GMT')
response.out.write('Hello, webapp World!')
response.out.write("Cookie:" + str(resource.cookie))
response.out.write("IP:" + resource.ip)
response.out.write("IP:" + resource.clienttype)
else:
response.out.write("You have already visited us once")
tlist = cc.get("clientlist")
response.out.write("Cookie:" + clientCookie)
response.out.write("IP:" + request.remote_addr)
response.out.write("IP:" + os.environ['HTTP_USER_AGENT'])
resource = Resource()
resource.ip = request.remote_addr
resource.clienttype = os.environ['HTTP_USER_AGENT']
resource.cookie = clientCookie
tlist.append(resource)
cc.replace("clientlist",tlist)
for obj in tlist:
if obj == "":
if obj.cookie == int(clientCookie):
logging,info("Details already in cache")
else:
logging.info("Inside for in client")
obj = resource
tlist.remove(obj)
tlist.append(obj)
cc.replace("clientlist",tlist)
logging.info("replaced client list")
def datstoremapcount(self):
self.mycache = cache()
count = self.mycache.get("mapcount")
return count
def initialize(self):
if self.getStatus() != "inactive" :
currentStatus = "initialized"
cc = cache()
cc.put("admin_data",self)
def changeStatus(self,newStatus):
self.currentStatus = newStatus
cc = cache()
cc.replace("admin_data",self)
