def primeDesireCache(step, startKey = None, stopKey = None):
import princeFunc
if stopKey is None and startKey is None:
startKey = 1
stopKey = int(meSchema.tradeCue.all(keys_only=True).order('-__key__').get().name())
elif stopKey is None or startKey is None:
raise(BaseException('Must define both startKey and stopKey, or both must be None!'))
memdict = {}
clockKeyStep = step
queryStr = princeFunc.getDesireQueryStr(max(step-405,0),step)
desires = db.GqlQuery(queryStr).fetch(20000)
for desire in desires:
desirekey = desire.key().name()
stckID = meTools.getStckID(desire.Symbol)
cueKey = desirekey.split("_")[-2] # Extract cueKey from middle.
memkey = 'desire_' + cueKey + '_' + str(stckID)
step = int(desirekey.split("_")[0]) # Extract step from front part of desirekey.
if not memdict.__contains__(memkey):
memdict[memkey] = step
elif memdict[memkey] < step:
memdict[memkey] = step
memcache.set_multi(memdict)
cachepy.set_multi(memdict, priority = 1)
cachepy.set('stepclock_' + str(startKey) + '_' + str(stopKey), clockKeyStep) # Manually syncing stepclock until get saner method.
def memGetPercentReturns(memkeylist, prefix):
EntityDict = {}
newMemEntities = {}
memCacheEntities = {}
memEntities = cachepy.get_multi(memkeylist)
missingKeys = meTools.getMissingKeys(memkeylist,memEntities)
memCacheEntities = memcache.get_multi(missingKeys)
cachepy.set_multi(memCacheEntities)
missingKeys = meTools.getMissingKeys(missingKeys,memCacheEntities)
memEntities.update(memCacheEntities)
if missingKeys:
missingKeys = [key.replace(prefix,'') for key in missingKeys]
if prefix == 'BTR-':
Entities = meSchema.backTestResult.get_by_key_name(missingKeys)
elif prefix == 'LAR-':
Entities = meSchema.liveAlg.get_by_key_name(missingKeys)
for entity in Entities:
if entity:
memkey = prefix + entity.key().name()
pReturn = entity.percentReturn
newMemEntities[memkey] = pReturn
memEntities[memkey] = pReturn
memcache.set_multi(newMemEntities)
cachepy.set_multi(newMemEntities)
return memEntities
def syncProcessCache(step,startKey,stopKey):
clockKey = 'stepclock_' + str(startKey) + '_' + str(stopKey)
stepclock = cachepy.get(clockKey, priority=1)
memkeylist = []
if stepclock not in [step-1, step]:
for i in range(startKey,stopKey + 1):
for stckID in [1,2,3,4]:
cuekey = meTools.buildTradeCueKey(i)
recency_key = 'desire_' + cuekey + '_' + str(stckID)
memkeylist.append(recency_key)
recentDesires = memcache.get_multi(memkeylist)
cachepy.set_multi(recentDesires, priority = 1)
'''
for des in recentDesires:
cachepy.set(des, recentDesires[des], priority=1)
'''
cachepy.set(clockKey,step,priority=1) # Set cachepy clockKey to current step since synced with Memcache.
def doCompoundReturns(stopStep, startStep, globalStop, namespace, name, i, cursor, model, JobID, callback, totalBatches, taskname, deadline = None):
from time import time
if not deadline:
deadline = time() + 540.00
entityModel = getattr(meSchema, model)
if entityModel == meSchema.backTestResult:
prefix = 'BTR-'
elif entityModel == meSchema.liveAlg:
prefix = 'LAR-'
else:
raise BaseException('Model must be backTestResult or liveAlg!')
count = 100
while count == 100:
query = entityModel.all().filter('stopStep =', stopStep).filter('startStep =', startStep).order('percentReturn')
if cursor != '':
query.with_cursor(cursor)
if deadline > time():
i += 1
entities = query.fetch(100)
count = len(entities)
memEntities = {}
for entity in entities:
memkey = prefix + entity.key().name()
memEntities[memkey] = entity.percentReturn
memcache.set_multi(memEntities)
cachepy.set_multi(memEntities)
doCompounds(stopStep, startStep, entities)
cursor = query.cursor()
else:
subTaskname = 'RVals-' + JobID + '-' + model + '-' + str(stopStep) + '-' + str(startStep) + '-' + str(i)
meTools.taskAdd(subTaskname, '/calculate/compounds/calculateCompounds', 'default', 0.5,
stopStep = stopStep, startStep = startStep, globalStop = globalStop, name = name,
i = i, cursor = cursor, model = model, JobID = JobID, callback = callback, totalBatches = totalBatches,
taskname = taskname)
return
if stopStep <= globalStop - 400:
stopStep += 400
startStep += 400
doCompoundReturns(stopStep,startStep,globalStop,namespace,name,0,'',model, JobID, callback, totalBatches, taskname, deadline)
elif callback:
meTools.taskAdd('callback-' + taskname, callback, 'default', 0.5,
JobID = JobID, taskname = taskname, totalBatches = totalBatches,
model = model, jobtype = 'callback', stepType = 'calculateRvals')
