def addTaskRange(initialStopStep, globalStop, unique, namespace, batchSize=5, stepsBack=1600, callback = ''):
'''
Task Ranges will branch out into weekly clumps for all algs.
Remove startAlg, stopAlg from task name and use stepRange value instead.
Must calculate expected count to be found for completion entities.
'''
import meSchema
from google.appengine.api import namespace_manager
from math import ceil
namespace_manager.set_namespace('')
startAlg = 1
stopAlg = int(meSchema.meAlg.all(keys_only=True).order('-__key__').get().name())
''' Pre-calculating number of batches for callback to check against. '''
numWeeks = ((globalStop - initialStopStep)/400) + 1
numAlgs = stopAlg - startAlg + 1
batchesWeek = ceil(numAlgs/float(batchSize))
totalBatches = int(numWeeks*batchesWeek)
JobID = meTools.buildJobID(namespace, unique, globalStop, initialStopStep, stepsBack)
''' Probably need to add a WorkQueue clear function just in case have overlapping JobIDs. '''
for i in range(initialStopStep, globalStop+1, 400):
stopStep = i
startStep = stopStep - stepsBack
stepRange = [startStep]
name = 'Main-backTestResult-' + JobID + '-' + str(stopStep).rjust(7,'0')
meTools.taskAdd(name, '/backtest/doBackTests', 'default', 0.5,
startAlg = startAlg, stopAlg = stopAlg, stopStep = stopStep, batchSize = batchSize,
stepRange = stepRange, uniquifier = unique, namespace = namespace, JobID = JobID,
totalBatches = totalBatches, callback = callback)
def processLiveAlgStepRange(start, stop, stepRange, algKeyFilter, namespace, JobID, callback, totalBatches, taskname):
namespace_manager.set_namespace(namespace)
currentStep = start
stopStepList = buildStopStepList(start, stop)
liveAlgInfo = getLiveAlgInfo(stop, stepRange, algKeyFilter)
for i in range(len(stopStepList)):
lastBackTestStop = stopStepList[i]
bestAlgs = getBestAlgs(lastBackTestStop, liveAlgInfo)
if i < len(stopStepList) - 1:
lastStep = stopStepList[i + 1]
else:
lastStep = stop
# Must deal with case where stop step is final step in stopStepList.
if currentStep < lastStep:
liveAlgInfo = processStepRangeDesires(currentStep, lastStep, bestAlgs, liveAlgInfo)
liveAlgInfo = getCurrentReturn(liveAlgInfo, lastStep)
currentStep = lastStep + 1
putList = []
for liveAlgKey in liveAlgInfo:
putList.append(liveAlgInfo[liveAlgKey])
meTools.retryPut(putList)
if callback:
meTools.taskAdd(
"callback-" + taskname,
callback,
"default",
0.5,
JobID=JobID,
taskname=taskname,
totalBatches=totalBatches,
jobtype="callback",
stepType="weeklyLiveAlgs",
model="",
)
def runBackTests(startAlg, stopAlg, stop, batchSize, stepRange, uniquifier, namespace, JobID, totalBatches, callback):
monthList = [str(step) for step in stepRange]
for batchStart in range(startAlg, stopAlg + 1, batchSize):
batchEnd = min(batchStart + batchSize - 1, stopAlg)
batchName = 'Batch-backTestResult-' + JobID + '-' + str(batchStart) + '-' + str(batchEnd) + '-' + str(stop)
meTools.taskAdd(batchName, '/backtest/doBackTestBatch', 'backTestQueue', 0.5,
startAlg = batchStart, stopAlg = batchEnd, monthBatch = monthList, stopStep = stop,
namespace = namespace, JobID = JobID, totalBatches = totalBatches, callback = callback)
def get(self):
from datetime import date
today = date.today()
if 'X-AppEngine-Cron' in self.request.headers and today.strftime('%w') == '3':
namespace = ''
unique = 'CRON' + today.strftime('%Y%m%d')
taskname = 'weeklySim-' + unique
meTools.taskAdd(taskname, '/simulate/weeklySimulationRun', 'default', 0.5,
namespace = namespace, unique = unique)
else:
raise(BaseException('Get handler only serves Cron!'))
def post(self):
JobID = self.request.get('JobID')
totalBatches = int(self.request.get('totalBatches'))
callback = self.request.get('callback')
taskname = self.request.headers['X-AppEngine-TaskName']
startAlg = int(self.request.get('startAlg'))
stopAlg = int(self.request.get('stopAlg'))
algBatch = [meTools.buildAlgKey(i) for i in range(startAlg, stopAlg + 1)]
monthBatch = self.request.get_all('monthBatch')
stopStep = self.request.get('stopStep')
namespace = str(self.request.get('namespace'))
backTestBatch(algBatch, monthBatch, stopStep, namespace)
if callback:
meTools.taskAdd('callback-' + taskname, callback, 'default', 0.5,
JobID = JobID, taskname = taskname, totalBatches = totalBatches,
model = '', jobtype = 'callback', stepType = 'weeklyBackTests')
def doCompoundReturns(stopStep, startStep, globalStop, namespace, name, i, cursor, model, JobID, callback, totalBatches, taskname, deadline = None):
from time import time
if not deadline:
deadline = time() + 540.00
entityModel = getattr(meSchema, model)
if entityModel == meSchema.backTestResult:
prefix = 'BTR-'
elif entityModel == meSchema.liveAlg:
prefix = 'LAR-'
else:
raise BaseException('Model must be backTestResult or liveAlg!')
count = 100
while count == 100:
query = entityModel.all().filter('stopStep =', stopStep).filter('startStep =', startStep).order('percentReturn')
if cursor != '':
query.with_cursor(cursor)
if deadline > time():
i += 1
entities = query.fetch(100)
count = len(entities)
memEntities = {}
for entity in entities:
memkey = prefix + entity.key().name()
memEntities[memkey] = entity.percentReturn
memcache.set_multi(memEntities)
cachepy.set_multi(memEntities)
doCompounds(stopStep, startStep, entities)
cursor = query.cursor()
else:
subTaskname = 'RVals-' + JobID + '-' + model + '-' + str(stopStep) + '-' + str(startStep) + '-' + str(i)
meTools.taskAdd(subTaskname, '/calculate/compounds/calculateCompounds', 'default', 0.5,
stopStep = stopStep, startStep = startStep, globalStop = globalStop, name = name,
i = i, cursor = cursor, model = model, JobID = JobID, callback = callback, totalBatches = totalBatches,
taskname = taskname)
return
if stopStep <= globalStop - 400:
stopStep += 400
startStep += 400
doCompoundReturns(stopStep,startStep,globalStop,namespace,name,0,'',model, JobID, callback, totalBatches, taskname, deadline)
elif callback:
meTools.taskAdd('callback-' + taskname, callback, 'default', 0.5,
JobID = JobID, taskname = taskname, totalBatches = totalBatches,
model = model, jobtype = 'callback', stepType = 'calculateRvals')
def fanoutTaskAdd(stopStep, startStep, globalStop, namespace, unique, model, callback = ''):
from google.appengine.api import namespace_manager
namespace_manager.set_namespace(namespace)
''' Partition range of steps into batches to add in parallel. '''
stepRange = stopStep - startStep
if stepRange == 800:
stepBlock = 2800
elif stepRange == 1600:
stepBlock = 2000
JobID = meTools.buildJobID(namespace, unique, globalStop, stopStep, stepRange)
totalBatches = ((globalStop - stopStep)/stepBlock) + 1
for i in range(stopStep, globalStop + 1, stepBlock):
newStopStep = i
newStartStep = newStopStep - stepRange
newGlobalStop = min(newStopStep + stepBlock - 1, globalStop)
subTaskname = 'RVals-' + JobID + '-' + model + '-' + str(newStopStep) + '-' + str(newStartStep) + '-0'
meTools.taskAdd(subTaskname, '/calculate/compounds/calculateCompounds', 'default', 0.5,
stopStep = newStopStep, startStep = newStartStep, globalStop = newGlobalStop, name = unique,
i = 0, cursor = '', model = model, JobID = JobID, callback = callback, totalBatches = totalBatches,
taskname = '')
namespace_manager.set_namespace('')
