def addTaskRange(initialStopStep, globalStop, unique, namespace, batchSize=5, stepsBack=1600, callback = ''):
    '''
      Task Ranges will branch out into weekly clumps for all algs.
      Remove startAlg, stopAlg from task name and use stepRange value instead.

      Must calculate expected count to be found for completion entities.
    '''
    import meSchema
    from google.appengine.api import namespace_manager
    from math import ceil
    namespace_manager.set_namespace('')
    startAlg = 1
    stopAlg = int(meSchema.meAlg.all(keys_only=True).order('-__key__').get().name())
    
    ''' Pre-calculating number of batches for callback to check against. '''
    numWeeks = ((globalStop - initialStopStep)/400) + 1
    numAlgs = stopAlg - startAlg + 1
    batchesWeek = ceil(numAlgs/float(batchSize))
    totalBatches = int(numWeeks*batchesWeek)
    JobID = meTools.buildJobID(namespace, unique, globalStop, initialStopStep, stepsBack)

    ''' Probably need to add a WorkQueue clear function just in case have overlapping JobIDs.  '''
    
    for i in range(initialStopStep, globalStop+1, 400):
        stopStep = i
        startStep = stopStep - stepsBack
        stepRange = [startStep]
        name = 'Main-backTestResult-' + JobID + '-' + str(stopStep).rjust(7,'0')
        meTools.taskAdd(name, '/backtest/doBackTests', 'default', 0.5,
                        startAlg = startAlg, stopAlg = stopAlg, stopStep = stopStep, batchSize = batchSize,
                        stepRange = stepRange, uniquifier = unique, namespace = namespace, JobID = JobID,
                        totalBatches = totalBatches, callback = callback)
Beispiel #2
0
def doAllLiveAlgs(initialStopStep, stepRange, globalStop, namespace, name, callback=""):
    JobID = meTools.buildJobID(namespace, name, globalStop, initialStopStep, stepRange)
    # techneCount = len(FTLtype)*len(NRtype)
    numWeeks = ((globalStop - initialStopStep) / 400) + 1
    totalBatches = numWeeks
    for i in range(initialStopStep, globalStop + 1, 400):
        stopStep = i
        calculateWeeklyLiveAlgs(stopStep, stepRange, namespace, JobID, totalBatches, callback)
def startSim(namespace, unique, globalStop, initialStop, stepRange, goNext):
    from google.appengine.api.datastore import Key
    
    JobID = meTools.buildJobID(namespace, unique, globalStop, initialStop, stepRange)

    persistStops = meSchema.WorkQueue(key_name = JobID, globalStop = globalStop, initialStop = initialStop)
    meTools.memPut_multi({persistStops.key().name() : persistStops}, priority = 1)
    
    if not globalStop >= initialStop:
        raise(BaseException('globalStop: %s is not >= lastStopStep: %s' % (globalStop, initialStop)))

    lastDesire = meSchema.desire.all(keys_only = True).filter('__key__ <', Key.from_path('desire','1000000_0000_00')).order('-__key__').get()
    if lastDesire:
        lastDesireStop = int(lastDesire.name().split('_')[0])
    else:
        lastDesireStop = 1
    desireFunc.primeDesireCache(lastDesireStop)
    for step in range(lastDesireStop, globalStop + 1):
        desireFunc.doDesires(step)

    if goNext == 'true':
        doNext(JobID, 'weeklyDesires','')
def fanoutTaskAdd(stopStep, startStep, globalStop, namespace, unique, model, callback = ''):
    from google.appengine.api import namespace_manager
    namespace_manager.set_namespace(namespace)
    ''' Partition range of steps into batches to add in parallel. '''
    stepRange = stopStep - startStep
    if stepRange == 800:
        stepBlock = 2800
    elif stepRange == 1600:
        stepBlock = 2000

    JobID = meTools.buildJobID(namespace, unique, globalStop, stopStep, stepRange)
    totalBatches = ((globalStop - stopStep)/stepBlock) + 1
    for i in range(stopStep, globalStop + 1, stepBlock):
        newStopStep = i
        newStartStep = newStopStep - stepRange
        newGlobalStop = min(newStopStep + stepBlock - 1, globalStop)
        subTaskname = 'RVals-' + JobID + '-' + model + '-' + str(newStopStep) + '-' + str(newStartStep) + '-0'
        meTools.taskAdd(subTaskname, '/calculate/compounds/calculateCompounds', 'default', 0.5,
                        stopStep = newStopStep, startStep = newStartStep, globalStop = newGlobalStop, name = unique,
                        i = 0, cursor = '', model = model, JobID = JobID, callback = callback, totalBatches = totalBatches,
                        taskname = '')
    namespace_manager.set_namespace('')