Python add_splay_iteration Examples

Example #1

File: AtA.py Project: dgleich/mrsimform

def runner(job):
    blocksize = gopts.getintkey('blocksize')
    schedule = gopts.getstrkey('reduce_schedule')
    schedule = schedule.split(',')
    for i,part in enumerate(schedule):
        if part.startswith('s'):
            mrmc.add_splay_iteration(job, part)
        else:
            nreducers = int(part)
            if i == 0:
                mapper = mrmc.AtA(blocksize=blocksize)
                reducer = mrmc.ArraySumReducer
            else:
                mapper = mrmc.ID_MAPPER
                reducer = mrmc.ArraySumReducer()
                nreducers = 1
            job.additer(mapper=mapper, reducer=reducer,
                        opts=[('numreducetasks', str(nreducers))])

Example #2

def runner(job):
    blocksize = gopts.getintkey('blocksize')
    schedule = gopts.getstrkey('reduce_schedule')
    schedule = schedule.split(',')
    for i, part in enumerate(schedule):
        if part.startswith('s'):
            mrmc.add_splay_iteration(job, part)
        else:
            nreducers = int(part)
            if i == 0:
                mapper = mrmc.AtA(blocksize=blocksize)
                reducer = mrmc.ArraySumReducer
            else:
                mapper = mrmc.ID_MAPPER
                reducer = mrmc.ArraySumReducer()
                nreducers = 1
            job.additer(mapper=mapper,
                        reducer=reducer,
                        opts=[('numreducetasks', str(nreducers))])