Python secondary Examples

Example #1

File: keys_join.py Project: pedroconcejero/devcon_python4bigdata

def runner(job):
    multimap = MultiMapper()
    multimap.add("hdi", primary(parse_hdi))
    multimap.add("disasters", secondary(parse_disasters))

    opts = [("inputformat", "text"), ("outputformat", "text")]
    o1 = job.additer(multimap, Reducer, opts=opts)

Example #2

def runner(job):
    multimap = MultiMapper()
    multimap.add("hdi", primary(parse_hdi))
    multimap.add("disasters", secondary(parse_disasters))

    opts = [
        ("inputformat", "text"),
        ("outputformat", "text"),
    ]
    o1 = job.additer(multimap, Reducer, opts=opts)

Example #3

File: join.py Project: acanalesg/campusparty_workshop

def runner(job):
    opts = [("inputformat", "text"),
            ("outputformat", "text"),
            ]

    multimap = MultiMapper()
    multimap.add("outputgrid", primary(mapper_locations))
    multimap.add("metropolitan", secondary(mapper_footfalls))

    o1 = job.additer(multimap,
                     Joiner, opts=opts)

Example #4

def runner(job):
    opts = [
        ("inputformat", "text"),
        ("outputformat", "text"),
    ]

    multimap = MultiMapper()
    multimap.add("outputgrid", primary(mapper_locations))
    multimap.add("metropolitan", secondary(mapper_footfalls))

    o1 = job.additer(multimap, Joiner, opts=opts)

Example #5

def runner(job):
    multimapper = dumbo.MultiMapper()
    multimapper.add("hostnames", primary(mapper1))
    multimapper.add("logs", secondary(mapper1))
    job.additer(multimapper, Reducer1)
    job.additer(mapper2, reducer2, combiner=reducer2)

Example #6

File: join.py Project: CyaLiven/dumbo

def runner(job):
    multimapper = dumbo.MultiMapper()
    multimapper.add("hostnames", primary(mapper1))
    multimapper.add("logs", secondary(mapper1))
    job.additer(multimapper, Reducer1)
    job.additer(mapper2, reducer2, combiner=reducer2)