Python fold 예제들

예제 #1

파일: test_parallel.py 프로젝트: yingw787/toolz

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, map=Pool().map) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == {1, 2, 3}
    assert (fold(setadd, [1, 2, 3], set(), chunksize=2,
                 combine=set.union) == {1, 2, 3})

    assert fold(add, range(10), default=no_default2) == fold(add, range(10))

예제 #2

파일: test_parallel.py 프로젝트: pytoolz/toolz

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == set((1, 2, 3))
    assert fold(setadd, [1, 2, 3], set(), chunksize=2, combine=set.union) == set((1, 2, 3))

    assert fold(add, range(10), default=no_default2) == fold(add, range(10))

예제 #3

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == set((1, 2, 3))
    assert (fold(setadd, [1, 2, 3], set(), chunksize=2,
                 combine=set.union) == set((1, 2, 3)))

예제 #4

 def np_or(seq):
     if seq:
         return fold((lambda x, y: x | y), seq)
     return True

예제 #5

to run a frequency count reduction in parallel
"""
import dill as pickle
from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce
from random import choice
import time


def combine_counts(left, right):
    unique_keys = set(left.keys()).union(set(right.keys()))
    return {k: left.get(k, 0) + right.get(k, 0) for k in unique_keys}


def make_counts(acc, nxt):
    acc[nxt] = acc.get(nxt, 0) + 1
    return acc


xs = [choice([1, 2, 3, 4, 5, 6]) for _ in range(5000000)]

with Pool(8) as P:
    start_time = time.time()
    fold(make_counts, xs, {}, map=P.map, combine=combine_counts)
    print('Parallel reduce time: {}'.format(time.time() - start_time))

start_time = time.time()
reduce(make_counts, xs, {})
print('Normal reduce time: {}'.format(time.time() - start_time))

예제 #6

파일: par-filter.py 프로젝트: sunyong2016/mastering-large-datasets

from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce


def map_combination(left, right):
    return left + right


def keep_if_even(acc, nxt):
    if nxt % 2 == 0:
        return acc + [nxt]
    else:
        return acc


with Pool() as P:
    fold(keep_if_even, range(500000), [], map=P.imap, combine=map_combination)

print(reduce(keep_if_even, range(500), []))

예제 #7

from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce


def my_add(left, right):
    return left + right


with Pool() as P:
    fold(my_add, range(500000), map=P.imap)

print(reduce(my_add, range(500)))