Python foldの例

コード例 #1

ファイル: test_parallel.py プロジェクト: yingw787/toolz

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, map=Pool().map) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == {1, 2, 3}
    assert (fold(setadd, [1, 2, 3], set(), chunksize=2,
                 combine=set.union) == {1, 2, 3})

    assert fold(add, range(10), default=no_default2) == fold(add, range(10))

コード例 #2

ファイル: test_parallel.py プロジェクト: pytoolz/toolz

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == set((1, 2, 3))
    assert fold(setadd, [1, 2, 3], set(), chunksize=2, combine=set.union) == set((1, 2, 3))

    assert fold(add, range(10), default=no_default2) == fold(add, range(10))

コード例 #3

def test_fold():
    assert fold(add, range(10), 0) == reduce(add, range(10), 0)
    assert fold(add, range(10), 0, chunksize=2) == reduce(add, range(10), 0)
    assert fold(add, range(10)) == fold(add, range(10), 0)

    def setadd(s, item):
        s = s.copy()
        s.add(item)
        return s

    assert fold(setadd, [1, 2, 3], set()) == set((1, 2, 3))
    assert (fold(setadd, [1, 2, 3], set(), chunksize=2,
                 combine=set.union) == set((1, 2, 3)))

コード例 #4

 def np_or(seq):
     if seq:
         return fold((lambda x, y: x | y), seq)
     return True

コード例 #5

to run a frequency count reduction in parallel
"""
import dill as pickle
from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce
from random import choice
import time


def combine_counts(left, right):
    unique_keys = set(left.keys()).union(set(right.keys()))
    return {k: left.get(k, 0) + right.get(k, 0) for k in unique_keys}


def make_counts(acc, nxt):
    acc[nxt] = acc.get(nxt, 0) + 1
    return acc


xs = [choice([1, 2, 3, 4, 5, 6]) for _ in range(5000000)]

with Pool(8) as P:
    start_time = time.time()
    fold(make_counts, xs, {}, map=P.map, combine=combine_counts)
    print('Parallel reduce time: {}'.format(time.time() - start_time))

start_time = time.time()
reduce(make_counts, xs, {})
print('Normal reduce time: {}'.format(time.time() - start_time))

コード例 #6

ファイル: par-filter.py プロジェクト: sunyong2016/mastering-large-datasets

from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce


def map_combination(left, right):
    return left + right


def keep_if_even(acc, nxt):
    if nxt % 2 == 0:
        return acc + [nxt]
    else:
        return acc


with Pool() as P:
    fold(keep_if_even, range(500000), [], map=P.imap, combine=map_combination)

print(reduce(keep_if_even, range(500), []))

コード例 #7

from pathos.multiprocessing import ProcessingPool as Pool
from toolz.sandbox.parallel import fold
from functools import reduce


def my_add(left, right):
    return left + right


with Pool() as P:
    fold(my_add, range(500000), map=P.imap)

print(reduce(my_add, range(500)))