Exemple #1
0
def test_do_func_raises_keyboard_interrupt():
    @processpool.do_func
    def error_do_func(filename, i):
        raise KeyboardInterrupt("whoops")
    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(KeyboardInterrupt):
        processpool.do(error_do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([])
Exemple #2
0
def test_do_func_raises_ioerror_exc():
    @processpool.do_func
    def error_do_func(filename, i):
        raise IOError("whoops")
    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(IOError):
        processpool.do(error_do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([])
Exemple #3
0
def test_do_func_raises_other_exception():
    # We want to ensure we do not stop extraction on unknown exceptions
    @processpool.do_func
    def error_do_func(filename, i):
        raise MyKindOfError("whoops")
    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(MyKindOfError):
        processpool.do(error_do_func, iterable, pool_size=1)
def test_do_func_raises_keyboard_interrupt():
    @processpool.do_func
    def error_do_func(filename, i):
        raise KeyboardInterrupt("whoops")

    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(KeyboardInterrupt):
        processpool.do(error_do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([])
def test_do_func_raises_ioerror_exc():
    @processpool.do_func
    def error_do_func(filename, i):
        raise IOError("whoops")

    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(IOError):
        processpool.do(error_do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([])
def test_do_func_raises_other_exception():
    # We want to ensure we do not stop extraction on unknown exceptions
    @processpool.do_func
    def error_do_func(filename, i):
        raise MyKindOfError("whoops")

    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    with pytest.raises(MyKindOfError):
        processpool.do(error_do_func, iterable, pool_size=1)
Exemple #7
0
def test_error_during_iteration_in_this_process():
    t = tempfile.NamedTemporaryFile()
    def error_during_iteration():
        yield (t.name, 1)
        yield (t.name, 2)
        raise MyKindOfError(t.name)
    iterable = error_during_iteration()
    with pytest.raises(MyKindOfError):
        processpool.do(do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([b'1', b'2'])
def test_error_during_iteration_in_this_process():
    t = tempfile.NamedTemporaryFile()

    def error_during_iteration():
        yield (t.name, 1)
        yield (t.name, 2)
        raise MyKindOfError(t.name)

    iterable = error_during_iteration()
    with pytest.raises(MyKindOfError):
        processpool.do(do_func, iterable, pool_size=1)
    assert set(t.read().split()) == set([b'1', b'2'])
def test_error_during_iteration():
    t = tempfile.NamedTemporaryFile()
    # We have to do a little dance to get an exception
    # that occurs during iteration to not screw up
    # the process pool.
    # If you want to see what I mean try
    def error_during_iteration():
        yield (t.name, 1)
        yield (t.name, 2)
        raise MyKindOfError(t.name)
    iterable = error_during_iteration()
    with pytest.raises(MyKindOfError):
        processpool.do(do_func, iterable)
    assert set(t.read().split()) == set([b'1', b'2'])
def test_error_during_iteration():
    t = tempfile.NamedTemporaryFile()

    # We have to do a little dance to get an exception
    # that occurs during iteration to not screw up
    # the process pool.
    # If you want to see what I mean try
    def error_during_iteration():
        yield (t.name, 1)
        yield (t.name, 2)
        time.sleep(0.01)
        raise MyKindOfError(t.name)

    iterable = error_during_iteration()
    with pytest.raises(MyKindOfError):
        processpool.do(do_func, iterable)
    assert set(t.read().split()) == set([b'1', b'2'])
def test_initializer_in_this_process():

    mutable = {}

    def initializer(value):
        mutable['prefix'] = value

    @processpool.do_func
    def do_func_with_initializer(filename, number):
        with open(filename, 'a+') as fp:
            fp.write('{0}:{1}\n'.format(mutable['prefix'], number))
            fp.flush()

    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func_with_initializer, ((t.name, i) for i in (7, 8)),
                   pool_size=1,
                   initializer=initializer,
                   initargs=('foo', ))
    assert set(t.read().split()) == set([b'foo:7', b'foo:8'])
Exemple #12
0
def test_initializer_in_this_process():

    mutable = {}
    def initializer(value):
        mutable['prefix'] = value

    @processpool.do_func
    def do_func_with_initializer(filename, number):
        with open(filename, 'a+') as fp:
            fp.write('{0}:{1}\n'.format(mutable['prefix'], number))
            fp.flush()
    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func_with_initializer,
        ((t.name, i) for i in (7, 8)),
        pool_size=1,
        initializer=initializer,
        initargs=('foo',)
    )
    assert set(t.read().split()) == set([b'foo:7', b'foo:8'])
def test_more_work_in_pool():
    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func, ((t.name, i) for i in (7, 8)))
    assert set(t.read().split()) == set([b'7', b'8', b'other=a', b'other=b'])
def test_do_in_this_process():
    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func, ((t.name, i) for i in range(3)), pool_size=1)
    assert set(t.read().split()) == set([b'0', b'1', b'2'])
def test_do_in_pool():
    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    processpool.do(do_func, iterable)
    assert set(t.read().split()) == set([b'0', b'1', b'2'])
Exemple #16
0
def test_do_in_pool():
    t = tempfile.NamedTemporaryFile()
    iterable = ((t.name, i) for i in range(3))
    processpool.do(do_func, iterable)
    assert set(t.read().split()) == set([b'0', b'1', b'2'])
Exemple #17
0
def test_more_work_in_pool():
    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func, ((t.name, i) for i in (7, 8)))
    assert set(t.read().split()) == set([b'7', b'8', b'other=a', b'other=b'])
Exemple #18
0
def test_do_in_this_process():
    t = tempfile.NamedTemporaryFile()
    processpool.do(do_func, ((t.name, i) for i in range(3)), pool_size=1)
    assert set(t.read().split()) == set([b'0', b'1', b'2'])