Python DataFlowKernel Examples

Example #1

File: test_98.py Project: zonca/parsl

def test_immutable_config(n=2):
    """Regression test for immutable config #98
    """

    original = str(config)
    dfk = DataFlowKernel(config=config)
    after = str(config)

    dfk.cleanup()
    assert original == after, "Config modified"

Example #2

File: test_98.py Project: pnookala/parsl

def test_immutable_config(n=2):
    """Regression test for immutable config #98
    """

    original = json.dumps(config, sort_keys=True)
    dfk = DataFlowKernel(config=config)
    after = json.dumps(config, sort_keys=True)

    dfk.cleanup()
    assert original == after, "Config modified"

Example #3

def run_checkpointed(checkpoints):
    # set_stream_logger()
    from parsl.tests.configs.local_threads_checkpoint_task_exit import config
    config.checkpoint_files = checkpoints
    dfk = DataFlowKernel(config=config)

    @App('python', dfk, cache=True)
    def cached_rand(x):
        import random
        return random.randint(0, 10000)

    items = []
    for i in range(0, 5):
        x = cached_rand(i)
        items.append(x)

    dfk.cleanup()
    return [i.result() for i in items], dfk.run_dir

Example #4

def run_race(sleep_dur):

    from parsl.tests.configs.local_threads_checkpoint_dfk_exit import config
    dfk = DataFlowKernel(config=config)

    @App('python', dfk, cache=True)
    def cached_rand(x, sleep_dur=0):
        import random
        import time
        time.sleep(sleep_dur)
        return random.randint(0, 10000)

    items = []
    for i in range(0, 5):
        x = cached_rand(i, sleep_dur=sleep_dur)
        items.append(x)

    dfk.cleanup()
    return [i.result for i in items]

Example #5

import argparse

import pytest

import parsl
from parsl.dataflow.dflow import DataFlowKernel
from parsl.app.app import App
from parsl.tests.conftest import load_dfk
from parsl.tests.configs.exex_local import config

parsl.clear()
dfk = DataFlowKernel(config=config)
parsl.set_stream_logger()

import logging
logger = logging.getLogger(__name__)


@App("python", dfk, executors=['Extreme_Local'])
def python_app_2():
    import os
    import threading
    import time
    time.sleep(1)
    return "Hello from PID[{}] TID[{}]".format(os.getpid(),
                                               threading.current_thread())


@App("python", dfk, executors=['Extreme_Local'])
def python_app_1():
    import os

Example #6

import argparse

import pytest

import parsl
from parsl.app.app import App
from parsl.dataflow.dflow import DataFlowKernel
from parsl.tests.configs.local_threads_no_cache import config

parsl.clear()
dfk = DataFlowKernel(config)


@App('python', dfk)
def random_uuid(x):
    import uuid
    return str(uuid.uuid4())


@pytest.mark.local
def test_python_memoization(n=4):
    """Testing python memoization disable via config
    """
    x = random_uuid(0)

    for i in range(0, n):
        foo = random_uuid(0)
        assert foo.result() != x.result(
        ), "Memoized results were used when memoization was disabled"