Example #1
0
    def heartbeat(self, context):
        """Calculate and return the heartbeat for an activity.

        The heartbeat represents when an actvitity should be sending a signal
        to SWF that it has not completed yet. The heartbeat is sent everytime
        a new task is going to be launched.

        Similar to the `BaseRunner.timeout`, the heartbeat is pessimistic, it
        looks at the largest heartbeat and set it up.

        Return:
            str: The heartbeat timeout (boto requires the timeout to be a string
                not a regular number.)
        """

        heartbeat = 0

        for task in flatten(self.tasks, context):
            task_details = getattr(task, '__garcon__', None)
            task_heartbeat = DEFAULT_TASK_HEARTBEAT

            if task_details:
                task_heartbeat = task_details.get(
                    'heartbeat', DEFAULT_TASK_HEARTBEAT)

            if task_heartbeat > heartbeat:
                heartbeat = task_heartbeat

        return heartbeat
Example #2
0
    def requirements(self, context):
        """Find all the requirements from the list of tasks and return it.

        If a task does not use the `task.decorate`, no assumptions can be made
        on which values from the context will be used, and it will raise an
        exception.

        Raise:
            NoRequirementFound: The exception when no requirements have been
                mentioned in at least one or more tasks.

        Return:
            set: the list of the required values from the context.
        """

        requirements = []

        # Get all the tasks and the lists (so the .fill on lists are also
        # considered.)
        all_tasks = list(self.tasks) + list(flatten(self.tasks, context))
        for task in all_tasks:
            task_details = getattr(task, '__garcon__', None)
            if task_details:
                requirements += task_details.get('requirements', [])
            else:
                raise NoRunnerRequirementsFound()
        return set(requirements)
Example #3
0
    def requirements(self, context):
        """Find all the requirements from the list of tasks and return it.

        If a task does not use the `task.decorate`, no assumptions can be made
        on which values from the context will be used, and it will raise an
        exception.

        Raise:
            NoRequirementFound: The exception when no requirements have been
                mentioned in at least one or more tasks.

        Return:
            set: the list of the required values from the context.
        """

        requirements = []

        # Get all the tasks and the lists (so the .fill on lists are also
        # considered.)
        all_tasks = list(self.tasks) + list(flatten(self.tasks, context))
        for task in all_tasks:
            task_details = getattr(task, '__garcon__', None)
            if task_details:
                requirements += task_details.get('requirements', [])
            else:
                raise NoRunnerRequirementsFound()
        return set(requirements)
Example #4
0
    def timeout(self, context):
        """Calculate and return the timeout for an activity.

        The calculation of the timeout is pessimistic: it takes the worse case
        scenario (even for asynchronous task lists, it supposes there is only
        one thread completed at a time.)

        Return:
            str: The timeout (boto requires the timeout to be a string and not
                a regular number.)
        """

        timeout = 0

        for task in flatten(self.tasks, context):
            task_timeout = DEFAULT_TASK_TIMEOUT
            task_details = getattr(task, '__garcon__', None)

            if task_details:
                task_timeout = task_details.get(
                    'timeout', DEFAULT_TASK_TIMEOUT)

            timeout = timeout + task_timeout

        return timeout
Example #5
0
    def timeout(self, context):
        """Calculate and return the timeout for an activity.

        The calculation of the timeout is pessimistic: it takes the worse case
        scenario (even for asynchronous task lists, it supposes there is only
        one thread completed at a time.)

        Return:
            str: The timeout (boto requires the timeout to be a string and not
                a regular number.)
        """

        timeout = 0

        for task in flatten(self.tasks, context):
            task_timeout = DEFAULT_TASK_TIMEOUT
            task_details = getattr(task, '__garcon__', None)

            if task_details:
                task_timeout = task_details.get('timeout',
                                                DEFAULT_TASK_TIMEOUT)

            timeout = timeout + task_timeout

        return timeout
Example #6
0
    def heartbeat(self, context):
        """Calculate and return the heartbeat for an activity.

        The heartbeat represents when an actvitity should be sending a signal
        to SWF that it has not completed yet. The heartbeat is sent everytime
        a new task is going to be launched.

        Similar to the `BaseRunner.timeout`, the heartbeat is pessimistic, it
        looks at the largest heartbeat and set it up.

        Return:
            str: The heartbeat timeout (boto requires the timeout to be a
                string not a regular number.)
        """

        heartbeat = 0

        for task in flatten(self.tasks, context):
            task_details = getattr(task, '__garcon__', None)
            task_heartbeat = DEFAULT_TASK_HEARTBEAT

            if task_details:
                task_heartbeat = task_details.get('heartbeat',
                                                  DEFAULT_TASK_HEARTBEAT)

            if task_heartbeat > heartbeat:
                heartbeat = task_heartbeat

        return heartbeat
Example #7
0
 def execute(self, activity, context):
     result = dict()
     for task in flatten(self.tasks, context):
         activity.heartbeat()
         task_context = dict(list(result.items()) + list(context.items()))
         resp = task(task_context, activity=activity)
         result.update(resp or dict())
     return result
Example #8
0
 def execute(self, activity, context):
     result = dict()
     for task in flatten(self.tasks, context):
         activity.heartbeat()
         task_context = dict(list(result.items()) + list(context.items()))
         resp = task(task_context, activity=activity)
         result.update(resp or dict())
     return result
Example #9
0
    def execute(self, activity, context):
        result = dict()
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            tasks = []
            for task in flatten(self.tasks, context):
                tasks.append(executor.submit(task, context, activity=activity))

            for future in futures.as_completed(tasks):
                activity.heartbeat()
                data = future.result()
                result.update(data or {})
        return result
Example #10
0
    def execute(self, activity, context):
        result = dict()
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            tasks = []
            for task in flatten(self.tasks, context):
                tasks.append(executor.submit(task, context, activity=activity))

            for future in futures.as_completed(tasks):
                activity.heartbeat()
                data = future.result()
                result.update(data or {})
        return result
Example #11
0
def test_synchronous_tasks(monkeypatch):
    """Test synchronous tasks.
    """

    monkeypatch.setattr(activity.Activity, '__init__', lambda self: None)
    monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None)

    resp = dict(foo='bar')
    current_runner = runner.Sync(MagicMock(), MagicMock(return_value=resp))
    current_activity = activity.Activity()
    current_activity.hydrate(dict(runner=current_runner))

    result = current_runner.execute(current_activity, EMPTY_CONTEXT)

    assert len(current_runner.tasks) == 2

    for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
        assert current_task.called

    assert resp == result
Example #12
0
def test_flatten():
    """Test the flatten function.
    """
    @task.decorate(timeout=10)
    def task_a(name):
        pass

    @task.decorate(timeout=10)
    def task_b(name):
        pass

    @task.list
    def task_generator(context):
        yield task_b
        if context.get('value'):
            yield task_a

    value = list(
        task.flatten((task_a, task_b, task_generator, task_a),
                     dict(value='something')))
    assert value == [task_a, task_b, task_b, task_a, task_a]
Example #13
0
def test_synchronous_tasks(monkeypatch):
    """Test synchronous tasks.
    """

    monkeypatch.setattr(activity.Activity, '__init__', lambda self: None)
    monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None)

    resp = dict(foo='bar')
    current_runner = runner.Sync(
        MagicMock(), MagicMock(return_value=resp))
    current_activity = activity.Activity()
    current_activity.hydrate(dict(runner=current_runner))

    result = current_runner.execute(current_activity, EMPTY_CONTEXT)

    assert len(current_runner.tasks) == 2

    for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
        assert current_task.called

    assert resp == result
Example #14
0
def test_synchronous_tasks(monkeypatch, boto_client):
    """Test synchronous tasks.
    """

    monkeypatch.setattr(activity.ActivityExecution, 'heartbeat',
        lambda self: None)

    resp = dict(foo='bar')
    current_runner = runner.Sync(
        MagicMock(), MagicMock(return_value=resp))
    current_activity = activity.ActivityExecution(
        boto_client, 'activityId', 'taskToken', '{"context": "value"}')

    result = current_runner.execute(current_activity, EMPTY_CONTEXT)

    assert len(current_runner.tasks) == 2

    for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
        assert current_task.called

    assert resp == result
Example #15
0
def test_flatten():
    """Test the flatten function.
    """

    spy = MagicMock

    @task.decorate(timeout=10)
    def task_a(name):
        pass

    @task.decorate(timeout=10)
    def task_b(name):
        pass

    @task.list
    def task_generator(context):
        yield task_b
        if context.get('value'):
            yield task_a

    value = list(task.flatten(
        (task_a, task_b, task_generator, task_a),
        dict(value='something')))
    assert value == [task_a, task_b, task_b, task_a, task_a]