def heartbeat(self, context):
"""Calculate and return the heartbeat for an activity.
The heartbeat represents when an actvitity should be sending a signal
to SWF that it has not completed yet. The heartbeat is sent everytime
a new task is going to be launched.
Similar to the `BaseRunner.timeout`, the heartbeat is pessimistic, it
looks at the largest heartbeat and set it up.
Return:
str: The heartbeat timeout (boto requires the timeout to be a string
not a regular number.)
"""
heartbeat = 0
for task in flatten(self.tasks, context):
task_details = getattr(task, '__garcon__', None)
task_heartbeat = DEFAULT_TASK_HEARTBEAT
if task_details:
task_heartbeat = task_details.get(
'heartbeat', DEFAULT_TASK_HEARTBEAT)
if task_heartbeat > heartbeat:
heartbeat = task_heartbeat
return heartbeat
def requirements(self, context):
"""Find all the requirements from the list of tasks and return it.
If a task does not use the `task.decorate`, no assumptions can be made
on which values from the context will be used, and it will raise an
exception.
Raise:
NoRequirementFound: The exception when no requirements have been
mentioned in at least one or more tasks.
Return:
set: the list of the required values from the context.
"""
requirements = []
# Get all the tasks and the lists (so the .fill on lists are also
# considered.)
all_tasks = list(self.tasks) + list(flatten(self.tasks, context))
for task in all_tasks:
task_details = getattr(task, '__garcon__', None)
if task_details:
requirements += task_details.get('requirements', [])
else:
raise NoRunnerRequirementsFound()
return set(requirements)
def requirements(self, context):
"""Find all the requirements from the list of tasks and return it.
If a task does not use the `task.decorate`, no assumptions can be made
on which values from the context will be used, and it will raise an
exception.
Raise:
NoRequirementFound: The exception when no requirements have been
mentioned in at least one or more tasks.
Return:
set: the list of the required values from the context.
"""
requirements = []
# Get all the tasks and the lists (so the .fill on lists are also
# considered.)
all_tasks = list(self.tasks) + list(flatten(self.tasks, context))
for task in all_tasks:
task_details = getattr(task, '__garcon__', None)
if task_details:
requirements += task_details.get('requirements', [])
else:
raise NoRunnerRequirementsFound()
return set(requirements)
def timeout(self, context):
"""Calculate and return the timeout for an activity.
The calculation of the timeout is pessimistic: it takes the worse case
scenario (even for asynchronous task lists, it supposes there is only
one thread completed at a time.)
Return:
str: The timeout (boto requires the timeout to be a string and not
a regular number.)
"""
timeout = 0
for task in flatten(self.tasks, context):
task_timeout = DEFAULT_TASK_TIMEOUT
task_details = getattr(task, '__garcon__', None)
if task_details:
task_timeout = task_details.get(
'timeout', DEFAULT_TASK_TIMEOUT)
timeout = timeout + task_timeout
return timeout
def timeout(self, context):
"""Calculate and return the timeout for an activity.
The calculation of the timeout is pessimistic: it takes the worse case
scenario (even for asynchronous task lists, it supposes there is only
one thread completed at a time.)
Return:
str: The timeout (boto requires the timeout to be a string and not
a regular number.)
"""
timeout = 0
for task in flatten(self.tasks, context):
task_timeout = DEFAULT_TASK_TIMEOUT
task_details = getattr(task, '__garcon__', None)
if task_details:
task_timeout = task_details.get('timeout',
DEFAULT_TASK_TIMEOUT)
timeout = timeout + task_timeout
return timeout
def heartbeat(self, context):
"""Calculate and return the heartbeat for an activity.
The heartbeat represents when an actvitity should be sending a signal
to SWF that it has not completed yet. The heartbeat is sent everytime
a new task is going to be launched.
Similar to the `BaseRunner.timeout`, the heartbeat is pessimistic, it
looks at the largest heartbeat and set it up.
Return:
str: The heartbeat timeout (boto requires the timeout to be a
string not a regular number.)
"""
heartbeat = 0
for task in flatten(self.tasks, context):
task_details = getattr(task, '__garcon__', None)
task_heartbeat = DEFAULT_TASK_HEARTBEAT
if task_details:
task_heartbeat = task_details.get('heartbeat',
DEFAULT_TASK_HEARTBEAT)
if task_heartbeat > heartbeat:
heartbeat = task_heartbeat
return heartbeat
def execute(self, activity, context):
result = dict()
for task in flatten(self.tasks, context):
activity.heartbeat()
task_context = dict(list(result.items()) + list(context.items()))
resp = task(task_context, activity=activity)
result.update(resp or dict())
return result
def execute(self, activity, context):
result = dict()
for task in flatten(self.tasks, context):
activity.heartbeat()
task_context = dict(list(result.items()) + list(context.items()))
resp = task(task_context, activity=activity)
result.update(resp or dict())
return result
def execute(self, activity, context):
result = dict()
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
tasks = []
for task in flatten(self.tasks, context):
tasks.append(executor.submit(task, context, activity=activity))
for future in futures.as_completed(tasks):
activity.heartbeat()
data = future.result()
result.update(data or {})
return result
def execute(self, activity, context):
result = dict()
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
tasks = []
for task in flatten(self.tasks, context):
tasks.append(executor.submit(task, context, activity=activity))
for future in futures.as_completed(tasks):
activity.heartbeat()
data = future.result()
result.update(data or {})
return result
def test_synchronous_tasks(monkeypatch):
"""Test synchronous tasks.
"""
monkeypatch.setattr(activity.Activity, '__init__', lambda self: None)
monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None)
resp = dict(foo='bar')
current_runner = runner.Sync(MagicMock(), MagicMock(return_value=resp))
current_activity = activity.Activity()
current_activity.hydrate(dict(runner=current_runner))
result = current_runner.execute(current_activity, EMPTY_CONTEXT)
assert len(current_runner.tasks) == 2
for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
assert current_task.called
assert resp == result
def test_flatten():
"""Test the flatten function.
"""
@task.decorate(timeout=10)
def task_a(name):
pass
@task.decorate(timeout=10)
def task_b(name):
pass
@task.list
def task_generator(context):
yield task_b
if context.get('value'):
yield task_a
value = list(
task.flatten((task_a, task_b, task_generator, task_a),
dict(value='something')))
assert value == [task_a, task_b, task_b, task_a, task_a]
def test_synchronous_tasks(monkeypatch):
"""Test synchronous tasks.
"""
monkeypatch.setattr(activity.Activity, '__init__', lambda self: None)
monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None)
resp = dict(foo='bar')
current_runner = runner.Sync(
MagicMock(), MagicMock(return_value=resp))
current_activity = activity.Activity()
current_activity.hydrate(dict(runner=current_runner))
result = current_runner.execute(current_activity, EMPTY_CONTEXT)
assert len(current_runner.tasks) == 2
for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
assert current_task.called
assert resp == result
def test_synchronous_tasks(monkeypatch, boto_client):
"""Test synchronous tasks.
"""
monkeypatch.setattr(activity.ActivityExecution, 'heartbeat',
lambda self: None)
resp = dict(foo='bar')
current_runner = runner.Sync(
MagicMock(), MagicMock(return_value=resp))
current_activity = activity.ActivityExecution(
boto_client, 'activityId', 'taskToken', '{"context": "value"}')
result = current_runner.execute(current_activity, EMPTY_CONTEXT)
assert len(current_runner.tasks) == 2
for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT):
assert current_task.called
assert resp == result
def test_flatten():
"""Test the flatten function.
"""
spy = MagicMock
@task.decorate(timeout=10)
def task_a(name):
pass
@task.decorate(timeout=10)
def task_b(name):
pass
@task.list
def task_generator(context):
yield task_b
if context.get('value'):
yield task_a
value = list(task.flatten(
(task_a, task_b, task_generator, task_a),
dict(value='something')))
assert value == [task_a, task_b, task_b, task_a, task_a]