def create_action(db_task):
task_spec = tasks.TaskSpec(db_task['task_spec'])
full_action_name = task_spec.get_full_action_name()
action_cls = get_action_class(full_action_name)
if not action_cls:
# If action is not found in registered actions try to find ad-hoc
# action definition.
action = _create_adhoc_action(db_task)
if action:
return action
else:
msg = 'Unknown action [workbook_name=%s, action=%s]' % \
(db_task['workbook_name'], full_action_name)
raise exc.ActionException(msg)
action_params = db_task['parameters'] or {}
if _has_action_context_param(action_cls):
action_params[_ACTION_CTX_PARAM] = _get_action_context(db_task)
try:
return action_cls(**action_params)
except Exception as e:
raise exc.ActionException('Failed to create action [db_task=%s]: %s' %
(db_task, e))
def run(self):
LOG.info("Running HTTP action "
"[url=%s, params=%s, method=%s, headers=%s, body=%s]" %
(self.url, self.params, self.method, self.headers, self.body))
try:
resp = requests.request(self.method,
self.url,
params=self.params,
headers=self.headers,
data=self.body)
except Exception as e:
raise exc.ActionException("Failed to send HTTP request: %s" % e)
LOG.info("HTTP action response:\n%s\n%s" %
(resp.status_code, resp.content))
# TODO: Not sure we need to have this check here in base HTTP action.
if resp.status_code not in range(200, 307):
raise exc.ActionException("Received error HTTP code: %s" %
resp.status_code)
# Construct all important resp data in readable structure.
headers = dict(resp.headers.items())
status = resp.status_code
try:
content = resp.json()
except Exception as e:
LOG.debug("HTTP action response is not json.")
content = resp.content
return {'content': content, 'headers': headers, 'status': status}
def run(self, context):
LOG.info('Running fail action.')
if self.error_data:
return actions.Result(error=self.error_data)
raise exc.ActionException('Fail action expected exception.')
def run(self):
try:
return self._convert_result(self.base_action.run())
except Exception as e:
raise exc.ActionException("Failed to convert result in action %s,"
"[Exception = %s" %
(self.action_spec.name, e))
def run(self):
LOG.info("Sending email message "
"[from=%s, to=%s, subject=%s, using smtp=%s, body=%s...]" %
(self.sender, self.to, self.subject, self.smtp_server,
self.body[:128]))
# TODO(dzimine): handle utf-8, http://stackoverflow.com/a/14506784
message = text.MIMEText(self.body)
message['Subject'] = self.subject
message['From'] = self.sender
message['To'] = self.to
try:
s = smtplib.SMTP(self.smtp_server)
if self.password is not None:
# Sequence to request TLS connection and log in (RFC-2487).
s.ehlo()
s.starttls()
s.ehlo()
s.login(self.sender, self.password)
s.sendmail(from_addr=self.sender,
to_addrs=self.to,
msg=message.as_string())
except (smtplib.SMTPException, IOError) as e:
raise exc.ActionException("Failed to send an email message: %s" %
e)
def run(self, context):
LOG.info(
"Sending email message "
"[from=%s, to=%s, subject=%s, using smtp=%s, body=%s...]",
self.sender, self.to, self.subject, self.smtp_server,
self.body[:128])
message = text.MIMEText(self.body, _charset='utf-8')
message['Subject'] = header.Header(self.subject, 'utf-8')
message['From'] = self.sender
message['To'] = ', '.join(self.to)
try:
s = smtplib.SMTP(self.smtp_server)
if self.password is not None:
# Sequence to request TLS connection and log in (RFC-2487).
s.ehlo()
s.starttls()
s.ehlo()
s.login(self.sender, self.password)
s.sendmail(from_addr=self.sender,
to_addrs=self.to,
msg=message.as_string())
except (smtplib.SMTPException, IOError) as e:
raise exc.ActionException("Failed to send an email message: %s" %
e)
def _create_adhoc_action(db_task):
task_spec = tasks.TaskSpec(db_task['task_spec'])
full_action_name = task_spec.get_full_action_name()
# TODO(rakhmerov): Fix model attributes during refactoring.
raw_action_spec = db_task['action_spec']
if not raw_action_spec:
return None
action_spec = actions.ActionSpec(raw_action_spec)
LOG.info('Using ad-hoc action [action=%s, db_task=%s]' %
(full_action_name, db_task))
# Create an ad-hoc action.
base_cls = get_action_class(action_spec.clazz)
action_context = None
if _has_action_context_param(base_cls):
action_context = _get_action_context(db_task)
if not base_cls:
msg = 'Ad-hoc action base class is not registered ' \
'[workbook_name=%s, action=%s, base_class=%s]' % \
(db_task['workbook_name'], full_action_name, base_cls)
raise exc.ActionException(msg)
action_params = db_task['parameters'] or {}
return std_actions.AdHocAction(action_context,
base_cls,
action_spec,
**action_params)
def run(self, context):
LOG.info(
"Running HTTP action "
"[url=%s, method=%s, params=%s, body=%s, headers=%s,"
" cookies=%s, auth=%s, timeout=%s, allow_redirects=%s,"
" proxies=%s, verify=%s]", self.url, self.method, self.params,
self.body, self.headers, self.cookies, self.auth, self.timeout,
self.allow_redirects, self.proxies, self.verify)
try:
resp = requests.request(self.method,
self.url,
params=self.params,
data=self.body,
headers=self.headers,
cookies=self.cookies,
auth=self.auth,
timeout=self.timeout,
allow_redirects=self.allow_redirects,
proxies=self.proxies,
verify=self.verify)
except Exception as e:
LOG.exception(
"Failed to send HTTP request for action execution: %s",
context.execution.action_execution_id)
raise exc.ActionException("Failed to send HTTP request: %s" % e)
LOG.info("HTTP action response:\n%s\n%s", resp.status_code,
resp.content)
# TODO(akuznetsova): Need to refactor Mistral serialiser and
# deserializer to have an ability to pass needed encoding and work
# with it. Now it can process only default 'utf-8' encoding.
# Appropriate bug #1676411 was created.
# Represent important resp data as a dictionary.
try:
content = resp.json(encoding=resp.encoding)
except Exception as e:
LOG.debug("HTTP action response is not json.")
content = resp.content
if content and resp.encoding not in (None, 'utf-8'):
content = content.decode(resp.encoding).encode('utf-8')
_result = {
'content': content,
'status': resp.status_code,
'headers': dict(resp.headers.items()),
'url': resp.url,
'history': resp.history,
'encoding': resp.encoding,
'reason': resp.reason,
'cookies': dict(resp.cookies.items()),
'elapsed': resp.elapsed.total_seconds()
}
if resp.status_code not in range(200, 307):
return actions.Result(error=_result)
return _result
def run(self):
resp = super(GetImageIdAction, self).run()
created_image_id = resp['content'].get('created_image_id')
if not uuidutils.is_uuid_like(created_image_id):
raise exceptions.ActionException(
'Image not built yet: image %(created_image_id)s,'
' state %(state)s' % resp['content'])
return resp['content']
def run(self):
resp = super(CheckStackAction, self).run()
LOG.debug('CheckStackAction resp:%s' % resp['content'].get('stack'))
if resp['content']['stack']['stack_status'] != 'UPDATE_COMPLETE':
raise exceptions.ActionException(
'Stack update not complete: stack %(stack_name)s/%(id)s,'
' status %(stack_status)s' % resp['content']['stack'])
return {'stack_status': resp['content']['stack']['stack_status']}
def raise_exc(parent_exc=None):
message = ("Failed to execute ssh cmd "
"'%s' on %s" % (self.cmd, self.host))
# We suppress the actual parent error messages in favor of
# more generic ones as we might be leaking information to the CLI
if parent_exc:
# The full error message needs to be logged regardless
LOG.exception(message + " Exception: %s", str(parent_exc))
raise exc.ActionException(message)
def run(self):
try:
method = self._get_client_method(self._get_client())
result = method(**self._kwargs_for_run)
if inspect.isgenerator(result):
return [v for v in result]
return result
except Exception as e:
raise exc.ActionException("%s failed: %s" %
(self.__class__.__name__, e))
def run(self, context):
try:
script = """function f() {
%s
}
f()
""" % self.script
return javascript.evaluate(script, self.js_context)
except Exception as e:
raise exc.ActionException("JavaScriptAction failed: %s" % str(e))
def run(self):
try:
method = self._get_client_method(self._get_client())
return method(**self._kwargs_for_run)
except Exception as e:
e_str = '%s: %s' % (type(e), e.message)
raise exc.ActionException(
"%s.%s failed: %s" %
(self.__class__.__name__, self.client_method_name, e_str))
def __init__(self, ref, parameters=None, st2_context=None):
if not st2_context or not st2_context.get('api_url'):
raise exc.ActionException(
'Failed to initialize %s [ref=%s]: Invalid st2 context.' %
(self.__class__.__name__, ref))
self.ref = ref
self.parameters = parameters
self.st2_context = st2_context
self.st2_context_log_safe = copy.deepcopy(st2_context)
self.st2_context_log_safe.pop('auth_token', None)
def run(self):
LOG.info("Running HTTP action "
"[url=%s, method=%s, params=%s, body=%s, headers=%s,"
" cookies=%s, auth=%s, timeout=%s, allow_redirects=%s,"
" proxies=%s, verify=%s]" %
(self.url, self.method, self.params, self.body, self.headers,
self.cookies, self.auth, self.timeout, self.allow_redirects,
self.proxies, self.verify))
try:
resp = requests.request(self.method,
self.url,
params=self.params,
data=self.body,
headers=self.headers,
cookies=self.cookies,
auth=self.auth,
timeout=self.timeout,
allow_redirects=self.allow_redirects,
proxies=self.proxies,
verify=self.verify)
except Exception as e:
raise exc.ActionException("Failed to send HTTP request: %s" % e)
LOG.info("HTTP action response:\n%s\n%s" %
(resp.status_code, resp.content))
# Represent important resp data as a dictionary.
try:
content = resp.json()
except Exception as e:
LOG.debug("HTTP action response is not json.")
content = resp.content
_result = {
'content': content,
'status': resp.status_code,
'headers': dict(resp.headers.items()),
'url': resp.url,
'history': resp.history,
'encoding': resp.encoding,
'reason': resp.reason,
'cookies': resp.cookies,
'elapsed': resp.elapsed
}
if resp.status_code not in range(200, 307):
return wf_utils.Result(error=_result)
return _result
def run(self, context):
LOG.info(
"Sending email message "
"[from=%s, to=%s, reply_to=%s, cc=%s, bcc=%s, subject=%s, "
"using smtp=%s, body=%s...]",
self.sender,
self.to,
self.reply_to,
self.cc,
self.bcc,
self.subject,
self.smtp_server,
self.body[:128]
)
if not self.html_body:
message = text.MIMEText(self.body, _charset='utf-8')
else:
message = multipart.MIMEMultipart('alternative')
message.attach(text.MIMEText(self.body,
'plain',
_charset='utf-8'))
message.attach(text.MIMEText(self.html_body,
'html',
_charset='utf-8'))
message['Subject'] = header.Header(self.subject, 'utf-8')
message['From'] = self.sender
message['Reply-To'] = header.Header(', '.join(self.reply_to))
message['To'] = ', '.join(self.to)
if self.cc:
message['cc'] = ', '.join(self.cc)
rcpt = self.cc + self.bcc + self.to
try:
s = smtplib.SMTP(self.smtp_server)
if self.password is not None:
# Sequence to request TLS connection and log in (RFC-2487).
s.ehlo()
s.starttls()
s.ehlo()
s.login(self.sender, self.password)
s.sendmail(from_addr=self.sender,
to_addrs=rcpt,
msg=message.as_string())
except (smtplib.SMTPException, IOError) as e:
raise exc.ActionException("Failed to send an email message: %s"
% e)
def run(self):
try:
method = self._get_client_method(self._get_client())
result = method(**self._kwargs_for_run)
if inspect.isgenerator(result):
return [v for v in result]
return result
except Exception as e:
e_str = '%s: %s' % (type(e), e.message)
raise exc.ActionException(
"%s.%s failed: %s" %
(self.__class__.__name__, self.client_method_name, e_str)
)
def get_action_class(action_full_name):
"""Finds action class by full action name (i.e. 'namespace.action_name').
:param action_full_name: Full action name (that includes namespace).
:return: Action class or None if not found.
"""
arr = action_full_name.split('.')
if len(arr) != 2:
raise exc.ActionException('Invalid action name: %s' %
action_full_name)
ns = _NAMESPACES.get(arr[0])
if not ns:
return None
return ns.get_action_class(arr[1])
def run(self, context):
try:
method = self._get_client_method(self._get_client(context))
result = method(**self._kwargs_for_run)
if inspect.isgenerator(result):
return [v for v in result]
return result
except Exception as e:
# Print the traceback for the last exception so that we can see
# where the issue comes from.
LOG.warning(traceback.format_exc())
raise exc.ActionException(
"%s.%s failed: %s" %
(self.__class__.__name__, self.client_method_name, str(e)))
def __init__(self,
url,
method="GET",
params=None,
body=None,
json=None,
headers=None,
cookies=None,
auth=None,
timeout=None,
allow_redirects=None,
proxies=None,
verify=None):
super(HTTPAction, self).__init__()
if auth and len(auth.split(':')) == 2:
self.auth = (auth.split(':')[0], auth.split(':')[1])
else:
self.auth = auth
if isinstance(headers, dict):
for key, val in headers.items():
if isinstance(val, (six.integer_types, float)):
headers[key] = str(val)
if body and json:
raise exc.ActionException(
"Only one of the parameters 'json' and 'body' can be passed")
self.url = url
self.method = method
self.params = params
self.body = utils.to_json_str(body) if isinstance(body, dict) else body
self.json = json
self.headers = headers
self.cookies = cookies
self.timeout = timeout
self.allow_redirects = allow_redirects
self.proxies = proxies
self.verify = verify
class DirectWorkflowRerunTest(base.EngineTestCase):
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]))
def test_rerun(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_2_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(2, len(task_2_action_exs))
self.assertEqual(states.ERROR, task_2_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_2_action_exs[1].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_3_ex.id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 10', # Mock task10 success for first run.
exc.ActionException(), # Mock task21 exception for first run.
'Task 21', # Mock task21 success for rerun.
'Task 22', # Mock task22 success.
'Task 30' # Mock task30 success.
]))
def test_rerun_diff_env_vars(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK_DIFF_ENV_VAR)
# Initial environment variables for the workflow execution.
env = {
'var1': 'fee fi fo fum',
'var2': 'mirror mirror',
'var3': 'heigh-ho heigh-ho'
}
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {}, env=env)
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
self.assertDictEqual(env, wf_ex.params['env'])
self.assertDictEqual(env, wf_ex.context['__env'])
task_10_ex = self._assert_single_item(task_execs, name='t10')
task_21_ex = self._assert_single_item(task_execs, name='t21')
task_30_ex = self._assert_single_item(task_execs, name='t30')
self.assertEqual(states.SUCCESS, task_10_ex.state)
self.assertEqual(states.ERROR, task_21_ex.state)
self.assertIsNotNone(task_21_ex.state_info)
self.assertEqual(states.ERROR, task_30_ex.state)
# Update env in workflow execution with the following.
updated_env = {'var1': 'Task 21', 'var2': 'Task 22', 'var3': 'Task 30'}
# Resume workflow and re-run failed task.
wf_ex = self.engine.rerun_workflow(task_21_ex.id, env=updated_env)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertDictEqual(updated_env, wf_ex.params['env'])
self.assertDictEqual(updated_env, wf_ex.context['__env'])
# Await t30 success.
self.await_task_success(task_30_ex.id)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(4, len(task_execs))
task_10_ex = self._assert_single_item(task_execs, name='t10')
task_21_ex = self._assert_single_item(task_execs, name='t21')
task_22_ex = self._assert_single_item(task_execs, name='t22')
task_30_ex = self._assert_single_item(task_execs, name='t30')
# Check action executions of task 10.
self.assertEqual(states.SUCCESS, task_10_ex.state)
task_10_action_exs = db_api.get_action_executions(
task_execution_id=task_10_ex.id)
self.assertEqual(1, len(task_10_action_exs))
self.assertEqual(states.SUCCESS, task_10_action_exs[0].state)
self.assertDictEqual({'output': 'Task 10'},
task_10_action_exs[0].input)
# Check action executions of task 21.
self.assertEqual(states.SUCCESS, task_21_ex.state)
self.assertIsNone(task_21_ex.state_info)
task_21_action_exs = db_api.get_action_executions(
task_execution_id=task_21_ex.id)
self.assertEqual(2, len(task_21_action_exs))
self.assertEqual(states.ERROR, task_21_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_21_action_exs[1].state)
self.assertDictEqual({'output': env['var1']},
task_21_action_exs[0].input)
self.assertDictEqual({'output': updated_env['var1']},
task_21_action_exs[1].input)
# Check action executions of task 22.
self.assertEqual(states.SUCCESS, task_22_ex.state)
task_22_action_exs = db_api.get_action_executions(
task_execution_id=task_22_ex.id)
self.assertEqual(1, len(task_22_action_exs))
self.assertEqual(states.SUCCESS, task_22_action_exs[0].state)
self.assertDictEqual({'output': updated_env['var2']},
task_22_action_exs[0].input)
# Check action executions of task 30.
self.assertEqual(states.SUCCESS, task_30_ex.state)
task_30_action_exs = db_api.get_action_executions(
task_execution_id=task_30_ex.id)
self.assertEqual(1, len(task_30_action_exs))
self.assertEqual(states.SUCCESS, task_30_action_exs[0].state)
self.assertDictEqual({'output': updated_env['var3']},
task_30_action_exs[0].input)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException() # Mock task2 exception for initial run.
]))
def test_rerun_from_prev_step(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs,
name='t1',
state=states.SUCCESS)
task_2_ex = self._assert_single_item(task_execs,
name='t2',
state=states.ERROR)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
e = self.assertRaises(exc.MistralError, self.engine.rerun_workflow,
task_1_ex.id)
self.assertIn('not supported', str(e))
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.0', # Mock task1 success for rerun.
'Task 1.2', # Mock task1 success for rerun.
'Task 2' # Mock task2 success.
]))
def test_rerun_with_items(self):
wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb3.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(1, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.assertEqual(states.ERROR, task_1_ex.state)
self.assertIsNotNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(3, len(task_1_action_exs))
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_1_ex.id, reset=False)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_success(wf_ex.id, delay=10)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertIsNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
# The single action execution that succeeded should not re-run.
self.assertEqual(5, len(task_1_action_exs))
self.assertListEqual(['Task 1.0', 'Task 1.1', 'Task 1.2'],
task_1_ex.published.get('v1'))
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
@testtools.skip('Restore concurrency support.')
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.3', # Mock task1 success for initial run.
'Task 1.0', # Mock task1 success for rerun.
'Task 1.2', # Mock task1 success for rerun.
'Task 2' # Mock task2 success.
]))
def test_rerun_with_items_concurrency(self):
wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_CONCURRENCY)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb3.wf1', {})
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(1, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
self.assertEqual(states.ERROR, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(4, len(task_1_action_exs))
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_1_ex.id, reset=False)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_success(wf_ex.id, delay=10)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(2, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertIsNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
# The action executions that succeeded should not re-run.
self.assertEqual(6, len(task_1_action_exs))
self.assertListEqual(['Task 1.0', 'Task 1.1', 'Task 1.2', 'Task 1.3'],
task_1_ex.published.get('v1'))
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.0', # Mock task1 success for rerun.
'Task 1.2', # Mock task1 success for rerun.
'Task 2' # Mock task2 success.
]))
def test_rerun_with_items_diff_env_vars(self):
wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_DIFF_ENV_VAR)
# Initial environment variables for the workflow execution.
env = {'var1': 'fee fi fo fum'}
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb3.wf1', {}, env=env)
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(1, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.assertEqual(states.ERROR, task_1_ex.state)
self.assertIsNotNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(3, len(task_1_action_exs))
# Update env in workflow execution with the following.
updated_env = {'var1': 'foobar'}
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_1_ex.id, reset=False, env=updated_env)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_success(wf_ex.id, delay=10)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertIsNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
expected_inputs = [
'Task 1.0 [%s]' % env['var1'], # Task 1 item 0 (error).
'Task 1.1 [%s]' % env['var1'], # Task 1 item 1.
'Task 1.2 [%s]' % env['var1'], # Task 1 item 2 (error).
'Task 1.0 [%s]' % updated_env['var1'], # Task 1 item 0 (rerun).
'Task 1.2 [%s]' % updated_env['var1'] # Task 1 item 2 (rerun).
]
# Assert that every expected input is in actual task input.
for action_ex in task_1_action_exs:
self.assertIn(action_ex.input['output'], expected_inputs)
# Assert that there was same number of unique inputs as action execs.
self.assertEqual(
len(task_1_action_exs),
len(
set([
action_ex.input['output']
for action_ex in task_1_action_exs
])))
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 1', # Mock task1 success for initial run.
'Task 2', # Mock task2 success for initial run.
exc.ActionException(), # Mock task3 exception for initial run.
'Task 3' # Mock task3 success for rerun.
]))
def test_rerun_on_join_task(self):
wb_service.create_workbook_v2(JOIN_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertEqual(states.ERROR, task_3_ex.state)
self.assertIsNotNone(task_3_ex.state_info)
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_3_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
self.assertEqual(states.SUCCESS, task_2_action_exs[0].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
self.assertIsNone(task_3_ex.state_info)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_execs[2].id)
self.assertEqual(2, len(task_3_action_exs))
self.assertEqual(states.ERROR, task_3_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_3_action_exs[1].state)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task1 exception for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 1', # Mock task1 success for rerun.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]))
def test_rerun_join_with_branch_errors(self):
wb_service.create_workbook_v2(JOIN_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
self.await_task_error(task_1_ex.id)
self.await_task_error(task_2_ex.id)
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.assertEqual(states.ERROR, task_1_ex.state)
self.assertIsNotNone(task_1_ex.state_info)
task_2_ex = self._assert_single_item(task_execs, name='t2')
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
wf_ex = self.engine.rerun_workflow(task_1_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
# Wait for the task to succeed.
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.await_task_success(task_1_ex.id)
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertEqual(states.ERROR, task_3_ex.state)
# Resume workflow and re-run failed task.
wf_ex = self.engine.rerun_workflow(task_2_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Join now should finally complete.
self.await_task_success(task_3_ex.id)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertIsNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(2, len(task_1_action_exs))
self.assertEqual(states.ERROR, task_1_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_1_action_exs[1].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(2, len(task_2_action_exs))
self.assertEqual(states.ERROR, task_2_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_2_action_exs[1].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_execs[2].id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task 1.0 error for run.
'Task 1.1', # Mock task 1.1 success for run.
exc.ActionException(), # Mock task 1.2 error for run.
exc.ActionException(), # Mock task 1.0 error for 1st rerun.
exc.ActionException(), # Mock task 1.2 error for 1st rerun.
exc.ActionException(), # Mock task 1.0 error for 2nd run.
'Task 1.1', # Mock task 1.1 success for 2nd run.
exc.ActionException(), # Mock task 1.2 error for 2nd run.
exc.ActionException(), # Mock task 1.0 error for 3rd rerun.
exc.ActionException(), # Mock task 1.2 error for 3rd rerun.
'Task 1.0', # Mock task 1.0 success for 4th rerun.
'Task 1.2', # Mock task 1.2 success for 4th rerun.
'Task 2' # Mock task 2 success.
]))
def test_multiple_reruns_with_items(self):
wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb3.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(1, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.await_task_error(task_1_ex.id)
self.assertIsNotNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(3, len(task_1_action_exs))
# Resume workflow and re-run failed task. Re-run #1 with no reset.
wf_ex = self.engine.rerun_workflow(task_1_ex.id, reset=False)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_error(wf_ex.id, delay=10)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(5, len(task_1_action_exs))
# Resume workflow and re-run failed task. Re-run #2 with reset.
self.engine.rerun_workflow(task_1_ex.id, reset=True)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_error(wf_ex.id, delay=10)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(8, len(task_1_action_exs))
# Resume workflow and re-run failed task. Re-run #3 with no reset.
self.engine.rerun_workflow(task_1_ex.id, reset=False)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_error(wf_ex.id, delay=10)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(10, len(task_1_action_exs))
# Resume workflow and re-run failed task. Re-run #4 with no reset.
self.engine.rerun_workflow(task_1_ex.id, reset=False)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.await_workflow_success(wf_ex.id, delay=10)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertIsNone(task_1_ex.state_info)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
# The single action execution that succeeded should not re-run.
self.assertEqual(12, len(task_1_action_exs))
self.assertListEqual(['Task 1.0', 'Task 1.1', 'Task 1.2'],
task_1_ex.published.get('v1'))
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]))
def test_rerun_subflow(self):
wb_service.create_workbook_v2(SUBFLOW_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_2_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_workflow_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(2, len(task_2_action_exs))
self.assertEqual(states.ERROR, task_2_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_2_action_exs[1].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_3_ex.id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]))
def test_rerun_subflow_task(self):
wb_service.create_workbook_v2(SUBFLOW_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
with db_api.transaction():
# Get subworkflow and related task
sub_wf_exs = db_api.get_workflow_executions(
task_execution_id=task_2_ex.id)
sub_wf_ex = sub_wf_exs[0]
sub_wf_task_execs = sub_wf_ex.task_executions
self.assertEqual(states.ERROR, sub_wf_ex.state)
self.assertIsNotNone(sub_wf_ex.state_info)
self.assertEqual(1, len(sub_wf_task_execs))
sub_wf_task_ex = self._assert_single_item(sub_wf_task_execs,
name='wf2_t1')
self.assertEqual(states.ERROR, sub_wf_task_ex.state)
self.assertIsNotNone(sub_wf_task_ex.state_info)
# Resume workflow and re-run failed subworkflow task.
self.engine.rerun_workflow(sub_wf_task_ex.id)
sub_wf_ex = db_api.get_workflow_execution(sub_wf_ex.id)
self.assertEqual(states.RUNNING, sub_wf_ex.state)
self.assertIsNone(sub_wf_ex.state_info)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Wait for the subworkflow to succeed.
self.await_workflow_success(sub_wf_ex.id)
with db_api.transaction():
sub_wf_ex = db_api.get_workflow_execution(sub_wf_ex.id)
sub_wf_task_execs = sub_wf_ex.task_executions
self.assertEqual(states.SUCCESS, sub_wf_ex.state)
self.assertIsNone(sub_wf_ex.state_info)
self.assertEqual(1, len(sub_wf_task_execs))
sub_wf_task_ex = self._assert_single_item(sub_wf_task_execs,
name='wf2_t1')
# Check action executions of subworkflow task.
self.assertEqual(states.SUCCESS, sub_wf_task_ex.state)
self.assertIsNone(sub_wf_task_ex.state_info)
sub_wf_task_ex_action_exs = db_api.get_action_executions(
task_execution_id=sub_wf_task_ex.id)
self.assertEqual(2, len(sub_wf_task_ex_action_exs))
self.assertEqual(states.ERROR, sub_wf_task_ex_action_exs[0].state)
self.assertEqual(states.SUCCESS, sub_wf_task_ex_action_exs[1].state)
# Wait for the main workflow to succeed.
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(task_execs))
task_1_ex = self._assert_single_item(task_execs, name='t1')
task_2_ex = self._assert_single_item(task_execs, name='t2')
task_3_ex = self._assert_single_item(task_execs, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_workflow_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(1, len(task_2_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_3_ex.id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
def run(self):
LOG.info('Running %s [action_context=%s, ref=%s, '
'parameters=%s, st2_context=%s]' %
(self.__class__.__name__, self.action_context, self.ref,
self.parameters, self.st2_context_log_safe))
method = 'POST'
endpoint = self.st2_context['endpoint']
st2_action_context = {
'parent': self.st2_context.get('parent'),
'mistral': self.action_context
}
headers = {
'content-type': 'application/json',
'st2-context': json.dumps(st2_action_context)
}
if 'auth_token' in self.st2_context:
headers['X-Auth-Token'] = self.st2_context.get('auth_token')
elif 'st2' in cfg.CONF and 'auth_token' in cfg.CONF.st2:
headers['X-Auth-Token'] = cfg.CONF.st2.auth_token
body = {
'action': self.ref,
'callback': {
'source': 'mistral',
'url': _build_callback_url(self.action_context)
}
}
notify = self.st2_context.get('notify', {})
skip_notify_tasks = self.st2_context.get('skip_notify_tasks', [])
task_name = self.action_context.get('task_name', 'unknown')
if task_name not in skip_notify_tasks and '*' not in skip_notify_tasks:
# We only include notifications settings if the task is not to be skipped
body['notify'] = notify
if self.parameters:
body['parameters'] = self.parameters
data = json.dumps(body) if isinstance(body, dict) else body
try:
resp = self.request(method, endpoint, headers, data)
except Exception as e:
raise exc.ActionException(
'Failed to send HTTP request for %s [action_context=%s, '
'ref=%s, parameters=%s, st2_context=%s]: %s' %
(self.__class__.__name__, self.action_context, self.ref,
self.parameters, self.st2_context_log_safe, e))
LOG.info('Received HTTP response for %s [action_context=%s, '
'ref=%s, parameters=%s, st2_context=%s]:\n%s\n%s' %
(self.__class__.__name__, self.action_context, self.ref,
self.parameters, self.st2_context_log_safe, resp.status_code,
resp.content))
try:
content = resp.json()
except Exception as e:
content = resp.content
result = {
'content': content,
'status': resp.status_code,
'headers': dict(resp.headers.items()),
'url': resp.url,
'history': resp.history,
'encoding': resp.encoding,
'reason': resp.reason,
'cookies': dict(resp.cookies.items()),
'elapsed': resp.elapsed.total_seconds()
}
if resp.status_code not in range(200, 307):
return wf_utils.Result(error=result)
return result
def run(self, context):
LOG.info(
"Running HTTP action "
"[url=%s, method=%s, params=%s, body=%s, headers=%s,"
" cookies=%s, auth=%s, timeout=%s, allow_redirects=%s,"
" proxies=%s, verify=%s]", self.url, self.method, self.params,
self.body, self.headers, self.cookies, self.auth, self.timeout,
self.allow_redirects, self.proxies, self.verify)
try:
url_data = six.moves.urllib.parse.urlsplit(self.url)
if 'https' == url_data.scheme:
action_verify = self.verify
else:
action_verify = None
resp = requests.request(self.method,
self.url,
params=self.params,
data=self.body,
headers=self.headers,
cookies=self.cookies,
auth=self.auth,
timeout=self.timeout,
allow_redirects=self.allow_redirects,
proxies=self.proxies,
verify=action_verify)
except Exception as e:
LOG.exception(
"Failed to send HTTP request for action execution: %s",
context.execution.action_execution_id)
raise exc.ActionException("Failed to send HTTP request: %s" % e)
LOG.info("HTTP action response:\n%s\n%s", resp.status_code,
resp.content)
# Represent important resp data as a dictionary.
try:
content = resp.json(encoding=resp.encoding)
except Exception as e:
LOG.debug("HTTP action response is not json.")
content = resp.content
if content and resp.encoding not in (None, 'utf-8'):
content = content.decode(resp.encoding).encode('utf-8')
_result = {
'content': content,
'status': resp.status_code,
'headers': dict(resp.headers.items()),
'url': resp.url,
'history': resp.history,
'encoding': resp.encoding,
'reason': resp.reason,
'cookies': dict(resp.cookies.items()),
'elapsed': resp.elapsed.total_seconds()
}
if resp.status_code not in range(200, 307):
return actions.Result(error=_result)
return _result
def test(self, context):
if self.error_data:
return actions.Result(error=self.error_data)
raise exc.ActionException('Fail action expected exception.')
class ExecutionStateInfoTest(base.EngineTestCase):
def test_state_info(self):
workflow = """---
version: '2.0'
test_wf:
type: direct
tasks:
task1:
action: std.fail
task2:
action: std.noop
"""
wf_service.create_workflows(workflow)
# Start workflow.
wf_ex = self.engine.start_workflow('test_wf', '', {})
self.await_workflow_error(wf_ex.id)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIn("error in tasks: task1", wf_ex.state_info)
def test_state_info_two_failed_branches(self):
workflow = """---
version: '2.0'
test_wf:
type: direct
tasks:
task1:
action: std.fail
task2:
action: std.fail
"""
wf_service.create_workflows(workflow)
# Start workflow.
wf_ex = self.engine.start_workflow('test_wf', '', {})
self.await_workflow_error(wf_ex.id)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIn("error in tasks: task1, task2", wf_ex.state_info)
def test_state_info_with_policies(self):
workflow = """---
version: '2.0'
test_wf:
type: direct
tasks:
task1:
action: std.fail
wait-after: 1
task2:
action: std.noop
wait-after: 3
"""
wf_service.create_workflows(workflow)
# Start workflow.
wf_ex = self.engine.start_workflow('test_wf', '', {})
self.await_workflow_error(wf_ex.id)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIn("error in tasks: task1", wf_ex.state_info)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(side_effect=[
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task1 exception for initial run.
'Task 1.0', # Mock task1 success for rerun.
'Task 1.2' # Mock task1 success for rerun.
]))
def test_state_info_with_items(self):
workflow = """---
version: '2.0'
wf:
type: direct
tasks:
t1:
with-items: i in <% list(range(0, 3)) %>
action: std.echo output="Task 1.<% $.i %>"
"""
wf_service.create_workflows(workflow)
wf_ex = self.engine.start_workflow('wf', '', {})
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.ERROR, wf_ex.state)
task_1_ex = self._assert_single_item(task_execs, name='t1')
self.assertEqual(states.ERROR, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(3, len(task_1_action_exs))
error_actions = [
action_ex for action_ex in task_1_action_exs
if action_ex.state == states.ERROR
]
self.assertEqual(2, len(error_actions))
success_actions = [
action_ex for action_ex in task_1_action_exs
if action_ex.state == states.SUCCESS
]
self.assertEqual(1, len(success_actions))
for action_ex in error_actions:
self.assertIn(action_ex.id, wf_ex.state_info)
for action_ex in success_actions:
self.assertNotIn(action_ex.id, wf_ex.state_info)
def raise_exc(parent_exc=None):
message = ("Failed to execute ssh cmd "
"'%s' on %s" % (self.cmd, self.host))
if parent_exc:
message += "\nException: %s" % str(parent_exc)
raise exc.ActionException(message)
class ReverseWorkflowRerunTest(base.EngineTestCase):
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(
side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]
)
)
def test_rerun(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3')
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_2_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
task_3_ex = self._assert_single_item(wf_ex.task_executions, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(2, len(task_2_action_exs))
self.assertEqual(states.ERROR, task_2_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_2_action_exs[1].state)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_3_ex.id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(
side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException(), # Mock task2 exception for initial run.
'Task 2', # Mock task2 success for rerun.
'Task 3' # Mock task3 success.
]
)
)
def test_rerun_diff_env_vars(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK_DIFF_ENV_VAR)
# Initial environment variables for the workflow execution.
env = {
'var1': 'fee fi fo fum',
'var2': 'foobar'
}
# Run workflow and fail task.
wf_ex = self.engine.start_workflow(
'wb1.wf1',
{},
task_name='t3',
env=env
)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(wf_ex.task_executions))
self.assertDictEqual(env, wf_ex.params['env'])
self.assertDictEqual(env, wf_ex.context['__env'])
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Update env in workflow execution with the following.
updated_env = {
'var1': 'Task 2',
'var2': 'Task 3'
}
# Resume workflow and re-run failed task.
self.engine.rerun_workflow(task_2_ex.id, env=updated_env)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertDictEqual(updated_env, wf_ex.params['env'])
self.assertDictEqual(updated_env, wf_ex.context['__env'])
# Wait for the workflow to succeed.
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsNone(wf_ex.state_info)
self.assertEqual(3, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
task_3_ex = self._assert_single_item(wf_ex.task_executions, name='t3')
# Check action executions of task 1.
self.assertEqual(states.SUCCESS, task_1_ex.state)
task_1_action_exs = db_api.get_action_executions(
task_execution_id=task_1_ex.id)
self.assertEqual(1, len(task_1_action_exs))
self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
self.assertDictEqual(
{'output': 'Task 1'},
task_1_action_exs[0].input
)
# Check action executions of task 2.
self.assertEqual(states.SUCCESS, task_2_ex.state)
self.assertIsNone(task_2_ex.state_info)
task_2_action_exs = db_api.get_action_executions(
task_execution_id=task_2_ex.id)
self.assertEqual(2, len(task_2_action_exs))
self.assertEqual(states.ERROR, task_2_action_exs[0].state)
self.assertEqual(states.SUCCESS, task_2_action_exs[1].state)
self.assertDictEqual(
{'output': env['var1']},
task_2_action_exs[0].input
)
self.assertDictEqual(
{'output': updated_env['var1']},
task_2_action_exs[1].input
)
# Check action executions of task 3.
self.assertEqual(states.SUCCESS, task_3_ex.state)
task_3_action_exs = db_api.get_action_executions(
task_execution_id=task_3_ex.id)
self.assertEqual(1, len(task_3_action_exs))
self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
self.assertDictEqual(
{'output': updated_env['var2']},
task_3_action_exs[0].input
)
@mock.patch.object(
std_actions.EchoAction,
'run',
mock.MagicMock(
side_effect=[
'Task 1', # Mock task1 success for initial run.
exc.ActionException() # Mock task2 exception for initial run.
]
)
)
def test_rerun_from_prev_step(self):
wb_service.create_workbook_v2(SIMPLE_WORKBOOK)
# Run workflow and fail task.
wf_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3')
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.ERROR, wf_ex.state)
self.assertIsNotNone(wf_ex.state_info)
self.assertEqual(2, len(wf_ex.task_executions))
task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1')
task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2')
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.ERROR, task_2_ex.state)
self.assertIsNotNone(task_2_ex.state_info)
# Resume workflow and re-run failed task.
e = self.assertRaises(
exc.MistralError,
self.engine.rerun_workflow,
task_1_ex.id
)
self.assertIn('not supported', str(e))
class PoliciesTest(base.EngineTestCase):
def setUp(self):
super(PoliciesTest, self).setUp()
self.wb_spec = spec_parser.get_workbook_spec_from_yaml(WORKBOOK)
self.wf_spec = self.wb_spec.get_workflows()['wf1']
self.task_spec = self.wf_spec.get_tasks()['task1']
def test_build_policies(self):
arr = policies.build_policies(self.task_spec.get_policies(),
self.wf_spec)
self.assertEqual(4, len(arr))
p = self._assert_single_item(arr, delay=2)
self.assertIsInstance(p, policies.WaitBeforePolicy)
p = self._assert_single_item(arr, delay=5)
self.assertIsInstance(p, policies.WaitAfterPolicy)
p = self._assert_single_item(arr, delay=10)
self.assertIsInstance(p, policies.RetryPolicy)
self.assertEqual(5, p.count)
self.assertEqual('<% $.my_val = 10 %>', p.break_on)
p = self._assert_single_item(arr, delay=7)
self.assertIsInstance(p, policies.TimeoutPolicy)
def test_task_policy_class(self):
policy = policies.base.TaskPolicy()
policy._schema = {"properties": {"delay": {"type": "integer"}}}
wf_ex = models.WorkflowExecution(id='1-2-3-4', context={}, input={})
task_ex = models.TaskExecution(in_context={'int_var': 5})
task_ex.workflow_execution = wf_ex
policy.delay = "<% $.int_var %>"
# Validation is ok.
policy.before_task_start(task_ex, None)
policy.delay = "some_string"
# Validation is failing now.
exception = self.assertRaises(exc.InvalidModelException,
policy.before_task_start, task_ex, None)
self.assertIn("Invalid data type in TaskPolicy", str(exception))
def test_build_policies_with_workflow_defaults(self):
wb_spec = spec_parser.get_workbook_spec_from_yaml(WB_WITH_DEFAULTS)
wf_spec = wb_spec.get_workflows()['wf1']
task_spec = wf_spec.get_tasks()['task1']
arr = policies.build_policies(task_spec.get_policies(), wf_spec)
self.assertEqual(4, len(arr))
p = self._assert_single_item(arr, delay=3)
self.assertIsInstance(p, policies.WaitBeforePolicy)
p = self._assert_single_item(arr, delay=5)
self.assertIsInstance(p, policies.WaitAfterPolicy)
p = self._assert_single_item(arr, delay=1)
self.assertIsInstance(p, policies.RetryPolicy)
self.assertEqual(2, p.count)
p = self._assert_single_item(arr, delay=7)
self.assertIsInstance(p, policies.TimeoutPolicy)
def test_wait_before_policy(self):
wb_service.create_workbook_v2(WAIT_BEFORE_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING_DELAYED, task_ex.state)
self.assertDictEqual({'wait_before_policy': {
'skip': True
}}, task_ex.runtime_context)
self.await_workflow_success(wf_ex.id)
def test_wait_before_policy_from_var(self):
wb_service.create_workbook_v2(WAIT_BEFORE_FROM_VAR)
# Start workflow.
exec_db = self.engine.start_workflow('wb.wf1', {'wait_before': 1})
# Note: We need to reread execution to access related tasks.
exec_db = db_api.get_workflow_execution(exec_db.id)
task_db = exec_db.task_executions[0]
self.assertEqual(states.RUNNING_DELAYED, task_db.state)
self.await_workflow_success(exec_db.id)
def test_wait_before_policy_two_tasks(self):
wf_text = """---
version: '2.0'
wf:
tasks:
a:
wait-before: 2
on-success: b
b:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf', {})
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(2, len(task_execs))
self._assert_multiple_items(task_execs, 2, state=states.SUCCESS)
def test_wait_after_policy(self):
wb_service.create_workbook_v2(WAIT_AFTER_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.assertDictEqual({}, task_ex.runtime_context)
self.await_task_delayed(task_ex.id, delay=0.5)
self.await_task_success(task_ex.id)
def test_wait_after_policy_from_var(self):
wb_service.create_workbook_v2(WAIT_AFTER_FROM_VAR)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {'wait_after': 2})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.assertDictEqual({}, task_ex.runtime_context)
# TODO(rakhmerov): This check doesn't make sense anymore because
# we don't store evaluated value anywhere.
# Need to create a better test.
# self.assertEqual(2, task_ex.in_context['wait_after'])
def test_retry_policy(self):
wb_service.create_workbook_v2(RETRY_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.assertDictEqual({}, task_ex.runtime_context)
self.await_task_delayed(task_ex.id, delay=0.5)
self.await_task_error(task_ex.id)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(
2, task_ex.runtime_context["retry_task_policy"]["retry_no"])
def test_retry_policy_from_var(self):
wb_service.create_workbook_v2(RETRY_WB_FROM_VAR)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {'count': 3, 'delay': 1})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.assertDictEqual({}, task_ex.runtime_context)
# TODO(rakhmerov): This check doesn't make sense anymore because
# we don't store evaluated values anywhere.
# Need to create a better test.
# self.assertEqual(3, task_ex.in_context["count"])
# self.assertEqual(1, task_ex.in_context["delay"])
def test_retry_policy_never_happen(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
tasks:
task1:
action: std.echo output="hello"
retry:
count: 3
delay: 1
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_success(task_ex.id)
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual({}, task_ex.runtime_context["retry_task_policy"])
def test_retry_policy_break_on(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
input:
- var: 4
tasks:
task1:
action: std.fail
retry:
count: 3
delay: 1
break-on: <% $.var >= 3 %>
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_error(task_ex.id)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual({}, task_ex.runtime_context["retry_task_policy"])
def test_retry_policy_break_on_not_happened(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
input:
- var: 2
tasks:
task1:
action: std.fail
retry:
count: 3
delay: 1
break-on: <% $.var >= 3 %>
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_error(task_ex.id)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(
2, task_ex.runtime_context['retry_task_policy']['retry_no'])
@mock.patch.object(std_actions.EchoAction, 'run',
mock.Mock(side_effect=[1, 2, 3, 4]))
def test_retry_continue_on(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
tasks:
task1:
action: std.echo output="mocked result"
retry:
count: 4
delay: 1
continue-on: <% task(task1).result < 3 %>
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_success(task_ex.id)
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(
2, task_ex.runtime_context['retry_task_policy']['retry_no'])
def test_retry_continue_on_not_happened(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
tasks:
task1:
action: std.echo output=4
retry:
count: 4
delay: 1
continue-on: <% task(task1).result <= 3 %>
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_success(task_ex.id)
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual({}, task_ex.runtime_context['retry_task_policy'])
def test_retry_policy_one_line(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
type: direct
tasks:
task1:
action: std.fail
retry: count=3 delay=1
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_error(task_ex.id)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(
2, task_ex.runtime_context['retry_task_policy']['retry_no'])
def test_retry_policy_subworkflow_force_fail(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
main:
tasks:
task1:
workflow: work
retry:
count: 3
delay: 1
work:
tasks:
do:
action: std.fail
on-error:
- fail
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.main', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_error(task_ex.id)
self.await_workflow_error(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(
2, task_ex.runtime_context['retry_task_policy']['retry_no'])
@mock.patch.object(
std_actions.EchoAction, 'run',
mock.Mock(side_effect=[exc.ActionException(), "mocked result"]))
def test_retry_policy_succeed_after_failure(self):
retry_wb = """---
version: '2.0'
name: wb
workflows:
wf1:
output:
result: <% task(task1).result %>
tasks:
task1:
action: std.echo output="mocked result"
retry:
count: 3
delay: 1
"""
wb_service.create_workbook_v2(retry_wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_success(task_ex.id)
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertDictEqual({'retry_no': 1},
task_ex.runtime_context['retry_task_policy'])
self.assertDictEqual({'result': 'mocked result'}, wf_ex.output)
def test_timeout_policy(self):
wb_service.create_workbook_v2(TIMEOUT_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.await_task_error(task_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self._assert_single_item(wf_ex.task_executions, name='task1')
self.await_workflow_success(wf_ex.id)
def test_timeout_policy_success_after_timeout(self):
wb_service.create_workbook_v2(TIMEOUT_WB2)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
self.await_workflow_error(wf_ex.id)
# Wait until timeout exceeds.
self._sleep(1)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
tasks_db = wf_ex.task_executions
# Make sure that engine did not create extra tasks.
self.assertEqual(1, len(tasks_db))
def test_timeout_policy_from_var(self):
wb_service.create_workbook_v2(TIMEOUT_FROM_VAR)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {'timeout': 1})
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(states.RUNNING, task_ex.state)
# TODO(rakhmerov): This check doesn't make sense anymore because
# we don't store evaluated 'timeout' value anywhere.
# Need to create a better test.
# self.assertEqual(1, task_ex.in_context['timeout'])
def test_pause_before_policy(self):
wb_service.create_workbook_v2(PAUSE_BEFORE_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
self.assertEqual(states.IDLE, task_ex.state)
self.await_workflow_paused(wf_ex.id)
self._sleep(1)
self.engine.resume_workflow(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self._assert_single_item(wf_ex.task_executions, name='task1')
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
next_task_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.SUCCESS, task_ex.state)
self.assertEqual(states.SUCCESS, next_task_ex.state)
def test_pause_before_with_delay_policy(self):
wb_service.create_workbook_v2(PAUSE_BEFORE_DELAY_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
self.assertEqual(states.IDLE, task_ex.state)
# Verify wf paused by pause-before
self.await_workflow_paused(wf_ex.id)
# Allow wait-before to expire
self._sleep(2)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Verify wf still paused (wait-before didn't reactivate)
self.await_workflow_paused(wf_ex.id)
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.IDLE, task_ex.state)
self.engine.resume_workflow(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self._assert_single_item(wf_ex.task_executions, name='task1')
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
next_task_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.SUCCESS, task_ex.state)
self.assertEqual(states.SUCCESS, next_task_ex.state)
def test_concurrency_is_in_runtime_context(self):
wb_service.create_workbook_v2(CONCURRENCY_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
self.assertEqual(states.SUCCESS, task_ex.state)
self.assertEqual(4, task_ex.runtime_context['concurrency'])
def test_concurrency_is_in_runtime_context_from_var(self):
wb_service.create_workbook_v2(CONCURRENCY_WB_FROM_VAR)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {'concurrency': 4})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
self.assertEqual(4, task_ex.runtime_context['concurrency'])
def test_wrong_policy_prop_type(self):
wb = """---
version: "2.0"
name: wb
workflows:
wf1:
type: direct
input:
- wait_before
tasks:
task1:
action: std.echo output="Hi!"
wait-before: <% $.wait_before %>
"""
wb_service.create_workbook_v2(wb)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {'wait_before': '1'})
self.assertIn('Invalid data type in WaitBeforePolicy',
wf_ex.state_info)
self.assertEqual(states.ERROR, wf_ex.state)
def test_delayed_task_and_correct_finish_workflow(self):
wf_delayed_state = """---
version: "2.0"
wf:
type: direct
tasks:
task1:
action: std.noop
wait-before: 1
task2:
action: std.noop
"""
wf_service.create_workflows(wf_delayed_state)
# Start workflow.
wf_ex = self.engine.start_workflow('wf', {})
self.await_workflow_success(wf_ex.id)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(2, len(wf_ex.task_executions))
SOURCE_WF_EX['source_execution_id'] = WF_EX.id
SOURCE_WF_EX['id'] = uuidutils.generate_uuid()
SOURCE_WF_EX_JSON_WITH_DESC = copy.deepcopy(WF_EX_JSON_WITH_DESC)
SOURCE_WF_EX_JSON_WITH_DESC['id'] = SOURCE_WF_EX.id
SOURCE_WF_EX_JSON_WITH_DESC['source_execution_id'] = \
SOURCE_WF_EX.source_execution_id
MOCK_WF_EX = mock.MagicMock(return_value=WF_EX)
MOCK_SUB_WF_EX = mock.MagicMock(return_value=SUB_WF_EX)
MOCK_SOURCE_WF_EX = mock.MagicMock(return_value=SOURCE_WF_EX)
MOCK_WF_EXECUTIONS = mock.MagicMock(return_value=[WF_EX])
MOCK_UPDATED_WF_EX = mock.MagicMock(return_value=UPDATED_WF_EX)
MOCK_DELETE = mock.MagicMock(return_value=None)
MOCK_EMPTY = mock.MagicMock(return_value=[])
MOCK_NOT_FOUND = mock.MagicMock(side_effect=exc.DBEntityNotFoundError())
MOCK_ACTION_EXC = mock.MagicMock(side_effect=exc.ActionException())
@mock.patch.object(rpc_base, '_IMPL_CLIENT', mock.Mock())
class TestExecutionsController(base.APITest):
@mock.patch.object(db_api, 'get_workflow_execution', MOCK_WF_EX)
def test_get(self):
resp = self.app.get('/v2/executions/123')
self.assertEqual(200, resp.status_int)
self.assertDictEqual(WF_EX_JSON_WITH_DESC, resp.json)
@mock.patch.object(db_api, 'get_workflow_execution')
def test_get_operational_error(self, mocked_get):
mocked_get.side_effect = [
# Emulating DB OperationalError