def _validate_name(self):
task_name = self._data.get('name')
if task_name in RESERVED_TASK_NAMES:
raise exc.InvalidModelException(
"Reserved keyword '%s' not allowed as task name." % task_name)
if len(task_name) > MAX_LENGTH_TASK_NAME:
raise exc.InvalidModelException(
"The length of a '{0}' task name must not exceed {1}"
" symbols".format(task_name, MAX_LENGTH_TASK_NAME))
def put(self, identifier=None):
"""Update one or more workflows.
:param identifier: Optional. If provided, it's UUID of a workflow.
Only one workflow can be updated with identifier param.
The text is allowed to have definitions of multiple workflows. In this
case they all will be updated.
"""
acl.enforce('workflows:update', context.ctx())
definition = pecan.request.text
scope = pecan.request.GET.get('scope', 'private')
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
LOG.info("Update workflow(s) [definition=%s]", definition)
db_wfs = workflows.update_workflows(definition,
scope=scope,
identifier=identifier)
models_dicts = [db_wf.to_dict() for db_wf in db_wfs]
workflow_list = [
resources.Workflow.from_dict(wf) for wf in models_dicts
]
return (workflow_list[0].to_json() if identifier else
resources.Workflows(workflows=workflow_list).to_json())
def validate_semantics(self):
super(WorkflowSpec, self).validate_semantics()
# Distinguish workflow name from workflow UUID.
if uuidutils.is_uuid_like(self._name):
raise exc.InvalidModelException(
"Workflow name cannot be in the format of UUID.")
def _parse_cmd_and_input(cmd_str):
# TODO(rakhmerov): Try to find a way with one expression.
cmd_matcher = CMD_PTRN.search(cmd_str)
if not cmd_matcher:
msg = "Invalid action/workflow task property: %s" % cmd_str
raise exc.InvalidModelException(msg)
cmd = cmd_matcher.group()
params = {}
for match in re.findall(PARAMS_PTRN, cmd_str):
k = match[0]
# Remove embracing quotes.
v = match[1].strip()
if v[0] == '"' or v[0] == "'":
v = v[1:-1]
else:
try:
v = json.loads(v)
except Exception:
pass
params[k] = v
return cmd, params
def _check_join_tasks(self):
join_tasks = [t for t in self.get_tasks() if t.get_join()]
err_msgs = []
for join_t in join_tasks:
t_name = join_t.get_name()
join_val = join_t.get_join()
in_tasks = self.find_inbound_task_specs(join_t)
if join_val == 'all':
if len(in_tasks) == 0:
err_msgs.append(
"No inbound tasks for task with 'join: all'"
" [task_name=%s]" % t_name)
continue
if join_val == 'one':
join_val = 1
if len(in_tasks) < join_val:
err_msgs.append("Not enough inbound tasks for task with 'join'"
" [task_name=%s, join=%s, inbound_tasks=%s]" %
(t_name, join_val, len(in_tasks)))
if len(err_msgs) > 0:
raise exc.InvalidModelException('\n'.join(err_msgs))
def put(self, identifier=None):
"""Update one or more actions.
:param identifier: Optional. If provided, it's UUID or name of an
action. Only one action can be updated with identifier param.
NOTE: This text is allowed to have definitions
of multiple actions. In this case they all will be updated.
"""
acl.enforce('actions:update', context.ctx())
definition = pecan.request.text
LOG.debug("Update action(s) [definition=%s]", definition)
scope = pecan.request.GET.get('scope', 'private')
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
with db_api.transaction():
db_acts = actions.update_actions(definition,
scope=scope,
identifier=identifier)
action_list = [
resources.Action.from_db_model(db_act) for db_act in db_acts
]
return resources.Actions(actions=action_list).to_json()
def post(self):
"""Create a new workflow.
NOTE: The text is allowed to have definitions
of multiple workflows. In this case they all will be created.
"""
acl.enforce('workflows:create', context.ctx())
definition = pecan.request.text
scope = pecan.request.GET.get('scope', 'private')
pecan.response.status = 201
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
LOG.info("Create workflow(s) [definition=%s]", definition)
db_wfs = workflows.create_workflows(definition, scope=scope)
models_dicts = [db_wf.to_dict() for db_wf in db_wfs]
workflow_list = [
resources.Workflow.from_dict(wf) for wf in models_dicts
]
return resources.Workflows(workflows=workflow_list).to_json()
def _validate_name(self):
task_name = self._data.get('name')
if len(task_name) > MAX_LENGTH_TASK_NAME:
raise exc.InvalidModelException(
"The length of a '{0}' task name must not exceed {1}"
" symbols".format(task_name, MAX_LENGTH_TASK_NAME))
def post(self, namespace=''):
"""Create a new workflow.
NOTE: The text is allowed to have definitions
of multiple workflows. In this case they all will be created.
:param namespace: Optional. The namespace to create the workflow
in. Workflows with the same name can be added to a given
project if are in two different namespaces.
"""
acl.enforce('workflows:create', context.ctx())
definition = pecan.request.text
scope = pecan.request.GET.get('scope', 'private')
pecan.response.status = 201
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
LOG.debug("Create workflow(s) [definition=%s]", definition)
db_wfs = rest_utils.rest_retry_on_db_error(workflows.create_workflows)(
definition, scope=scope, namespace=namespace)
workflow_list = [
resources.Workflow.from_db_model(db_wf) for db_wf in db_wfs
]
return resources.Workflows(workflows=workflow_list).to_json()
def validate(self):
if not all(k in self._data for k in self._required_keys):
message = ("Wrong model definition for: %s. It should contain"
" required keys: %s" %
(self.__class__.__name__, self._required_keys))
raise exceptions.InvalidModelException(message)
return True
def put(self, identifier=None):
"""Update one or more actions.
NOTE: This text is allowed to have definitions
of multiple actions. In this case they all will be updated.
"""
acl.enforce('actions:update', context.ctx())
definition = pecan.request.text
LOG.info("Update action(s) [definition=%s]", definition)
scope = pecan.request.GET.get('scope', 'private')
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
with db_api.transaction():
db_acts = actions.update_actions(definition,
scope=scope,
identifier=identifier)
models_dicts = [db_act.to_dict() for db_act in db_acts]
action_list = [resources.Action.from_dict(act) for act in models_dicts]
return resources.Actions(actions=action_list).to_json()
def post(self):
"""Create a new action.
NOTE: This text is allowed to have definitions
of multiple actions. In this case they all will be created.
"""
acl.enforce('actions:create', context.ctx())
definition = pecan.request.text
scope = pecan.request.GET.get('scope', 'private')
pecan.response.status = 201
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
LOG.info("Create action(s) [definition=%s]", definition)
with db_api.transaction():
db_acts = actions.create_actions(definition, scope=scope)
models_dicts = [db_act.to_dict() for db_act in db_acts]
action_list = [resources.Action.from_dict(act) for act in models_dicts]
return resources.Actions(actions=action_list).to_json()
def validate_schema(self):
super(BaseListSpec, self).validate_schema()
if len(self._data.keys()) < 2:
raise exc.InvalidModelException(
'At least one item must be in the list [data=%s].' %
self._data)
def _validate_task_link(self, task_name, allow_engine_cmds=True):
valid_task = self._task_exists(task_name)
if allow_engine_cmds:
valid_task |= task_name in tasks.RESERVED_TASK_NAMES
if not valid_task:
raise exc.InvalidModelException("Task '%s' not found." % task_name)
def _validate_task_link(self, task_name, allow_engine_cmds=True):
valid_task = self._task_exists(task_name)
if allow_engine_cmds:
valid_task |= task_name in ENGINE_COMMANDS
if not valid_task:
raise exc.InvalidModelException("Task '%s' not found." % task_name)
def validate_cron_trigger_input(pattern, first_time, count):
if not (first_time or pattern):
raise exc.InvalidModelException("Pattern or first_execution_time must"
" be specified.")
if first_time:
if (datetime.datetime.now() + datetime.timedelta(0, 60)) > first_time:
raise exc.InvalidModelException("First_execution_time must be at"
" least one minute in the future.")
if not pattern and count > 1:
raise exc.InvalidModelException("Pattern must be provided if count"
" is superior to 1.")
if pattern:
try:
croniter(pattern)
except (ValueError, KeyError):
raise exc.InvalidModelException("The specified pattern is not"
" valid: {}".format(pattern))
def create_cron_trigger(name, workflow_name, workflow_input,
workflow_params=None, pattern=None, first_time=None,
count=None, start_time=None, workflow_id=None):
if not start_time:
start_time = datetime.datetime.now()
if isinstance(first_time, six.string_types):
try:
first_time = datetime.datetime.strptime(
first_time,
'%Y-%m-%d %H:%M'
)
except ValueError as e:
raise exc.InvalidModelException(e.message)
validate_cron_trigger_input(pattern, first_time, count)
first_utc_time = first_time
if first_time:
first_second = time.mktime(first_time.timetuple())
first_utc_time = datetime.datetime.utcfromtimestamp(first_second)
next_time = first_utc_time
if not (pattern or count):
count = 1
else:
next_time = get_next_execution_time(pattern, start_time)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(
workflow_id if workflow_id else workflow_name
)
eng_utils.validate_input(
wf_def,
workflow_input or {},
parser.get_workflow_spec(wf_def.spec)
)
values = {
'name': name,
'pattern': pattern,
'first_execution_time': first_utc_time,
'next_execution_time': next_time,
'remaining_executions': count,
'workflow_name': wf_def.name,
'workflow_id': wf_def.id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'scope': 'private'
}
security.add_trust_id(values)
trig = db_api.create_cron_trigger(values)
return trig
def validate_semantics(self):
if not self._branch and not self._global and not self._atomic:
raise exc.InvalidModelException(
"Either 'branch', 'global' or 'atomic' must be specified: " %
self._data)
self.validate_expr(self._branch)
self.validate_expr(self._global)
self.validate_expr(self._atomic)
def validate_name_has_no_spaces(name):
"""Makes sure name does not contain spaces."""
if name:
if " " in name:
msg = ("Name '{}' must not contain spaces").format(name)
LOG.error(msg)
raise exc.InvalidModelException(msg)
def instantiate_spec(spec_cls, data):
"""Instantiates specification accounting for specification hierarchies.
:param spec_cls: Specification concrete or base class. In case if base
class or the hierarchy is provided this method relies on attributes
_polymorphic_key and _polymorphic_value in order to find a concrete
class that needs to be instantiated.
:param data: Raw specification data as a dictionary.
"""
if issubclass(spec_cls, BaseSpecList):
# Ignore polymorphic search for specification lists because
# it doesn't make sense for them.
return spec_cls(data)
if not hasattr(spec_cls, '_polymorphic_key'):
spec = spec_cls(data)
spec.validate_semantics()
return spec
# In order to do polymorphic search we need to make sure that
# a spec is backed by a dictionary. Otherwise we can't extract
# a polymorphic key.
if not isinstance(data, dict):
raise exc.InvalidModelException(
"A specification with polymorphic key must be backed by"
" a dictionary [spec_cls=%s, data=%s]" % (spec_cls, data))
key = spec_cls._polymorphic_key
if not isinstance(key, tuple):
key_name = key
key_default = None
else:
key_name = key[0]
key_default = key[1]
for cls in utils.iter_subclasses(spec_cls):
if not hasattr(cls, '_polymorphic_value'):
raise exc.DSLParsingException(
"Class '%s' is expected to have attribute '_polymorphic_value'"
" because it's a part of specification hierarchy inherited "
"from class '%s'." % (cls, spec_cls))
if cls._polymorphic_value == data.get(key_name, key_default):
spec = cls(data)
spec.validate_semantics()
return spec
raise exc.DSLParsingException(
'Failed to find a specification class to instantiate '
'[spec_cls=%s, data=%s]' % (spec_cls, data))
def validate_schema(self):
super(WorkflowSpec, self).validate_schema()
if not self._data.get('tasks'):
raise exc.InvalidModelException(
"Workflow doesn't have any tasks [data=%s]" % self._data)
# Validate expressions.
self.validate_expr(self._data.get('output', {}))
self.validate_expr(self._data.get('vars', {}))
def validate_cron_trigger_input(pattern, first_time, count):
if not (first_time or pattern):
raise exc.InvalidModelException(
'Pattern or first_execution_time must be specified.')
if first_time:
valid_min_time = datetime.datetime.utcnow() + datetime.timedelta(0, 60)
if valid_min_time > first_time:
raise exc.InvalidModelException(
'first_execution_time must be at least 1 minute in the future.'
)
if not pattern and count and count > 1:
raise exc.InvalidModelException(
'Pattern must be provided if count is superior to 1.')
if pattern:
try:
croniter.croniter(pattern)
except (ValueError, KeyError):
raise exc.InvalidModelException(
'The specified pattern is not valid: {}'.format(pattern))
def _validate(self):
"""Validation of types after YAQL evaluation."""
props = inspect_utils.get_public_fields(self)
try:
jsonschema.validate(props, self._schema)
except Exception as e:
raise exc.InvalidModelException(
"Invalid data type in %s: %s. Value(s) can be shown after "
"YAQL evaluating. If you use YAQL here, please correct it." %
(self.__class__.__name__, str(e)))
def _get_with_items_as_dict(self):
raw = self._data.get('with-items', [])
with_items = {}
if isinstance(raw, str):
raw = [raw]
for item in raw:
if not isinstance(item, str):
raise exc.InvalidModelException(
"'with-items' elements should be strings: %s" % self._data)
match = re.match(WITH_ITEMS_PTRN, item)
if not match:
raise exc.InvalidModelException(
"Wrong format of 'with-items' property. Please use "
"format 'var in {[some, list] | <%% $.array %%> }: "
"%s" % self._data)
match_groups = match.groups()
var_name = match_groups[0]
array = match_groups[1]
# Validate YAQL expression that may follow after "in" for the
# with-items syntax "var in {[some, list] | <% $.array %> }".
self.validate_expr(array)
if array.startswith('['):
try:
array = json.loads(array)
except Exception as e:
msg = ("Invalid array in 'with-items' clause: "
"%s, error: %s" % (array, str(e)))
raise exc.InvalidModelException(msg)
with_items[var_name] = array
return with_items
def validate_semantics(self):
# Validate YAQL expressions.
self._validate_transitions(self._on_complete)
self._validate_transitions(self._on_success)
self._validate_transitions(self._on_error)
if self._join:
join_task_name = self.get_name()
if len(join_task_name) > MAX_LENGTH_JOIN_TASK_NAME:
raise exc.InvalidModelException(
"The length of a '{0}' join task name must not exceed {1} "
"symbols".format(join_task_name,
MAX_LENGTH_JOIN_TASK_NAME))
def validate(self):
super(DirectWorkflowTaskSpec, self).validate()
if 'join' in self._data:
join = self._data.get('join')
if not (isinstance(join, int) or join in ['all', 'one']):
msg = ("Task property 'join' is only allowed to be an"
" integer, 'all' or 'one': %s" % self._data)
raise exc.InvalidModelException(msg)
# Validate YAQL expressions.
self._validate_transitions('on-complete')
self._validate_transitions('on-success')
self._validate_transitions('on-error')
def validate_schema(self):
"""Validates DSL entity schema that this specification represents.
By default, this method just validate schema of DSL entity that this
specification represents using "_schema" class attribute.
Additionally, child classes may implement additional logic to validate
more specific things like YAQL expressions in their fields.
Note that this method is called before construction of specification
fields and validation logic should only rely on raw data provided as
a dictionary accessible through '_data' instance field.
"""
try:
jsonschema.validate(self._data, self.get_schema())
except jsonschema.ValidationError as e:
raise exc.InvalidModelException("Invalid DSL: %s" % e)
def validate(self):
super(TaskSpec, self).validate()
action = self._data.get('action')
workflow = self._data.get('workflow')
if action and workflow:
msg = ("Task properties 'action' and 'workflow' can't be"
" specified both: %s" % self._data)
raise exc.InvalidModelException(msg)
self._transform_with_items()
# Validate YAQL expressions.
if action or workflow:
inline_params = self._parse_cmd_and_input(action or workflow)[1]
self.validate_yaql_expr(inline_params)
self.validate_yaql_expr(self._data.get('input', {}))
self.validate_yaql_expr(self._data.get('publish', {}))
self.validate_yaql_expr(self._data.get('keep-result', {}))
def put(self):
"""Update one or more actions.
NOTE: This text is allowed to have definitions
of multiple actions. In this case they all will be updated.
"""
definition = pecan.request.text
LOG.info("Update action(s) [definition=%s]" % definition)
scope = pecan.request.GET.get('scope', 'private')
if scope not in SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (SCOPE_TYPES.values, scope))
db_acts = actions.update_actions(definition, scope=scope)
models_dicts = [db_act.to_dict() for db_act in db_acts]
action_list = [Action.from_dict(act) for act in models_dicts]
return Actions(actions=action_list).to_string()
def put(self, identifier=None, namespace=''):
"""Update one or more workflows.
:param identifier: Optional. If provided, it's UUID of a workflow.
Only one workflow can be updated with identifier param.
:param namespace: Optional. If provided int's the namespace of the
workflow/workflows. currently namespace cannot be
changed.
The text is allowed to have definitions of multiple workflows. In this
case they all will be updated.
"""
acl.enforce('workflows:update', context.ctx())
definition = pecan.request.text
scope = pecan.request.GET.get('scope', 'private')
if scope not in resources.SCOPE_TYPES.values:
raise exc.InvalidModelException(
"Scope must be one of the following: %s; actual: "
"%s" % (resources.SCOPE_TYPES.values, scope))
LOG.debug("Update workflow(s) [definition=%s]", definition)
db_wfs = rest_utils.rest_retry_on_db_error(workflows.update_workflows)(
definition,
scope=scope,
identifier=identifier,
namespace=namespace)
workflow_list = [
resources.Workflow.from_db_model(db_wf) for db_wf in db_wfs
]
return (workflow_list[0].to_json() if identifier else
resources.Workflows(workflows=workflow_list).to_json())
