class MistralSecureModelBase(MistralModelBase):
"""Base class for all secure models."""
__abstract__ = True
scope = sa.Column(sa.String(80), default='private')
project_id = sa.Column(sa.String(80), default=security.get_project_id)
created_at = sa.Column(sa.DateTime, default=lambda: utils.utc_now_sec())
updated_at = sa.Column(sa.DateTime, onupdate=lambda: utils.utc_now_sec())
def handle_expired_actions():
LOG.debug("Running heartbeat checker...")
interval = CONF.action_heartbeat.check_interval
max_missed = CONF.action_heartbeat.max_missed_heartbeats
exp_date = utils.utc_now_sec() - datetime.timedelta(seconds=max_missed *
interval)
try:
with db_api.transaction():
action_exs = db_api.get_running_expired_sync_actions(exp_date)
LOG.debug("Found {} running and expired actions.".format(
len(action_exs)))
if action_exs:
LOG.info("Actions executions to transit to error, because "
"heartbeat wasn't received: {}".format(action_exs))
for action_ex in action_exs:
result = mistral_lib.Result(
error="Heartbeat wasn't received.")
action_handler.on_action_complete(action_ex, result)
finally:
schedule(interval)
def _schedule_call(factory_method_path, target_method_name,
run_after, serializers=None, key=None, **method_args):
"""Schedules call and lately invokes target_method.
Add this call specification to DB, and then after run_after
seconds service CallScheduler invokes the target_method.
:param factory_method_path: Full python-specific path to
factory method that creates a target object that the call will be
made against.
:param target_method_name: Name of a method which will be invoked.
:param run_after: Value in seconds.
:param serializers: map of argument names and their serializer class
paths. Use when an argument is an object of specific type, and needs
to be serialized. Example:
{ "result": "mistral.utils.serializer.ResultSerializer"}
Serializer for the object type must implement serializer interface
in mistral/utils/serializer.py
:param key: Key which can potentially be used for squashing similar
delayed calls.
:param method_args: Target method keyword arguments.
"""
ctx_serializer = context.RpcContextSerializer()
ctx = (
ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {}
)
execution_time = (utils.utc_now_sec() +
datetime.timedelta(seconds=run_after))
if serializers:
for arg_name, serializer_path in serializers.items():
if arg_name not in method_args:
raise exc.MistralException(
"Serializable method argument %s"
" not found in method_args=%s"
% (arg_name, method_args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError(
"Cannot import class %s: %s" % (serializer_path, e)
)
method_args[arg_name] = serializer.serialize(method_args[arg_name])
values = {
'factory_method_path': factory_method_path,
'target_method_name': target_method_name,
'execution_time': execution_time,
'auth_context': ctx,
'serializers': serializers,
'key': key,
'method_arguments': method_args,
'processing': False
}
db_api.create_delayed_call(values)
def get_scheduled_jobs_to_start(time, batch_size=None, session=None):
query = b.model_query(models.ScheduledJob)
execute_at_col = models.ScheduledJob.execute_at
captured_at_col = models.ScheduledJob.captured_at
# Filter by execution time accounting for a configured job pickup interval.
query = query.filter(
execute_at_col <
time - datetime.timedelta(seconds=CONF.scheduler.pickup_job_after)
)
# Filter by captured time accounting for a configured captured job timeout.
min_captured_at = (
utils.utc_now_sec() -
datetime.timedelta(seconds=CONF.scheduler.captured_job_timeout)
)
query = query.filter(
sa.or_(
captured_at_col == sa.null(),
captured_at_col <= min_captured_at
)
)
query = query.order_by(execute_at_col)
query = query.limit(batch_size)
return query.all()
def handle_expired_actions():
LOG.debug("Running heartbeat checker...")
interval = CONF.action_heartbeat.check_interval
max_missed = CONF.action_heartbeat.max_missed_heartbeats
exp_date = utils.utc_now_sec() - datetime.timedelta(
seconds=max_missed * interval
)
with db_api.transaction():
action_exs = db_api.get_running_expired_sync_action_executions(
exp_date,
CONF.action_heartbeat.batch_size
)
LOG.debug("Found {} running and expired actions.".format(
len(action_exs))
)
if action_exs:
LOG.info(
"Actions executions to transit to error, because "
"heartbeat wasn't received: {}".format(action_exs)
)
for action_ex in action_exs:
result = mistral_lib.Result(
error="Heartbeat wasn't received."
)
action_handler.on_action_complete(action_ex, result)
def schedule_call(factory_method_path, target_method_name,
run_after, serializers=None, key=None, **method_args):
"""Schedules call and lately invokes target_method.
Add this call specification to DB, and then after run_after
seconds service CallScheduler invokes the target_method.
:param factory_method_path: Full python-specific path to
factory method that creates a target object that the call will be
made against.
:param target_method_name: Name of a method which will be invoked.
:param run_after: Value in seconds.
:param serializers: map of argument names and their serializer class
paths. Use when an argument is an object of specific type, and needs
to be serialized. Example:
{ "result": "mistral.utils.serializer.ResultSerializer"}
Serializer for the object type must implement serializer interface
in mistral/utils/serializer.py
:param key: Key which can potentially be used for squashing similar
delayed calls.
:param method_args: Target method keyword arguments.
"""
ctx_serializer = context.RpcContextSerializer()
ctx = (
ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {}
)
execution_time = (utils.utc_now_sec() +
datetime.timedelta(seconds=run_after))
if serializers:
for arg_name, serializer_path in serializers.items():
if arg_name not in method_args:
raise exc.MistralException(
"Serializable method argument %s"
" not found in method_args=%s"
% (arg_name, method_args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError(
"Cannot import class %s: %s" % (serializer_path, e)
)
method_args[arg_name] = serializer.serialize(method_args[arg_name])
values = {
'factory_method_path': factory_method_path,
'target_method_name': target_method_name,
'execution_time': execution_time,
'auth_context': ctx,
'serializers': serializers,
'key': key,
'method_arguments': method_args,
'processing': False
}
db_api.create_delayed_call(values)
def upgrade():
op.add_column(
'action_executions_v2',
Column('last_heartbeat',
DateTime,
default=lambda: utils.utc_now_sec() + datetime.timedelta(
seconds=CONF.action_heartbeat.first_heartbeat_timeout)))
op.add_column('action_executions_v2',
Column('is_sync', Boolean, default=None, nullable=True))
def upgrade():
op.add_column(
'action_executions_v2',
Column(
'last_heartbeat',
DateTime,
default=lambda: utils.utc_now_sec() + datetime.timedelta(
seconds=CONF.action_heartbeat.first_heartbeat_timeout
)
)
)
op.add_column(
'action_executions_v2',
Column('is_sync', Boolean, default=None, nullable=True)
)
def report_running_actions(self, action_ex_ids):
with db_api.transaction():
now = u.utc_now_sec()
for exec_id in action_ex_ids:
try:
db_api.update_action_execution(exec_id,
{"last_heartbeat": now},
insecure=True)
except exceptions.DBEntityNotFoundError:
LOG.debug(
"Action execution heartbeat update failed. {}".format(
exec_id),
exc_info=True)
# Ignore this error and continue with the
# remaining ids.
pass
def report_running_actions(self, action_ex_ids):
with db_api.transaction():
now = u.utc_now_sec()
for exec_id in action_ex_ids:
try:
db_api.update_action_execution(
exec_id,
{"last_heartbeat": now},
insecure=True
)
except exceptions.DBEntityNotFoundError:
LOG.debug("Action execution heartbeat update failed. {}"
.format(exec_id), exc_info=True)
# Ignore this error and continue with the
# remaining ids.
pass
def test_filter_tasks_without_task_execution(self, task_execution_result,
task_executions):
task_execution_result.return_value = 'task_execution_result'
time_now = utils.utc_now_sec()
task = type("obj", (object,), {
'id': 'id',
'name': 'name',
'published': 'published',
'result': task_execution_result(),
'spec': 'spec',
'state': 'state',
'state_info': 'state_info',
'type': 'type',
'workflow_execution_id': 'workflow_execution_id',
'created_at': time_now,
'updated_at': time_now + datetime.timedelta(seconds=1),
})()
task_executions.return_value = [task]
ctx = {
'__task_execution': None,
'__execution': {
'id': 'some'
}
}
result = self._evaluator.evaluate('tasks(some)', ctx)
self.assertEqual(1, len(result))
self.assertDictEqual({
'id': task.id,
'name': task.name,
'published': task.published,
'result': task.result,
'spec': task.spec,
'state': task.state,
'state_info': task.state_info,
'type': task.type,
'workflow_execution_id': task.workflow_execution_id,
'created_at': task.created_at.isoformat(' '),
'updated_at': task.updated_at.isoformat(' ')
}, result[0])
class ActionExecution(Execution):
"""Contains action execution information."""
__tablename__ = 'action_executions_v2'
__table_args__ = (sa.Index('%s_project_id' % __tablename__, 'project_id'),
sa.Index('%s_scope' % __tablename__, 'scope'),
sa.Index('%s_state' % __tablename__, 'state'),
sa.Index('%s_updated_at' % __tablename__, 'updated_at'))
# Main properties.
accepted = sa.Column(sa.Boolean(), default=False)
input = sa.Column(st.JsonLongDictType(), nullable=True)
output = sa.orm.deferred(sa.Column(st.JsonLongDictType(), nullable=True))
last_heartbeat = sa.Column(
sa.DateTime,
default=lambda: utils.utc_now_sec() + datetime.timedelta(
seconds=CONF.action_heartbeat.first_heartbeat_timeout))
is_sync = sa.Column(sa.Boolean(), default=None, nullable=True)
def _persist_job(cls, job):
ctx_serializer = context.RpcContextSerializer()
ctx = (
ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {}
)
execute_at = (utils.utc_now_sec() +
datetime.timedelta(seconds=job.run_after))
args = job.func_args
arg_serializers = job.func_arg_serializers
if arg_serializers:
for arg_name, serializer_path in arg_serializers.items():
if arg_name not in args:
raise exc.MistralException(
"Serializable function argument %s"
" not found in func_args=%s"
% (arg_name, args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError(
"Cannot import class %s: %s" % (serializer_path, e)
)
args[arg_name] = serializer.serialize(args[arg_name])
values = {
'run_after': job.run_after,
'target_factory_func_name': job.target_factory_func_name,
'func_name': job.func_name,
'func_args': args,
'func_arg_serializers': arg_serializers,
'auth_ctx': ctx,
'execute_at': execute_at,
'captured_at': None
}
return db_api.create_scheduled_job(values)
def _process_store_jobs(self):
# Select and capture eligible jobs.
with db_api.transaction():
candidate_jobs = db_api.get_scheduled_jobs_to_start(
utils.utc_now_sec(),
self._batch_size
)
captured_jobs = [
job for job in candidate_jobs
if self._capture_scheduled_job(job)
]
# Invoke and delete scheduled jobs.
for job in captured_jobs:
auth_ctx, func, func_args = self._prepare_job(job)
self._invoke_job(auth_ctx, func, func_args)
self._delete_scheduled_job(job)
def test_filter_tasks_without_task_execution(self, task_execution_result,
task_executions):
task_execution_result.return_value = 'task_execution_result'
time_now = utils.utc_now_sec()
task = type(
"obj", (object, ), {
'id': 'id',
'name': 'name',
'published': 'published',
'result': task_execution_result(),
'spec': 'spec',
'state': 'state',
'state_info': 'state_info',
'type': 'type',
'workflow_execution_id': 'workflow_execution_id',
'created_at': time_now,
'updated_at': time_now + datetime.timedelta(seconds=1),
})()
task_executions.return_value = [task]
ctx = {'__task_execution': None, '__execution': {'id': 'some'}}
result = self._evaluator.evaluate('tasks(some)', ctx)
self.assertEqual(1, len(result))
self.assertDictEqual(
{
'id': task.id,
'name': task.name,
'published': task.published,
'result': task.result,
'spec': task.spec,
'state': task.state,
'state_info': task.state_info,
'type': task.type,
'workflow_execution_id': task.workflow_execution_id,
'created_at': task.created_at.isoformat(' '),
'updated_at': task.updated_at.isoformat(' ')
}, result[0])
def _capture_scheduled_job(scheduled_job):
"""Capture a scheduled persistent job in a job store.
:param scheduled_job: Job.
:return: True if the job has been captured, False if not.
"""
# Mark this job as captured in order to prevent calling from
# a parallel transaction. We don't use query filter
# {'captured_at': None} to account for a case when the job needs
# to be recaptured after a maximum capture time has elapsed. If this
# method was called for job that has non-empty "captured_at" then
# it means that it is already eligible for recapturing and the
# Job Store selected it.
_, updated_cnt = db_api.update_scheduled_job(
id=scheduled_job.id,
values={'captured_at': utils.utc_now_sec()},
query_filter={'captured_at': scheduled_job.captured_at}
)
# If updated_cnt != 1 then another scheduler
# has already updated it.
return updated_cnt == 1
def _capture_calls(batch_size):
"""Captures delayed calls eligible for processing (based on time).
The intention of this method is to select delayed calls based on time
criteria and mark them in DB as being processed so that no other
threads could process them in parallel.
:return: A list of delayed calls captured for further processing.
"""
result = []
time_filter = utils.utc_now_sec() + datetime.timedelta(seconds=1)
with db_api.transaction():
candidates = db_api.get_delayed_calls_to_start(
time_filter,
batch_size
)
for call in candidates:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
db_call, updated_cnt = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False}
)
# If updated_cnt != 1 then another scheduler
# has already updated it.
if updated_cnt == 1:
result.append(db_call)
LOG.debug("Scheduler captured %s delayed calls.", len(result))
return result
def _capture_calls(batch_size):
"""Captures delayed calls eligible for processing (based on time).
The intention of this method is to select delayed calls based on time
criteria and mark them in DB as being processed so that no other
threads could process them in parallel.
:return: A list of delayed calls captured for further processing.
"""
result = []
time_filter = utils.utc_now_sec() + datetime.timedelta(seconds=1)
with db_api.transaction():
candidates = db_api.get_delayed_calls_to_start(
time_filter,
batch_size
)
for call in candidates:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
db_call, updated_cnt = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False}
)
# If updated_cnt != 1 then another scheduler
# has already updated it.
if updated_cnt == 1:
result.append(db_call)
LOG.debug("Scheduler captured %s delayed calls.", len(result))
return result
def get_scheduled_jobs_to_start(time, batch_size=None, session=None):
query = b.model_query(models.ScheduledJob)
execute_at_col = models.ScheduledJob.execute_at
captured_at_col = models.ScheduledJob.captured_at
# Filter by execution time accounting for a configured job pickup interval.
query = query.filter(
execute_at_col < time -
datetime.timedelta(seconds=CONF.scheduler.pickup_job_after))
# Filter by captured time accounting for a configured captured job timeout.
min_captured_at = (
utils.utc_now_sec() -
datetime.timedelta(seconds=CONF.scheduler.captured_job_timeout))
query = query.filter(
sa.or_(captured_at_col == sa.null(),
captured_at_col <= min_captured_at))
query = query.order_by(execute_at_col)
query = query.limit(batch_size)
return query.all()
def save_finished_time(self, value='default'):
if not self.task_ex:
return
time = value if value is not 'default' else utils.utc_now_sec()
self.task_ex.finished_at = time
def _create_workflow_executions():
time_now = utils.utc_now_sec()
wf_execs = [{
'id': 'success_expired',
'name': 'success_expired',
'created_at': time_now - datetime.timedelta(minutes=60),
'updated_at': time_now - datetime.timedelta(minutes=59),
'workflow_name': 'test_exec',
'state': "SUCCESS",
}, {
'id': 'error_expired',
'name': 'error_expired',
'created_at': time_now - datetime.timedelta(days=3, minutes=10),
'updated_at': time_now - datetime.timedelta(days=3),
'workflow_name': 'test_exec',
'state': "ERROR",
}, {
'id': 'running_not_expired',
'name': 'running_not_expired',
'created_at': time_now - datetime.timedelta(days=3, minutes=10),
'updated_at': time_now - datetime.timedelta(days=3),
'workflow_name': 'test_exec',
'state': "RUNNING",
}, {
'id': 'running_not_expired2',
'name': 'running_not_expired2',
'created_at': time_now - datetime.timedelta(days=3, minutes=10),
'updated_at': time_now - datetime.timedelta(days=4),
'workflow_name': 'test_exec',
'state': "RUNNING",
}, {
'id': 'success_not_expired',
'name': 'success_not_expired',
'created_at': time_now - datetime.timedelta(minutes=15),
'updated_at': time_now - datetime.timedelta(minutes=5),
'workflow_name': 'test_exec',
'state': "SUCCESS",
}, {
'id': 'abc',
'name': 'cancelled_expired',
'created_at': time_now - datetime.timedelta(minutes=60),
'updated_at': time_now - datetime.timedelta(minutes=59),
'workflow_name': 'test_exec',
'state': "CANCELLED",
}, {
'id': 'cancelled_not_expired',
'name': 'cancelled_not_expired',
'created_at': time_now - datetime.timedelta(minutes=15),
'updated_at': time_now - datetime.timedelta(minutes=6),
'workflow_name': 'test_exec',
'state': "CANCELLED",
}]
for wf_exec in wf_execs:
db_api.create_workflow_execution(wf_exec)
# Create a nested workflow execution.
db_api.create_task_execution({
'id': 'running_not_expired',
'workflow_execution_id': 'success_not_expired',
'name': 'my_task'
})
db_api.create_workflow_execution({
'id':
'expired_but_not_a_parent',
'name':
'expired_but_not_a_parent',
'created_at':
time_now - datetime.timedelta(days=15),
'updated_at':
time_now - datetime.timedelta(days=10),
'workflow_name':
'test_exec',
'state':
"SUCCESS",
'task_execution_id':
'running_not_expired'
})
class _MistralModelBase(oslo_models.ModelBase, oslo_models.TimestampMixin):
"""Base class for all Mistral SQLAlchemy DB Models."""
created_at = sa.Column(sa.DateTime, default=lambda: utils.utc_now_sec())
updated_at = sa.Column(sa.DateTime, onupdate=lambda: utils.utc_now_sec())
__table__ = None
__hash__ = object.__hash__
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def __eq__(self, other):
if type(self) is not type(other):
return False
for col in self.__table__.columns:
# In case of single table inheritance a class attribute
# corresponding to a table column may not exist so we need
# to skip these attributes.
if (hasattr(self, col.name) and hasattr(other, col.name)
and getattr(self, col.name) != getattr(other, col.name)):
return False
return True
def __ne__(self, other):
return not self.__eq__(other)
def to_dict(self):
"""sqlalchemy based automatic to_dict method."""
d = {col_name: col_val for col_name, col_val in self.iter_columns()}
utils.datetime_to_str_in_dict(d, 'created_at')
utils.datetime_to_str_in_dict(d, 'updated_at')
return d
def iter_column_names(self):
"""Returns an iterator for loaded column names.
:return: A generator function for column names.
"""
# If a column is unloaded at this point, it is
# probably deferred. We do not want to access it
# here and thereby cause it to load.
unloaded = attributes.instance_state(self).unloaded
for col in self.__table__.columns:
if col.name not in unloaded and hasattr(self, col.name):
yield col.name
def iter_columns(self):
"""Returns an iterator for loaded columns.
:return: A generator function that generates
tuples (column name, column value).
"""
for col_name in self.iter_column_names():
yield col_name, getattr(self, col_name)
def get_clone(self):
"""Clones current object, loads all fields and returns the result."""
m = self.__class__()
for col in self.__table__.columns:
if hasattr(self, col.name):
setattr(m, col.name, getattr(self, col.name))
setattr(m, 'created_at',
utils.datetime_to_str(getattr(self, 'created_at')))
updated_at = getattr(self, 'updated_at')
# NOTE(nmakhotkin): 'updated_at' field is empty for just created
# object since it has not updated yet.
if updated_at:
setattr(m, 'updated_at', utils.datetime_to_str(updated_at))
return m
def __repr__(self):
return '%s %s' % (type(self).__name__, self.to_dict().__repr__())
def save_finished_time(self, value='default'):
if not self.task_ex:
return
time = value if value is not 'default' else utils.utc_now_sec()
self.task_ex.finished_at = time
