def _step_create():
for path in (self._book_path, self._flow_path, self._task_path):
try:
misc.ensure_tree(path)
except EnvironmentError as e:
raise exc.StorageError("Unable to create logbooks"
" required child path %s" % path, e)
def _destroy_logbook(self, session, lb_id):
try:
lb = _logbook_get_model(lb_id, session=session)
session.delete(lb)
except sa_exc.DBAPIError as e:
LOG.exception('Failed destroying logbook')
raise exc.StorageError("Failed destroying logbook %s" % lb_id, e)
def _make_session(self):
try:
return self._session_maker()
except sa_exc.SQLAlchemyError as e:
LOG.exception('Failed creating database session')
raise exc.StorageError("Failed creating database session: %s" % e,
e)
def close(self):
self._validated = False
if not self._owned:
return
try:
k_utils.finalize_client(self._client)
except (k_exc.KazooException, k_exc.ZookeeperError) as e:
raise exc.StorageError("Unable to finalize client", e)
def get_logbook(self, book_uuid):
session = self._make_session()
try:
lb = _logbook_get_model(book_uuid, session=session)
return _convert_lb_to_external(lb)
except sa_exc.DBAPIError as e:
LOG.exception('Failed getting logbook')
raise exc.StorageError("Failed getting logbook %s" % book_uuid, e)
def get_logbooks(self):
try:
books = list(self._get_logbooks())
except EnvironmentError as e:
raise exc.StorageError("Unable to fetch logbooks", e)
else:
for b in books:
yield b
def logbook_get(lb_id):
session = db_session.get_session()
try:
lb_m = _logbook_get_model(lb_id, session=session)
return _convert_lb_to_external(lb_m)
except sql_exc.DBAPIError as e:
raise exc.StorageError("Failed getting"
" logbook %s: %s" % (lb_id, e), e)
def _destroy_tasks(task_details):
for task_detail in task_details:
task_path = os.path.join(self._task_path, task_detail.uuid)
try:
shutil.rmtree(task_path)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise exc.StorageError("Unable to remove task"
" directory %s" % task_path, e)
def _clear_all(self, session):
# NOTE(harlowja): due to how we have our relationship setup and
# cascading deletes are enabled, this will cause all associated
# task details and flow details to automatically be purged.
try:
return session.query(models.LogBook).delete()
except sa_exc.DBAPIError as e:
LOG.exception('Failed clearing all entries')
raise exc.StorageError("Failed clearing all entries: %s" % e, e)
def logbook_destroy(lb_id):
session = db_session.get_session()
with session.begin():
try:
lb = _logbook_get_model(lb_id, session=session)
session.delete(lb)
except sql_exc.DBAPIError as e:
raise exc.StorageError(
"Failed destroying"
" logbook %s: %s" % (lb_id, e), e)
def clear_all():
session = db_session.get_session()
with session.begin():
# NOTE(harlowja): due to how we have our relationship setup and
# cascading deletes are enabled, this will cause all associated task
# details and flow details to automatically be purged.
try:
return session.query(models.LogBook).delete()
except sql_exc.DBAPIError as e:
raise exc.StorageError("Failed clearing all entries: %s" % e, e)
def get_logbooks(self):
session = self._make_session()
try:
raw_books = session.query(models.LogBook).all()
books = [_convert_lb_to_external(lb) for lb in raw_books]
except sa_exc.DBAPIError as e:
LOG.exception('Failed getting logbooks')
raise exc.StorageError("Failed getting logbooks: %s" % e, e)
for lb in books:
yield lb
def _destroy_book():
book = self._get_logbook(book_uuid)
book_path = os.path.join(self._book_path, book.uuid)
self._run_with_process_lock("flow", _destroy_flows, list(book))
try:
shutil.rmtree(book_path)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise exc.StorageError("Unable to remove book"
" directory %s" % book_path, e)
def _destroy_flows(flow_details):
for flow_detail in flow_details:
flow_path = os.path.join(self._flow_path, flow_detail.uuid)
self._run_with_process_lock("task", _destroy_tasks,
list(flow_detail))
try:
shutil.rmtree(flow_path)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise exc.StorageError("Unable to remove flow"
" directory %s" % flow_path, e)
def _run_with_process_lock(self, lock_name, functor, *args, **kwargs):
lock_path = os.path.join(self.backend.lock_path, lock_name)
with lock_utils.InterProcessLock(lock_path):
try:
return functor(*args, **kwargs)
except exc.TaskFlowException:
raise
except Exception as e:
LOG.exception("Failed running locking file based session")
# NOTE(harlowja): trap all other errors as storage errors.
raise exc.StorageError("Storage backend internal error", e)
def _run_in_session(self, functor, *args, **kwargs):
"""Runs a function in a session and makes sure that sqlalchemy
exceptions aren't emitted from that sessions actions (as that would
expose the underlying backends exception model).
"""
try:
session = self._make_session()
with session.begin():
return functor(session, *args, **kwargs)
except sa_exc.SQLAlchemyError as e:
LOG.exception('Failed running database session')
raise exc.StorageError("Storage backend internal error", e)
def upgrade(self):
try:
with contextlib.closing(self._engine.connect()) as conn:
# NOTE(imelnikov): Alembic does not support SQLite,
# and we don't recommend to use SQLite in production
# deployments, so migrations are rarely needed
# for SQLite. So we don't bother about working around
# SQLite limitations, and create database from models
# when it is in use.
if 'sqlite' in self._engine.url.drivername:
models.BASE.metadata.create_all(conn)
else:
migration.db_sync(conn)
except sa_exc.SQLAlchemyError as e:
LOG.exception('Failed upgrading database version')
raise exc.StorageError("Failed upgrading database version", e)
def _exc_wrapper(self):
"""Exception wrapper which wraps kazoo exceptions and groups them
to taskflow exceptions.
"""
try:
yield
except self._client.handler.timeout_exception as e:
raise exc.ConnectionFailure("Storage backend timeout: %s" % e)
except k_exc.SessionExpiredError as e:
raise exc.ConnectionFailure("Storage backend session"
" has expired: %s" % e)
except k_exc.NoNodeError as e:
raise exc.NotFound("Storage backend node not found: %s" % e)
except k_exc.NodeExistsError as e:
raise exc.AlreadyExists("Storage backend duplicate node: %s" % e)
except (k_exc.KazooException, k_exc.ZookeeperError) as e:
raise exc.StorageError("Storage backend internal error", e)
def _save_logbook(self, session, lb):
try:
lb_m = _logbook_get_model(lb.uuid, session=session)
# NOTE(harlowja): Merge them (note that this doesn't provide
# 100% correct update semantics due to how databases have
# MVCC). This is where a stored procedure or a better backing
# store would handle this better by allowing this merge logic
# to exist in the database itself.
lb_m = _logbook_merge(lb_m, lb)
except exc.NotFound:
lb_m = _convert_lb_to_internal(lb)
try:
lb_m = session.merge(lb_m)
return _convert_lb_to_external(lb_m)
except sa_exc.DBAPIError as e:
LOG.exception('Failed saving logbook')
raise exc.StorageError("Failed saving logbook %s" % lb.uuid, e)
def upgrade(self):
def _step_create():
for path in (self._book_path, self._flow_path, self._task_path):
try:
misc.ensure_tree(path)
except EnvironmentError as e:
raise exc.StorageError("Unable to create logbooks"
" required child path %s" % path, e)
for path in (self._backend.base_path, self._backend.lock_path):
try:
misc.ensure_tree(path)
except EnvironmentError as e:
raise exc.StorageError("Unable to create logbooks required"
" path %s" % path, e)
self._run_with_process_lock("init", _step_create)
def logbook_save(lb):
session = db_session.get_session()
with session.begin():
try:
lb_m = _logbook_get_model(lb.uuid, session=session)
# NOTE(harlowja): Merge them (note that this doesn't provide 100%
# correct update semantics due to how databases have MVCC). This
# is where a stored procedure or a better backing store would
# handle this better (something more suited to this type of data).
for fd in lb:
existing_fd = False
for fd_m in lb_m.flowdetails:
if fd_m.uuid == fd.uuid:
existing_fd = True
if fd_m.meta != fd.meta:
fd_m.meta = fd.meta
if fd_m.state != fd.state:
fd_m.state = fd.state
for td in fd:
existing_td = False
for td_m in fd_m.taskdetails:
if td_m.uuid == td.uuid:
existing_td = True
td_m = _taskdetails_merge(td_m, td)
break
if not existing_td:
td_m = _convert_td_to_internal(td, fd_m.uuid)
fd_m.taskdetails.append(td_m)
if not existing_fd:
lb_m.flowdetails.append(
_convert_fd_to_internal(fd, lb_m.uuid))
except exc.NotFound:
lb_m = _convert_lb_to_internal(lb)
try:
lb_m = session.merge(lb_m)
return _convert_lb_to_external(lb_m)
except sql_exc.DBAPIError as e:
raise exc.StorageError(
"Failed saving"
" logbook %s: %s" % (lb.uuid, e), e)
