def update_atom_details(self, ad):
"""Update a atom detail transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
ad = self._update_atom_details(ad, txn)
k_utils.checked_commit(txn)
return ad
def update_flow_details(self, fd):
"""Update a flow detail transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
fd = self._update_flow_details(fd, txn)
k_utils.checked_commit(txn)
return fd
def destroy_logbook(self, lb_uuid):
"""Destroy (delete) a log_book transactionally."""
def _destroy_atom_details(ad_uuid, txn):
ad_path = paths.join(self.atom_path, ad_uuid)
if not self._client.exists(ad_path):
raise exc.NotFound("No atom details found with id: %s"
% ad_uuid)
txn.delete(ad_path)
def _destroy_flow_details(fd_uuid, txn):
fd_path = paths.join(self.flow_path, fd_uuid)
if not self._client.exists(fd_path):
raise exc.NotFound("No flow details found with id: %s"
% fd_uuid)
for ad_uuid in self._client.get_children(fd_path):
_destroy_atom_details(ad_uuid, txn)
txn.delete(paths.join(fd_path, ad_uuid))
txn.delete(fd_path)
def _destroy_logbook(lb_uuid, txn):
lb_path = paths.join(self.book_path, lb_uuid)
if not self._client.exists(lb_path):
raise exc.NotFound("No logbook found with id: %s" % lb_uuid)
for fd_uuid in self._client.get_children(lb_path):
_destroy_flow_details(fd_uuid, txn)
txn.delete(paths.join(lb_path, fd_uuid))
txn.delete(lb_path)
with self._exc_wrapper():
txn = self._client.transaction()
_destroy_logbook(lb_uuid, txn)
k_utils.checked_commit(txn)
def clear_all(self, delete_dirs=True):
"""Delete all data transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
# Delete all data under logbook path.
for lb_uuid in self._client.get_children(self.book_path):
lb_path = paths.join(self.book_path, lb_uuid)
for fd_uuid in self._client.get_children(lb_path):
txn.delete(paths.join(lb_path, fd_uuid))
txn.delete(lb_path)
# Delete all data under flow detail path.
for fd_uuid in self._client.get_children(self.flow_path):
fd_path = paths.join(self.flow_path, fd_uuid)
for ad_uuid in self._client.get_children(fd_path):
txn.delete(paths.join(fd_path, ad_uuid))
txn.delete(fd_path)
# Delete all data under atom detail path.
for ad_uuid in self._client.get_children(self.atom_path):
ad_path = paths.join(self.atom_path, ad_uuid)
txn.delete(ad_path)
# Delete containing directories.
if delete_dirs:
txn.delete(self.book_path)
txn.delete(self.atom_path)
txn.delete(self.flow_path)
k_utils.checked_commit(txn)
def clear_all(self, delete_dirs=True):
"""Delete all data transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
# Delete all data under logbook path.
for lb_uuid in self._client.get_children(self.book_path):
lb_path = paths.join(self.book_path, lb_uuid)
for fd_uuid in self._client.get_children(lb_path):
txn.delete(paths.join(lb_path, fd_uuid))
txn.delete(lb_path)
# Delete all data under flow detail path.
for fd_uuid in self._client.get_children(self.flow_path):
fd_path = paths.join(self.flow_path, fd_uuid)
for ad_uuid in self._client.get_children(fd_path):
txn.delete(paths.join(fd_path, ad_uuid))
txn.delete(fd_path)
# Delete all data under atom detail path.
for ad_uuid in self._client.get_children(self.atom_path):
ad_path = paths.join(self.atom_path, ad_uuid)
txn.delete(ad_path)
# Delete containing directories.
if delete_dirs:
txn.delete(self.book_path)
txn.delete(self.atom_path)
txn.delete(self.flow_path)
k_utils.checked_commit(txn)
def update_flow_details(self, fd):
"""Update a flow detail transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
fd = self._update_flow_details(fd, txn)
k_utils.checked_commit(txn)
return fd
def update_atom_details(self, ad):
"""Update a atom detail transactionally."""
with self._exc_wrapper():
txn = self._client.transaction()
ad = self._update_atom_details(ad, txn)
k_utils.checked_commit(txn)
return ad
def destroy_logbook(self, lb_uuid):
"""Destroy (delete) a log_book transactionally."""
def _destroy_atom_details(ad_uuid, txn):
ad_path = paths.join(self.atom_path, ad_uuid)
if not self._client.exists(ad_path):
raise exc.NotFound("No atom details found with id: %s" %
ad_uuid)
txn.delete(ad_path)
def _destroy_flow_details(fd_uuid, txn):
fd_path = paths.join(self.flow_path, fd_uuid)
if not self._client.exists(fd_path):
raise exc.NotFound("No flow details found with id: %s" %
fd_uuid)
for ad_uuid in self._client.get_children(fd_path):
_destroy_atom_details(ad_uuid, txn)
txn.delete(paths.join(fd_path, ad_uuid))
txn.delete(fd_path)
def _destroy_logbook(lb_uuid, txn):
lb_path = paths.join(self.book_path, lb_uuid)
if not self._client.exists(lb_path):
raise exc.NotFound("No logbook found with id: %s" % lb_uuid)
for fd_uuid in self._client.get_children(lb_path):
_destroy_flow_details(fd_uuid, txn)
txn.delete(paths.join(lb_path, fd_uuid))
txn.delete(lb_path)
with self._exc_wrapper():
txn = self._client.transaction()
_destroy_logbook(lb_uuid, txn)
k_utils.checked_commit(txn)
def save_logbook(self, lb):
"""Save (update) a log_book transactionally."""
def _create_logbook(lb_path, txn):
lb_data = lb.to_dict(marshal_time=True)
txn.create(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
for fd in lb:
# NOTE(harlowja): create an entry in the logbook path
# for the provided flow detail so that a reference exists
# from the logbook to its flow details.
txn.create(paths.join(lb_path, fd.uuid))
fd_path = paths.join(self.flow_path, fd.uuid)
fd_data = jsonutils.dumps(fd.to_dict())
txn.create(fd_path, misc.binary_encode(fd_data))
for ad in fd:
# NOTE(harlowja): create an entry in the flow detail path
# for the provided atom detail so that a reference exists
# from the flow detail to its atom details.
txn.create(paths.join(fd_path, ad.uuid))
ad_path = paths.join(self.atom_path, ad.uuid)
ad_data = base._format_atom(ad)
txn.create(ad_path,
misc.binary_encode(jsonutils.dumps(ad_data)))
return lb
def _update_logbook(lb_path, lb_data, txn):
e_lb = logbook.LogBook.from_dict(misc.decode_json(lb_data),
unmarshal_time=True)
e_lb = e_lb.merge(lb)
lb_data = e_lb.to_dict(marshal_time=True)
txn.set_data(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
for fd in lb:
fd_path = paths.join(lb_path, fd.uuid)
if not self._client.exists(fd_path):
# NOTE(harlowja): create an entry in the logbook path
# for the provided flow detail so that a reference exists
# from the logbook to its flow details.
txn.create(fd_path)
e_fd = self._update_flow_details(fd, txn, create_missing=True)
e_lb.add(e_fd)
return e_lb
with self._exc_wrapper():
txn = self._client.transaction()
# Determine whether the desired data exists or not.
lb_path = paths.join(self.book_path, lb.uuid)
try:
lb_data, _zstat = self._client.get(lb_path)
except k_exc.NoNodeError:
# Create a new logbook since it doesn't exist.
e_lb = _create_logbook(lb_path, txn)
else:
# Otherwise update the existing logbook instead.
e_lb = _update_logbook(lb_path, lb_data, txn)
k_utils.checked_commit(txn)
return e_lb
def save_logbook(self, lb):
"""Save (update) a log_book transactionally."""
def _create_logbook(lb_path, txn):
lb_data = lb.to_dict(marshal_time=True)
txn.create(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
for fd in lb:
# NOTE(harlowja): create an entry in the logbook path
# for the provided flow detail so that a reference exists
# from the logbook to its flow details.
txn.create(paths.join(lb_path, fd.uuid))
fd_path = paths.join(self.flow_path, fd.uuid)
fd_data = jsonutils.dumps(fd.to_dict())
txn.create(fd_path, misc.binary_encode(fd_data))
for ad in fd:
# NOTE(harlowja): create an entry in the flow detail path
# for the provided atom detail so that a reference exists
# from the flow detail to its atom details.
txn.create(paths.join(fd_path, ad.uuid))
ad_path = paths.join(self.atom_path, ad.uuid)
ad_data = base._format_atom(ad)
txn.create(ad_path,
misc.binary_encode(jsonutils.dumps(ad_data)))
return lb
def _update_logbook(lb_path, lb_data, txn):
e_lb = logbook.LogBook.from_dict(misc.decode_json(lb_data),
unmarshal_time=True)
e_lb = e_lb.merge(lb)
lb_data = e_lb.to_dict(marshal_time=True)
txn.set_data(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
for fd in lb:
fd_path = paths.join(lb_path, fd.uuid)
if not self._client.exists(fd_path):
# NOTE(harlowja): create an entry in the logbook path
# for the provided flow detail so that a reference exists
# from the logbook to its flow details.
txn.create(fd_path)
e_fd = self._update_flow_details(fd, txn, create_missing=True)
e_lb.add(e_fd)
return e_lb
with self._exc_wrapper():
txn = self._client.transaction()
# Determine whether the desired data exists or not.
lb_path = paths.join(self.book_path, lb.uuid)
try:
lb_data, _zstat = self._client.get(lb_path)
except k_exc.NoNodeError:
# Create a new logbook since it doesn't exist.
e_lb = _create_logbook(lb_path, txn)
else:
# Otherwise update the existing logbook instead.
e_lb = _update_logbook(lb_path, lb_data, txn)
k_utils.checked_commit(txn)
return e_lb
def claim(self, job, who):
def _unclaimable_try_find_owner(cause):
try:
owner = self.find_owner(job)
except Exception:
owner = None
if owner:
message = "Job %s already claimed by '%s'" % (job.uuid, owner)
else:
message = "Job %s already claimed" % (job.uuid)
excp.raise_with_cause(excp.UnclaimableJob, message, cause=cause)
with self._wrap(job.uuid,
job.path,
fail_msg_tpl="Claiming failure: %s"):
# NOTE(harlowja): post as json which will allow for future changes
# more easily than a raw string/text.
value = jsonutils.dumps({
'owner': who,
})
# Ensure the target job is still existent (at the right version).
job_data, job_stat = self._client.get(job.path)
txn = self._client.transaction()
# This will abort (and not create the lock) if the job has been
# removed (somehow...) or updated by someone else to a different
# version...
txn.check(job.path, version=job_stat.version)
txn.create(job.lock_path,
value=misc.binary_encode(value),
ephemeral=True)
try:
kazoo_utils.checked_commit(txn)
except k_exceptions.NodeExistsError as e:
_unclaimable_try_find_owner(e)
except kazoo_utils.KazooTransactionException as e:
if len(e.failures) < 2:
raise
else:
if isinstance(e.failures[0], k_exceptions.NoNodeError):
excp.raise_with_cause(
excp.NotFound,
"Job %s not found to be claimed" % job.uuid,
cause=e.failures[0])
if isinstance(e.failures[1], k_exceptions.NodeExistsError):
_unclaimable_try_find_owner(e.failures[1])
else:
excp.raise_with_cause(
excp.UnclaimableJob,
"Job %s claim failed due to transaction"
" not succeeding" % (job.uuid),
cause=e)
def claim(self, job, who):
def _unclaimable_try_find_owner(cause):
try:
owner = self.find_owner(job)
except Exception:
owner = None
if owner:
message = "Job %s already claimed by '%s'" % (job.uuid, owner)
else:
message = "Job %s already claimed" % (job.uuid)
excp.raise_with_cause(excp.UnclaimableJob,
message, cause=cause)
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Claiming failure: %s"):
# NOTE(harlowja): post as json which will allow for future changes
# more easily than a raw string/text.
value = jsonutils.dumps({
'owner': who,
})
# Ensure the target job is still existent (at the right version).
job_data, job_stat = self._client.get(job.path)
txn = self._client.transaction()
# This will abort (and not create the lock) if the job has been
# removed (somehow...) or updated by someone else to a different
# version...
txn.check(job.path, version=job_stat.version)
txn.create(job.lock_path, value=misc.binary_encode(value),
ephemeral=True)
try:
kazoo_utils.checked_commit(txn)
except k_exceptions.NodeExistsError as e:
_unclaimable_try_find_owner(e)
except kazoo_utils.KazooTransactionException as e:
if len(e.failures) < 2:
raise
else:
if isinstance(e.failures[0], k_exceptions.NoNodeError):
excp.raise_with_cause(
excp.NotFound,
"Job %s not found to be claimed" % job.uuid,
cause=e.failures[0])
if isinstance(e.failures[1], k_exceptions.NodeExistsError):
_unclaimable_try_find_owner(e.failures[1])
else:
excp.raise_with_cause(
excp.UnclaimableJob,
"Job %s claim failed due to transaction"
" not succeeding" % (job.uuid), cause=e)
def abandon(self, job, who):
with self._wrap(job.uuid, job.path, "Abandonment failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
raise excp.JobFailure("Can not abandon a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not abandon a job %s"
" which is not owned by %s"
% (job.uuid, who))
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
kazoo_utils.checked_commit(txn)
def abandon(self, job, who):
with self._wrap(job.uuid, job.path, "Abandonment failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
raise excp.JobFailure("Can not abandon a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not abandon a job %s"
" which is not owned by %s" %
(job.uuid, who))
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
kazoo_utils.checked_commit(txn)
def consume(self, job, who):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Consumption failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.NotFound,
"Can not consume a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not consume a job %s"
" which is not owned by %s"
% (job.uuid, who))
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
kazoo_utils.checked_commit(txn)
self._remove_job(job.path)
def consume(self, job, who):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Consumption failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.NotFound,
"Can not consume a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not consume a job %s"
" which is not owned by %s"
% (job.uuid, who))
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
kazoo_utils.checked_commit(txn)
self._remove_job(job.path)
def trash(self, job, who):
with self._wrap(job.uuid, job.path, "Trash failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.JobFailure,
"Can not trash a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not trash a job %s"
" which is not owned by %s"
% (job.uuid, who))
trash_path = job.path.replace(self.path, self.trash_path)
value = misc.binary_encode(jsonutils.dumps(data))
txn = self._client.transaction()
txn.create(trash_path, value=value)
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
kazoo_utils.checked_commit(txn)
def trash(self, job, who):
with self._wrap(job.uuid, job.path, fail_msg_tpl="Trash failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(
excp.NotFound, "Can not trash a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not trash a job %s"
" which is not owned by %s" %
(job.uuid, who))
trash_path = job.path.replace(self.path, self.trash_path)
value = misc.binary_encode(jsonutils.dumps(data))
txn = self._client.transaction()
txn.create(trash_path, value=value)
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
kazoo_utils.checked_commit(txn)
def _transaction(self):
transaction = self._client.transaction()
with self._exc_wrapper():
yield transaction
k_utils.checked_commit(transaction)
def _transaction(self):
transaction = self._client.transaction()
with self._exc_wrapper():
yield transaction
k_utils.checked_commit(transaction)