def register_entity(self, entity):
entity_type = entity.kind
if entity_type == c_base.Conductor.ENTITY_KIND:
entity_path = k_paths.join(self.entity_path, entity_type)
try:
self._client.ensure_path(entity_path)
self._client.create(k_paths.join(entity_path, entity.name),
value=misc.binary_encode(
zag_json.dumps(entity.to_dict())),
ephemeral=True)
except k_exceptions.NodeExistsError:
pass
except self._client.handler.timeout_exception:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, operation"
" timed out" % (entity.name, entity_path))
except k_exceptions.SessionExpiredError:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, session"
" expired" % (entity.name, entity_path))
except k_exceptions.KazooException:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, internal"
" error" % (entity.name, entity_path))
else:
raise excp.NotImplementedError(
"Not implemented for other entity type '%s'" % entity_type)
def _do_job_posting(self, name, job_uuid, job_posting, book, details,
job_priority):
raw_job_posting = misc.binary_encode(zag_json.dumps(job_posting))
with self._wrap(job_uuid,
None,
fail_msg_tpl="Posting failure: %s",
ensure_known=False):
job_path = self._client.create(self._job_base,
value=raw_job_posting,
sequence=True,
ephemeral=False)
job = ZookeeperJob(self,
name,
self._client,
job_path,
backend=self._persistence,
book=book,
details=details,
uuid=job_uuid,
book_data=job_posting.get('book'),
priority=job_priority)
with self._job_cond:
self._known_jobs[job_path] = job
self._job_cond.notify_all()
self._try_emit(base.POSTED, details={'job': job})
return job
def _change_owner(self, new_owner):
children = self.client.storage.get_children("/zag", only_direct=False)
altered = 0
for p, data in six.iteritems(children):
if p.endswith(".lock"):
self.client.set(
p, misc.binary_encode(zag_json.dumps({'owner':
new_owner})))
altered += 1
return altered
def claim(self, job, who):
def _unclaimable_try_find_owner(cause):
try:
owner = self.find_owner(job)
except Exception:
owner = None
if owner:
message = "Job %s already claimed by '%s'" % (job.uuid, owner)
else:
message = "Job %s already claimed" % (job.uuid)
excp.raise_with_cause(excp.UnclaimableJob, message, cause=cause)
with self._wrap(job.uuid,
job.path,
fail_msg_tpl="Claiming failure: %s"):
# NOTE(harlowja): post as json which will allow for future changes
# more easily than a raw string/text.
value = zag_json.dumps({
'owner': who,
})
# Ensure the target job is still existent (at the right version).
job_data, job_stat = self._client.get(job.path)
txn = self._client.transaction()
# This will abort (and not create the lock) if the job has been
# removed (somehow...) or updated by someone else to a different
# version...
txn.check(job.path, version=job_stat.version)
txn.create(job.lock_path,
value=misc.binary_encode(value),
ephemeral=True)
try:
kazoo_utils.checked_commit(txn)
except k_exceptions.NodeExistsError as e:
_unclaimable_try_find_owner(e)
except kazoo_utils.KazooTransactionException as e:
if len(e.failures) < 2:
raise
else:
if isinstance(e.failures[0], k_exceptions.NoNodeError):
excp.raise_with_cause(
excp.NotFound,
"Job %s not found to be claimed" % job.uuid,
cause=e.failures[0])
if isinstance(e.failures[1], k_exceptions.NodeExistsError):
_unclaimable_try_find_owner(e.failures[1])
else:
excp.raise_with_cause(
excp.UnclaimableJob,
"Job %s claim failed due to transaction"
" not succeeding" % (job.uuid),
cause=e)
def join(self, key_piece, *more_key_pieces):
"""Create and return a namespaced key from many segments.
NOTE(harlowja): all pieces that are text/unicode are converted into
their binary equivalent (if they are already binary no conversion
takes place) before being joined (as redis expects binary keys and not
unicode/text ones).
"""
namespace_pieces = []
if self._namespace is not None:
namespace_pieces = [self._namespace, self.NAMESPACE_SEP]
else:
namespace_pieces = []
key_pieces = [key_piece]
if more_key_pieces:
key_pieces.extend(more_key_pieces)
for i in compat_range(0, len(namespace_pieces)):
namespace_pieces[i] = misc.binary_encode(namespace_pieces[i])
for i in compat_range(0, len(key_pieces)):
key_pieces[i] = misc.binary_encode(key_pieces[i])
namespace = b"".join(namespace_pieces)
key = self.KEY_PIECE_SEP.join(key_pieces)
return namespace + key
def test_posting_owner_lost(self):
with base.connect_close(self.board):
with self.flush(self.client):
j = self.board.post('test', test_utils.test_factory)
self.assertEqual(states.UNCLAIMED, j.state)
with self.flush(self.client):
self.board.claim(j, self.board.name)
self.assertEqual(states.CLAIMED, j.state)
# Forcefully delete the owner from the backend storage to make
# sure the job becomes unclaimed (this may happen if some admin
# manually deletes the lock).
paths = list(six.iteritems(self.client.storage.paths))
for (path, value) in paths:
if path in self.bad_paths:
continue
if path.endswith('lock'):
value['data'] = misc.binary_encode(zag_json.dumps({}))
self.assertEqual(states.UNCLAIMED, j.state)
def trash(self, job, who):
with self._wrap(job.uuid, job.path, fail_msg_tpl="Trash failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(
excp.NotFound, "Can not trash a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not trash a job %s"
" which is not owned by %s" %
(job.uuid, who))
trash_path = job.path.replace(self.path, self.trash_path)
value = misc.binary_encode(zag_json.dumps(data))
txn = self._client.transaction()
txn.create(trash_path, value=value)
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
kazoo_utils.checked_commit(txn)
def dumps(topic, tasks):
return misc.binary_encode(
zag_json.dumps({
'topic': topic,
'tasks': tasks,
}))
def test_unicode_other_encoding(self):
result = misc.binary_encode(u'mañana', 'latin-1')
self.assertIsInstance(result, six.binary_type)
self.assertEqual(u'mañana'.encode('latin-1'), result)
def _check(self, data, expected_result):
result = misc.binary_encode(data)
self.assertIsInstance(result, six.binary_type)
self.assertEqual(expected_result, result)
def _write_to(self, filename, contents):
contents = misc.binary_encode(contents, encoding=self.backend.encoding)
with io.open(filename, 'wb') as fp:
fp.write(contents)
self.backend.file_cache.pop(filename, None)
