def trash(self, job, who): script = self._get_script('trash') with _translate_failures(): raw_who = self._encode_owner(who) raw_result = script(keys=[job.owner_key, self.listings_key, job.last_modified_key, self.trash_key], args=[raw_who, job.key, self._dumps(timeutils.utcnow())]) result = self._loads(raw_result) status = result['status'] if status != self.SCRIPT_STATUS_OK: reason = result.get('reason') if reason == self.SCRIPT_UNKNOWN_JOB: raise exc.NotFound("Job %s not found to be" " trashed" % (job.uuid)) elif reason == self.SCRIPT_UNKNOWN_OWNER: raise exc.NotFound("Can not trash job %s" " which we can not determine" " the owner of" % (job.uuid)) elif reason == self.SCRIPT_NOT_EXPECTED_OWNER: raw_owner = result.get('owner') if raw_owner: owner = self._decode_owner(raw_owner) raise exc.JobFailure("Can not trash job %s" " which is not owned by %s (it is" " actively owned by %s)" % (job.uuid, who, owner)) else: raise exc.JobFailure("Can not trash job %s" " which is not owned by %s" % (job.uuid, who)) else: raise exc.JobFailure("Failure to trash job %s," " unknown internal error (reason=%s)" % (job.uuid, reason))
def connect(self): self.close() if self._owns_client: self._client = self._make_client(self._conf) with _translate_failures(): # The client maintains a connection pool, so do a ping and # if that works then assume the connection works, which may or # may not be continuously maintained (if the server dies # at a later time, we will become aware of that when the next # op occurs). self._client.ping() is_new_enough, redis_version = ru.is_server_new_enough( self._client, self.MIN_REDIS_VERSION) if not is_new_enough: wanted_version = ".".join([str(p) for p in self.MIN_REDIS_VERSION]) if redis_version: raise exc.JobFailure("Redis version %s or greater is" " required (version %s is to" " old)" % (wanted_version, redis_version)) else: raise exc.JobFailure("Redis version %s or greater is" " required" % (wanted_version)) else: self._redis_version = redis_version script_params = { # Status field values. 'ok': self.SCRIPT_STATUS_OK, 'error': self.SCRIPT_STATUS_ERROR, # Known error reasons (when status field is error). 'not_expected_owner': self.SCRIPT_NOT_EXPECTED_OWNER, 'unknown_owner': self.SCRIPT_UNKNOWN_OWNER, 'unknown_job': self.SCRIPT_UNKNOWN_JOB, 'already_claimed': self.SCRIPT_ALREADY_CLAIMED, } prepared_scripts = {} for n, raw_script_tpl in six.iteritems(self.SCRIPT_TEMPLATES): script_tpl = string.Template(raw_script_tpl) script_blob = script_tpl.substitute(**script_params) script = self._client.register_script(script_blob) prepared_scripts[n] = script self._scripts.update(prepared_scripts) self._closed = False
def post_scheduled(self, schedule, name, flow_factory, factory_args=None, factory_kwargs=None, store=None, priority=JobPriority.NORMAL): """Post a scheduled job. This takes a crontab-like schedule and will run the job on that schedule. Each time a job is finished, it will be re-posted at the next start time for that schedule. """ try: delay = _next_scheduled_delay(schedule) except ValueError: raise excp.JobFailure( "Schedule '%s' for scheduled job is not " "valid", schedule) else: if delay is None: raise excp.JobFailure( "Schedule '%s' for scheduled job does " "not include any future times", schedule) post_data = self._prep_job_posting( name, { 'schedule': schedule, 'run_at': int(time.time() + delay) }, flow_factory, factory_args=factory_args, factory_kwargs=factory_kwargs, store=store, priority=priority, ) return self._do_job_posting(*post_data)
def claim(self, job, who, expiry=None): if expiry is None: # On the lua side none doesn't translate to nil so we have # do to this string conversion to make sure that we can tell # the difference. ms_expiry = "none" else: ms_expiry = int(expiry * 1000.0) if ms_expiry <= 0: raise ValueError("Provided expiry (when converted to" " milliseconds) must be greater" " than zero instead of %s" % (expiry)) script = self._get_script('claim') with _translate_failures(): raw_who = self._encode_owner(who) raw_result = script(keys=[job.owner_key, self.listings_key, job.last_modified_key], args=[raw_who, job.key, # NOTE(harlowja): we need to send this # in as a blob (even if it's not # set/used), since the format can not # currently be created in lua... self._dumps(timeutils.utcnow()), ms_expiry]) result = self._loads(raw_result) status = result['status'] if status != self.SCRIPT_STATUS_OK: reason = result.get('reason') if reason == self.SCRIPT_UNKNOWN_JOB: raise exc.NotFound("Job %s not found to be" " claimed" % (job.uuid)) elif reason == self.SCRIPT_ALREADY_CLAIMED: raw_owner = result.get('owner') if raw_owner: owner = self._decode_owner(raw_owner) raise exc.UnclaimableJob("Job %s already" " claimed by %s" % (job.uuid, owner)) else: raise exc.UnclaimableJob("Job %s already" " claimed" % (job.uuid)) else: raise exc.JobFailure("Failure to claim job %s," " unknown internal error (reason=%s)" % (job.uuid, reason))
def abandon(self, job, who): with self._wrap(job.uuid, job.path, fail_msg_tpl="Abandonment failure: %s"): try: owner_data = self._get_owner_and_data(job) lock_data, lock_stat, data, data_stat = owner_data except k_exceptions.NoNodeError: excp.raise_with_cause( excp.NotFound, "Can not abandon a job %s" " which we can not determine" " the owner of" % (job.uuid)) if lock_data.get("owner") != who: raise excp.JobFailure("Can not abandon a job %s" " which is not owned by %s" % (job.uuid, who)) txn = self._client.transaction() txn.delete(job.lock_path, version=lock_stat.version) kazoo_utils.checked_commit(txn)
def trash(self, job, who): with self._wrap(job.uuid, job.path, fail_msg_tpl="Trash failure: %s"): try: owner_data = self._get_owner_and_data(job) lock_data, lock_stat, data, data_stat = owner_data except k_exceptions.NoNodeError: excp.raise_with_cause( excp.NotFound, "Can not trash a job %s" " which we can not determine" " the owner of" % (job.uuid)) if lock_data.get("owner") != who: raise excp.JobFailure("Can not trash a job %s" " which is not owned by %s" % (job.uuid, who)) trash_path = job.path.replace(self.path, self.trash_path) value = misc.binary_encode(zag_json.dumps(data)) txn = self._client.transaction() txn.create(trash_path, value=value) txn.delete(job.lock_path, version=lock_stat.version) txn.delete(job.path, version=data_stat.version) kazoo_utils.checked_commit(txn)
def _do_job_posting(self, name, job_uuid, job_posting, book, details, job_priority): with _translate_failures(): sequence = self._client.incr(self.sequence_key) job_posting.update({ 'sequence': sequence, }) with _translate_failures(): raw_posting = self._dumps(job_posting) raw_job_uuid = six.b(job_uuid) was_posted = bool(self._client.hsetnx(self.listings_key, raw_job_uuid, raw_posting)) if not was_posted: raise exc.JobFailure("New job located at '%s[%s]' could not" " be posted" % (self.listings_key, raw_job_uuid)) else: return RedisJob(self, name, sequence, raw_job_uuid, uuid=job_uuid, details=details, created_on=job_posting['created_on'], book=book, book_data=job_posting.get('book'), backend=self._persistence, priority=job_priority)