class MyQueue(Queue):
"""
A queue that will store the dates of it's first and last successful job
"""
first_job_date = fields.InstanceHashField()
last_job_date = fields.InstanceHashField()
jobs_counter = fields.InstanceHashField()
class Group(TestRedisModel):
namespace = 'contrib-collection'
collection_manager = ExtendedCollectionManager
id = fields.AutoPKField()
name = fields.InstanceHashField(indexable=True)
active = fields.InstanceHashField(indexable=True, default=1)
public = fields.InstanceHashField(indexable=True, default=1)
class Person(model.RedisModel):
"""
A simple model for which we want to compute fullname based on firstname and
lastname
"""
database = BaseJobsModel.database
firstname = fields.InstanceHashField()
lastname = fields.InstanceHashField()
fullname = fields.InstanceHashField()
class MyModel(TestRedisModel):
name = fields.InstanceHashField()
last_modification_date = fields.InstanceHashField()
def post_command(self, sender, name, result, args, kwargs):
if isinstance(sender, fields.RedisField) and sender.name == "name":
if name in sender.available_modifiers:
self.last_modification_date.hset(datetime.now())
elif name == "hget":
result = "modifed_result"
return result
class Boat(TestRedisModel):
"""
Use also InstanceHashField.
"""
name = fields.StringField(unique=True)
power = fields.InstanceHashField(indexable=True, default="sail")
launched = fields.StringField(indexable=True)
length = fields.StringField()
class DateTimeModelTest(TestRedisModel):
date = fields.InstanceHashField(indexable=True, indexes=[DateIndex])
unique_date = fields.InstanceHashField(indexable=True,
indexes=[DateIndex],
unique=True)
time = fields.InstanceHashField(indexable=True, indexes=[TimeIndex])
unique_time = fields.InstanceHashField(indexable=True,
indexes=[TimeIndex],
unique=True)
datetime = fields.InstanceHashField(indexable=True,
indexes=[DateTimeIndex])
simple_datetime = fields.InstanceHashField(indexable=True,
indexes=[SimpleDateTimeIndex])
unique_datetime = fields.InstanceHashField(indexable=True,
indexes=[DateTimeIndex],
unique=True)
unique_simple_datetime = fields.InstanceHashField(
indexable=True, indexes=[SimpleDateTimeIndex], unique=True)
class MyJob(Job):
"""
A job that will use Person's PK as identifier, and will store results of
callback in a new field
"""
result = fields.StringField() # to store the result of the task
queue_model = MyQueue
queue_name = QUEUE_NAME
start = fields.InstanceHashField(indexable=True)
def get_object(self):
"""
Return the object concerned by the current job's identifier
"""
return Person.get(self.identifier.hget())
def run(self, queue):
"""
Create the fullname, and store a a message serving as result in the job
"""
# add some random time to simulate a long job
time.sleep(random.random())
# compute the fullname
obj = self.get_object()
obj.fullname.hset('%s %s' % tuple(obj.hmget('firstname', 'lastname')))
# this will the "result" of the job
result = 'Created fullname for Person %s: %s' % (obj.pk.get(),
obj.fullname.hget())
# save the result of the callback in the job itself
self.result.set(result)
# return the result for future use in the worker
return result
class Train(TestRedisModel):
namespace = "test_pkfield_cannot_be_deleted"
name = fields.InstanceHashField(unique=True)
class Train(TestRedisModel):
namespace = "test_instancehashfield_keys_are_deleted"
name = fields.InstanceHashField(unique=True)
kind = fields.InstanceHashField(indexable=True)
wagons = fields.InstanceHashField(default=10)
class Queue(BaseJobsModel):
name = fields.InstanceHashField(indexable=True)
priority = fields.InstanceHashField(indexable=True, default=0) # the higher, the better
waiting = fields.ListField()
success = fields.ListField()
errors = fields.ListField()
delayed = fields.SortedSetField()
@classmethod
def get_queue(cls, name, priority=0, **fields_if_new):
"""
Get, or create, and return the wanted queue.
If the queue is created, fields in fields_if_new will be set for the new
queue.
"""
queue_kwargs = {'name': name, 'priority': priority}
retries = 0
while retries < 10:
retries += 1
try:
queue, created = cls.get_or_connect(**queue_kwargs)
except IndexError:
# Failure during the retrieval https://friendpaste.com/5U63a8aFuV44SEgQckgMP
# => retry
continue
except ValueError:
# more than one (race condition https://github.com/yohanboniface/redis-limpyd/issues/82 ?)
try:
queue = cls.collection(**queue_kwargs).instances()[0]
except IndexError:
# but no more now ?!
# => retry
continue
else:
created = False
# ok we have our queue, stop now
break
if created and fields_if_new:
queue.set_fields(**fields_if_new)
return queue
def delay_job(self, job, delayed_until):
"""
Add the job to the delayed list (zset) of the queue.
"""
timestamp = datetime_to_score(delayed_until)
self.delayed.zadd({job.ident: timestamp})
def enqueue_job(self, job, prepend=False):
"""
Add the job to the waiting list, at the end (it's a fifo list). If
`prepend` is True, add it at the beginning of the list.
"""
push_method = getattr(self.waiting, 'lpush' if prepend else 'rpush')
push_method(job.ident)
@staticmethod
def _get_iterable_for_names(names):
"""
Ensure that we have an iterable list of names, even if we have a single
name
"""
if isinstance(names, basestring):
names = (names, )
return names
@classmethod
def get_all(cls, names):
"""
Return all queues for the given names (for all available priorities)
"""
names = cls._get_iterable_for_names(names)
queues = []
for queue_name in names:
queues.extend(list(cls.collection(name=queue_name).instances()))
return queues
@classmethod
def get_all_by_priority(cls, names):
"""
Return all the queues with the given names, sorted by priorities (higher
priority first), then by name
"""
names = cls._get_iterable_for_names(names)
queues = cls.get_all(names)
# sort all queues by priority
queues.sort(key=lambda q: int(q.priority.hget() or 0), reverse=True)
return queues
@classmethod
def get_waiting_keys(cls, names):
"""
Return a list of all queue waiting keys, to use with blpop
"""
return [queue.waiting.key for queue in cls.get_all_by_priority(names)]
@classmethod
def count_waiting_jobs(cls, names):
"""
Return the number of all jobs waiting in queues with the given names
"""
return sum([queue.waiting.llen() for queue in cls.get_all(names)])
@classmethod
def count_delayed_jobs(cls, names):
"""
Return the number of all delayed jobs in queues with the given names
"""
return sum([queue.delayed.zcard() for queue in cls.get_all(names)])
@property
def first_delayed(self):
"""
Return the first entry in the delayed zset (a tuple with the job's pk
and the score of the zset, which it's delayed time as a timestamp)
Returns None if no delayed jobs
"""
entries = self.delayed.zrange(0, 0, withscores=True)
return entries[0] if entries else None
@property
def first_delayed_time(self):
"""
Get the timestamp representation of the first delayed job to be ready.
"""
# get the first job which will be ready
first_entry = self.first_delayed
return first_entry[1] if first_entry else None
def requeue_delayed_jobs(self):
"""
Put all delayed jobs that are now ready, back in the queue waiting list
Return a list of failures
"""
lock_key = self.make_key(
self._name,
self.pk.get(),
"requeue_all_delayed_ready_jobs",
)
connection = self.get_connection()
if connection.exists(lock_key):
# if locked, a worker is already on it, don't wait and exit
return []
with Lock(connection, lock_key, timeout=60):
# stop here if we know we have nothing
first_delayed_time = self.first_delayed_time
if not first_delayed_time:
return []
# get when we are :)
now_timestamp = datetime_to_score(datetime.utcnow())
# the first job will be ready later, and so the other ones too, then
# abort
if float(first_delayed_time) > now_timestamp:
return []
failures = []
while True:
# get the first entry
first_entry = self.first_delayed
# no first entry, another worker took all from us !
if not first_entry:
break
# split into vars for readability
job_ident, delayed_until = first_entry
# if the date of the job is in the future, another work took the
# job we wanted, so we let this job here and stop the loop as we
# know (its a zset sorted by date) that no other jobs are ready
if delayed_until > now_timestamp:
break
# remove the entry we just got from the delayed ones
self.delayed.zrem(job_ident)
# and add it to the waiting queue
try:
job = Job.get_from_ident(job_ident)
if job.status.hget() == STATUSES.DELAYED:
job.status.hset(STATUSES.WAITING)
self.enqueue_job(job)
except Exception as e:
failures.append((job_ident, '%s' % e))
return failures
class Plane(TestRedisModel):
my_pk = fields.PKField()
name = fields.InstanceHashField()
is_first = fields.InstanceHashField(indexable=True)
class Event(TestRedisModel):
year = fields.InstanceHashField()
class Plane2(TestRedisModel):
my_pk = fields.PKField()
name = fields.InstanceHashField()
is_first = fields.InstanceHashField(indexable=True)
collection_manager = ExtendedCollectionManager
class Error(BaseJobsModel):
job_model_repr = fields.InstanceHashField(indexable=True)
job_pk = fields.InstanceHashField(indexable=True)
identifier = fields.InstanceHashField(indexable=True)
queue_name = fields.InstanceHashField(indexable=True)
date_time = fields.InstanceHashField(indexable=True, indexes=[SimpleDateTimeIndex])
date = fields.InstanceHashField(indexable=True) # kept for now for compatibility
time = fields.InstanceHashField() # kept for now for compatibility
type = fields.InstanceHashField(indexable=True)
code = fields.InstanceHashField(indexable=True)
message = fields.InstanceHashField()
traceback = fields.InstanceHashField()
@classmethod
def add_error(cls, queue_name, job, error, when=None, trace=None,
**additional_fields):
"""
Add a new error in redis.
`job` is a job which generated the error
`queue_name` is the name of the queue where the error arrived
`error` is an exception, which can has a code (better if it is)
`when` is the datetime of the error, utcnow will be used if not defined
`trace` is the traceback generated by the exception, may be null
The new created instance is returned, with additional_fields set for
aubclasses.
"""
if when is None:
when = datetime.utcnow()
when = str(when)
fields = dict(
queue_name=queue_name,
job_model_repr=job.get_model_repr(),
job_pk=job.pk.get(),
identifier=getattr(job, '_cached_identifier', job.identifier.hget()),
date_time=when,
date=when[:10],
time=when[11:],
message=str(error),
)
try:
# exception can be a class (should not, but just in case...)
fields['type'] = error.__name__
except AttributeError:
# or excetion is an instance
fields['type'] = error.__class__.__name__
error_code = getattr(error, 'code', None)
if error_code is not None:
fields['code'] = error_code
if trace:
fields['traceback'] = trace
error = cls(**fields)
if additional_fields:
error.set_fields(**additional_fields)
return error
@classmethod
def collection_for_job(cls, job):
"""
Helper to return a collection of errors for the given job
"""
return cls.collection(job_model_repr=job.get_model_repr(), identifier=getattr(job, '_cached_identifier', job.identifier.hget()))
@property
def datetime(self):
"""
Property which return a real datetime object based on the date and time
fields
"""
return parse(self.date_time.hget())
class Train(TestRedisModel):
namespace = "test_model_delete"
name = fields.InstanceHashField(unique=True)
kind = fields.StringField(indexable=True)
wagons = fields.InstanceHashField(default=10)
class Group(TestRedisModel):
namespace = 'collection'
collection_manager = ActiveGroups
name = fields.InstanceHashField()
active = fields.InstanceHashField(indexable=True, default=1)
class HMTestModel(TestRedisModel):
foo = fields.InstanceHashField()
bar = fields.InstanceHashField(indexable=True)
baz = fields.InstanceHashField(unique=True)
class Singer(TestRedisModel):
name = fields.InstanceHashField()
class Query(TestRedisModel):
namespace = 'FieldOrModelAsValueForSortAndFilterTest'
collection_manager = ExtendedCollectionManager
name = fields.InstanceHashField()
active = fields.InstanceHashField()
public = fields.InstanceHashField()
class Band(TestRedisModel):
name = fields.InstanceHashField(unique=True)
started_in = fields.InstanceHashField()
genre = fields.InstanceHashField(indexable=True)
class Job(BaseJobsModel):
identifier = fields.InstanceHashField(indexable=True) # ex: "myobj:123:update"
status = fields.InstanceHashField(indexable=True) # see STATUSES constants
priority = fields.InstanceHashField(indexable=True, default=0)
added = fields.InstanceHashField()
start = fields.InstanceHashField()
end = fields.InstanceHashField()
tries = fields.InstanceHashField()
delayed_until = fields.InstanceHashField()
queued = fields.InstanceHashField(indexable=True) # '1' if queued
cancel_on_error = fields.InstanceHashField()
always_cancel_on_error = False
queue_model = Queue
queue_name = None
@classmethod
def get_model_repr(cls):
"""
Return a string representation of the current class
"""
return '%s.%s' % (cls.__module__, cls.__name__)
@property
def ident(self):
"""
Return the string to use in queues, include the job's class and its pk
"""
return '%s:%s' % (self.get_model_repr(), self.pk.get())
@classmethod
def get_from_ident(self, ident):
"""
Take a string as returned by get_ident and return a job,
based on the class representation and the job's pk from the ident
"""
model_repr, job_pk = ident.split(':', 1)
klass = import_class(model_repr)
return klass.get(job_pk)
@classmethod
def _get_queue_name(cls, queue_name=None):
"""
Return the given queue_name if defined, else the class's one.
If both are None, raise an Exception
"""
if queue_name is None and cls.queue_name is None:
raise LimpydJobsException("Queue's name not defined")
if queue_name is None:
return cls.queue_name
return queue_name
@classmethod
def add_job(cls, identifier, queue_name=None, priority=0, queue_model=None,
prepend=False, delayed_for=None, delayed_until=None,
**fields_if_new):
"""
Add a job to a queue.
If this job already exists, check it's current priority. If its higher
than the new one, don't touch it, else move the job to the wanted queue.
Before setting/moving the job to the queue, check for a `delayed_for`
(int/foat/timedelta) or `delayed_until` (datetime) argument to see if
it must be delayed instead of queued.
If the job is created, fields in fields_if_new will be set for the new
job.
Finally return the job.
"""
# check for delayed_for/delayed_until arguments
delayed_until = compute_delayed_until(delayed_for, delayed_until)
# create the job or get an existing one
job_kwargs = {'identifier': identifier, 'queued': '1'}
retries = 0
while retries < 10:
retries += 1
try:
job, created = cls.get_or_connect(**job_kwargs)
except IndexError:
# Failure during the retrieval https://friendpaste.com/5U63a8aFuV44SEgQckgMP
# => retry
continue
except ValueError:
# more than one already in the queue !
try:
job = cls.collection(**job_kwargs).instances()[0]
except IndexError:
# but no more now ?!
# => retry
continue
else:
created = False
# ok we have our job, stop now
break
try:
# check queue_name
queue_name = cls._get_queue_name(queue_name)
# if the job already exists, and we want a higher priority or move it,
# start by updating it
if not created:
current_priority = int(job.priority.hget() or 0)
# if the job has a higher priority, or don't need to be moved,
# don't move it
if not prepend and current_priority >= priority:
return job
# cancel it temporarily, we'll set it as waiting later
job.status.hset(STATUSES.CANCELED)
# remove it from the current queue, we'll add it to the new one later
if queue_model is None:
queue_model = cls.queue_model
current_queue = queue_model.get_queue(queue_name, current_priority)
current_queue.waiting.lrem(0, job.ident)
else:
job.set_fields(added=str(datetime.utcnow()), **(fields_if_new or {}))
# add the job to the queue
job.enqueue_or_delay(queue_name, priority, delayed_until, prepend, queue_model)
return job
except Exception:
job.queued.delete()
raise
def run(self, queue):
"""
The method called by the work to do some stuff. Must be overriden to
your needs.
The optional return value of this function will be passed to the
job_success method of the worker.
"""
raise NotImplementedError('You must implement your own action')
@property
def must_be_cancelled_on_error(self):
"""
Return True if either the "always_cancel_on_error" attribute is True,
of if the "cancel_on_error" field holds a True value
"""
return self.always_cancel_on_error or self.cancel_on_error.hget()
@property
def duration(self):
"""
If the start and end times of the job are defined, return a timedelta,
else return None
"""
try:
start, end = self.hmget('start', 'end')
return parse(end) - parse(start)
except:
return None
def requeue(self, queue_name=None, priority=None, delayed_for=None,
delayed_until=None, queue_model=None):
"""
Requeue the job in the given queue if it has previously failed
"""
queue_name = self._get_queue_name(queue_name)
# we can only requeue a job that raised an error
if self.status.hget() != STATUSES.ERROR:
raise LimpydJobsException('Job cannot be requeued if not in ERROR status')
self.hdel('start', 'end')
if priority is None:
priority = self.priority.hget()
delayed_until = compute_delayed_until(delayed_for, delayed_until)
self.enqueue_or_delay(queue_name, priority, delayed_until, queue_model=queue_model)
def enqueue_or_delay(self, queue_name=None, priority=None,
delayed_until=None, prepend=False, queue_model=None):
"""
Will enqueue or delay the job depending of the delayed_until.
"""
queue_name = self._get_queue_name(queue_name)
fields = {'queued': '1'}
if priority is not None:
fields['priority'] = priority
else:
priority = self.priority.hget()
in_the_future = delayed_until and delayed_until > datetime.utcnow()
if in_the_future:
fields['delayed_until'] = str(delayed_until)
fields['status'] = STATUSES.DELAYED
else:
self.delayed_until.delete()
fields['status'] = STATUSES.WAITING
self.hmset(**fields)
if queue_model is None:
queue_model = self.queue_model
queue = queue_model.get_queue(queue_name, priority)
if in_the_future:
queue.delay_job(self, delayed_until)
else:
queue.enqueue_job(self, prepend)
