def _set_with_ordering(self, skey):
"""
Final call for finally ordering the looked-up collection.
The ordering will be done by Redis itself and stored as a temporary set.
:return: a Set of `id`
"""
num, start = self._get_limit_and_offset()
old_set_key = skey
for ordering, alpha in self._ordering:
if ordering.startswith('-'):
desc = True
ordering = ordering.lstrip('-')
else:
desc = False
new_set_key = "%s#%s.%s" % (old_set_key, ordering, id(self))
by = "%s->%s" % (self.model_class._key['*'], ordering)
self.db.sort(old_set_key,
by=by,
store=new_set_key,
alpha=alpha,
start=start,
num=num,
desc=desc)
if old_set_key != self.key:
Set(old_set_key, db=self.db).set_expire()
new_list = List(new_set_key, db=self.db)
new_list.set_expire()
return new_list
def _set_without_ordering(self, skey):
# sort by id
num, start = self._get_limit_and_offset()
old_set_key = skey
new_set_key = "%s#.%s" % (old_set_key, id(self))
self.db.sort(old_set_key, store=new_set_key, start=start, num=num)
self._expire_or_delete.append(old_set_key)
self._expire_or_delete.append(new_set_key)
return List(new_set_key)
def main():
"""
Main function to run forever...
"""
global run_ok
lists = [settings.WORKER_FETCH_FULL_KEY % priority for priority in range(settings.WORKER_FETCH_FULL_MAX_PRIORITY, -1, -1)]
redis_instance = redis.Redis(**settings.REDIS_PARAMS)
nb = 0
max_nb = 500
while run_ok:
# wait for new data
list_name, json = redis_instance.blpop(lists)
priority = int(list_name[-1])
nb += 1
len_list = redis_instance.llen(list_name)
sys.stderr.write("\n[%s #%d | left(%s) : %d] %s\n" % (datetime.utcnow(), nb, list_name, len_list, RE_IGNORE_IMPORT.sub('', json)))
try:
# unserialize
data = parse_json(json, priority)
if not data:
raise Exception('Invalid data : %s' % data)
except:
sys.stderr.write("\n".join(traceback.format_exception(*sys.exc_info())))
List(settings.WORKER_FETCH_FULL_ERROR_KEY).append(json)
else:
if data.get('ignore', False):
sys.stderr.write(" => ignore\n")
else:
# we're good
params = dict(
token = data['token'],
depth = data['depth'],
async = False
)
if data.get('notify_user', None):
params['notify_user'] = data['notify_user']
_, error = data['object'].fetch_full(**params)
if error and isinstance(error, (DatabaseError, IntegrityError)):
# stop the process if integrityerror to start a new transaction
run_ok = False
if nb >= max_nb:
run_ok = False
def _write(self, _new=False):
"""Writes the values of the attributes to the datastore.
This method also creates the indices and saves the lists
associated to the object.
"""
pipeline = self.db.pipeline()
self._create_membership(pipeline)
self._update_indices(pipeline)
h = {}
# attributes
for k, v in self.attributes.items():
if isinstance(v, DateTimeField):
if v.auto_now:
setattr(self, k, datetime.now(tz=tzutc()))
if v.auto_now_add and _new:
setattr(self, k, datetime.now(tz=tzutc()))
elif isinstance(v, DateField):
if v.auto_now:
setattr(self, k, datetime.now(tz=tzutc()))
if v.auto_now_add and _new:
setattr(self, k, datetime.now(tz=tzutc()))
for_storage = getattr(self, k)
if for_storage is not None:
h[k] = v.typecast_for_storage(for_storage)
# indices
for index in self.indices:
if index not in self.lists and index not in self.attributes:
v = getattr(self, index)
if callable(v):
v = v()
if v:
try:
h[index] = unicode(v)
except UnicodeError:
h[index] = unicode(v.decode('utf-8'))
pipeline.delete(self.key())
if h:
pipeline.hmset(self.key(), h)
# lists
for k, v in self.lists.items():
l = List(self.key()[k], pipeline=pipeline)
l.clear()
values = getattr(self, k)
if values:
if v._redisco_model:
l.extend([item.id for item in values])
else:
l.extend(values)
pipeline.execute()
def _set_without_ordering(self, skey):
"""
Final call for "non-ordered" looked up.
We order by id anyway and this is done by redis (same as above).
:returns: A Set of `id`
"""
# sort by id
num, start = self._get_limit_and_offset()
old_set_key = skey
new_set_key = "%s#.%s" % (old_set_key, id(self))
self.db.sort(old_set_key, store=new_set_key, start=start, num=num)
if old_set_key != self.key:
Set(old_set_key, db=self.db).set_expire()
new_list = List(new_set_key, db=self.db)
new_list.set_expire()
return new_list
def __get__(self, instance, owner):
try:
return getattr(instance, '_' + self.name)
except AttributeError:
if instance.is_new():
val = self.default
else:
key = instance.key()[self.name]
val = List(key).members
if val is not None:
klass = self.value_type()
if self._redisco_model:
val = filter(lambda o: o is not None, [klass.objects.get_by_id(v) for v in val])
else:
val = [klass(v) for v in val]
self.__set__(instance, val)
return val
def _set_with_ordering(self, skey):
num, start = self._get_limit_and_offset()
old_set_key = skey
for ordering, alpha in self._ordering:
if ordering.startswith('-'):
desc = True
ordering = ordering.lstrip('-')
else:
desc = False
new_set_key = "%s#%s.%s" % (old_set_key, ordering, id(self))
by = "%s->%s" % (self.model_class._key['*'], ordering)
self.db.sort(old_set_key,
by=by,
store=new_set_key,
alpha=alpha,
start=start,
num=num,
desc=desc)
self._expire_or_delete.append(old_set_key)
self._expire_or_delete.append(new_set_key)
return List(new_set_key)