def set(self, **data):
"""Saves given key=value pairs and updates cache
"""
if not self.exists():
self.create()
save_data = {}
save_data['updated_at'] = int(time.time())
for key, value in data.iteritems():
"""Special cases that require more actions than just adding
them to the article hash
"""
if key == 'author':
# remove previous author
if 'author' in self._data:
key = 'author:{0}:articles'.format(self._data['author'])
rc().srem(rn(key), self._id)
rc().sadd(rn('author:{0}:articles'.format(key)), self._id)
if key == 'title':
if self.has('title'):
old_url_title = self._title_url(self.get('title'))
rc().delete(rn('article:title_to_id:{0}'\
.format(old_url_title)))
url_title = self._title_url(value)
if len(url_title) > 0:
rc().set(rn('article:title_to_id:{0}'.format(url_title)),
self._id)
if key == 'text':
save_data['text_compiled'] = markdown.markdown(value)
save_data[key] = value
rc().hmset(rn('article:{0}'.format(self._id)), save_data)
self._set_data(**save_data)
def publish(self):
if self.has('publish_at'):
publish_at = self.get('publish_at')
else:
publish_at = int(time.time())
rc().zadd(rn('articles:published'), publish_at, self._id)
self.set(publish_at=publish_at)
clear_published_articles()
def create(self):
"""Creates initial values for an article"""
now = int(time.time())
rc().zadd(rn('articles'), now, self._id)
rc().hmset(rn('article:{0}'.format(self._id)),
{
'created_at': now,
'id': self._id
})
def write(key, data, timeout=86400):
"""set(), which would be a better name, is also an internal function, so
use 'write' as an alternative
@param key: Ientifies the data in redis
@param data: The data that will be pickled and commited to redis
@param timeout=86400: Timeout in seconds. 86400s=1d is default
"""
rc(redis_connection.REDIS_CACHE_DB).set(rn(key), pickle.dumps(data))
rc(redis_connection.REDIS_CACHE_DB).expire(rn(key), timeout)
def get_private():
"""Returns a list of articles that are neither published, nor scheduled,
and therefore private
"""
"""Intersecting three sets with 10M entries each takes about 75s in python,
so simply read all articles, published articles and pipeline from redis and
diff those in python
"""
articles = set(rc().zrevrange(rn('articles'), 0, -1))
published = set(rc().zrevrange(rn('articles:published'), 0, -1))
scheduled = set(rc().zrevrange(rn('pipeline'), 0, -1))
article_ids = articles - published - scheduled
return mget(*article_ids)
def _get_data_all(self):
pipeline = rc().pipeline()
pipeline.hgetall(rn('article:{0}'.format(self._id)))\
.get(rn('article:{0}:last_retrieval_at'.format(self._id)))\
.zincrby(rn('articles:times_retrieved'), self._id, 1)\
.set(rn('article:{0}:last_retrieval_at'.format(self._id)),
int(time.time()))
data = pipeline.execute()
return_data = data[0]
return_data['last_retrieval_at'] = data[1]
return return_data
def mread(*keys):
pipeline = rc(redis_connection.REDIS_CACHE_DB).pipeline()
for key in keys:
pipeline.get(rn(key))
values = pipeline.execute()
unpickled = []
for value in values:
if value is None:
unpickled.append(value)
else:
unpickled.append(pickle.loads(value))
return unpickled
def delete(self):
if not self.exists():
return # My work here is done
pipeline = rc().pipeline()
url_title = self._title_url(self.get('title'))
pipeline.delete(rn('article:title_to_id:{0}'.format(url_title)))\
.delete(rn('article:{0}'.format(self._id)))\
.delete(rn('article:{0}:last_retrieval_at'.format(self._id)))\
.zrem(rn('articles'), self._id)\
.zrem(rn('articles:published'), self._id)\
.zrem(rn('articles:times_retrieved'), self._id)\
.zrem(rn('pipeline'), self._id)
pipeline.execute()
clear_published_articles()
self._delete(self)
def read(key, data_provider, data_provider_parameters = []):
"""The complement to write would be read, not get - and set() can't be
used.
Articles is stored as a string in redis with SET, since SET is
O(1) and all other types are O(n)+ on retrieval. To be stored as a
string, it has to be pickled (serialized) and unpickled.
"""
data = rc(redis_connection.REDIS_CACHE_DB).get(rn(key))
if data is None:
data = data_provider(*data_provider_parameters)
write(key, data)
else:
data = pickle.loads(data)
return data
def _get_data(self):
self._data = rc().hgetall(rn('author:{0}'.format(self._id)))
def get_by_title(title):
id = rc().get(rn('article:title_to_id:{0}'.format(title)))
return get(id)
def get_articles(max_):
# -1 denotes the last entry in redis sets, 0 to -1 = all
if max_ is None:
max_ = -1
return rc().zrevrange(rn('articles:published'), 0, max_)
def unpublish(self):
rc().zrem(rn('articles:published'), self._id)
if self.has('publish_at'):
self.rem('publish_at')
clear_published_articles()
def schedule(self, timestamp):
rc().zadd(rn('pipeline'), timestamp, self._id)
self.set(publish_at=timestamp)
def is_scheduled(self):
return rc().zscore(rn('pipeline'), self._id) is not None
def save(new_slogan):
global slogan
slogan = new_slogan
rc().set(rn('slogan'), new_slogan)
rc().zadd(rn('slogans'), int(time.time()), new_slogan)
def delete(key):
rc(redis_connection.REDIS_CACHE_DB).delete(rn(key))
def is_published(self):
return rc().zscore(rn('articles:published'), self._id) is not None
def get():
global slogan
if slogan is None:
slogan = rc().get(rn('slogan'))
return slogan
def get_scheduled():
"""Returns a list of Article's that were written and scheduled for
publication sometime in the future
"""
article_ids = rc().zrevrange(rn('pipeline'), 0, -1)
return mget(*article_ids)
def get_by_name(name):
id_ = rc().get(rn('author:name_to_id:{0}'.format(name)))
return get(id_)
def exists(self):
"""zscore is faster than zrank"""
return rc().zscore(rn('articles'), self._id) is not None
from hobonaut.models.redis_connection import rc, rn
author = {
'id': '1',
'name': 'some_author',
'email': '*****@*****.**',
'password_hash': '',
'password_salt': ''
}
rc().hmset(rn('author:{0}'.format(author['id'])), author)
rc().set(rn('author:name_to_id:{0}'.format(author['name'])), author['id'])
rc().sadd(rn('authors'), author['id'])
super(Test, self).__init__()
def _get_data_all(self):
data = cc.hgetall(rn(self._id))
if len(data) == 0:
raise cache.ObjectNotFoundError(self._id)
return data
def set(self, **kwargs):
cc.hmset(rn(self._id), kwargs)
self._set_data(**kwargs)
if __name__ == '__main__':
redis_connection.set_namespace('testing')
cc = rc(redis_connection.REDIS_CACHE_DB)
data = [
('testing',
{
'a': random.randint(40, 50),
'b': random.randint(51, 60),
'c': random.randint(71, 80)
}
),
('the_lord',
{
'name': 'Lord Testing',
'county': 'Testingshire',
'planet': 'Cerebrus XXIV'
}
),
from hobonaut.models.redis_connection import rn, rc
from hobonaut.models import articles
import time
if __name__ == '__main__':
last_run = rc().get(rn('pipeline:last_run'))
if last_run is None:
last_run = '-inf'
now = int(time.time())
publishable_articles = rc().zrevrangebyscore(rn('pipeline'), now, last_run)
for article_id in publishable_articles:
article = articles.get(article_id)
article.publish()
rc().zrem(rn('pipeline'), article_id)
rc().set(rn('pipeline:last_run'), now)
def get(id_):
if not rc().sismember(rn('authors'), id_):
raise AuthorNotFoundError(id_)
return Author(id_)
def _cb_rem(self, fields):
""" Delete fields from article hash """
for field in fields:
rc().hdel(rn('article:{0}'.format(self._id)), field)
