def set(self, **data):
"""Saves given key=value pairs and updates cache
"""
if not self.exists():
self.create()
save_data = {}
save_data['updated_at'] = int(time.time())
for key, value in data.iteritems():
"""Special cases that require more actions than just adding
them to the article hash
"""
if key == 'author':
# remove previous author
if 'author' in self._data:
key = 'author:{0}:articles'.format(self._data['author'])
rc().srem(rn(key), self._id)
rc().sadd(rn('author:{0}:articles'.format(key)), self._id)
if key == 'title':
if self.has('title'):
old_url_title = self._title_url(self.get('title'))
rc().delete(rn('article:title_to_id:{0}'\
.format(old_url_title)))
url_title = self._title_url(value)
if len(url_title) > 0:
rc().set(rn('article:title_to_id:{0}'.format(url_title)),
self._id)
if key == 'text':
save_data['text_compiled'] = markdown.markdown(value)
save_data[key] = value
rc().hmset(rn('article:{0}'.format(self._id)), save_data)
self._set_data(**save_data)
def create(self):
"""Creates initial values for an article"""
now = int(time.time())
rc().zadd(rn('articles'), now, self._id)
rc().hmset(rn('article:{0}'.format(self._id)),
{
'created_at': now,
'id': self._id
})
def write(key, data, timeout=86400):
"""set(), which would be a better name, is also an internal function, so
use 'write' as an alternative
@param key: Ientifies the data in redis
@param data: The data that will be pickled and commited to redis
@param timeout=86400: Timeout in seconds. 86400s=1d is default
"""
rc(redis_connection.REDIS_CACHE_DB).set(rn(key), pickle.dumps(data))
rc(redis_connection.REDIS_CACHE_DB).expire(rn(key), timeout)
def _get_data_all(self):
pipeline = rc().pipeline()
pipeline.hgetall(rn('article:{0}'.format(self._id)))\
.get(rn('article:{0}:last_retrieval_at'.format(self._id)))\
.zincrby(rn('articles:times_retrieved'), self._id, 1)\
.set(rn('article:{0}:last_retrieval_at'.format(self._id)),
int(time.time()))
data = pipeline.execute()
return_data = data[0]
return_data['last_retrieval_at'] = data[1]
return return_data
def get_private():
"""Returns a list of articles that are neither published, nor scheduled,
and therefore private
"""
"""Intersecting three sets with 10M entries each takes about 75s in python,
so simply read all articles, published articles and pipeline from redis and
diff those in python
"""
articles = set(rc().zrevrange(rn('articles'), 0, -1))
published = set(rc().zrevrange(rn('articles:published'), 0, -1))
scheduled = set(rc().zrevrange(rn('pipeline'), 0, -1))
article_ids = articles - published - scheduled
return mget(*article_ids)
def publish(self):
if self.has('publish_at'):
publish_at = self.get('publish_at')
else:
publish_at = int(time.time())
rc().zadd(rn('articles:published'), publish_at, self._id)
self.set(publish_at=publish_at)
clear_published_articles()
def mread(*keys):
pipeline = rc(redis_connection.REDIS_CACHE_DB).pipeline()
for key in keys:
pipeline.get(rn(key))
values = pipeline.execute()
unpickled = []
for value in values:
if value is None:
unpickled.append(value)
else:
unpickled.append(pickle.loads(value))
return unpickled
def read(key, data_provider, data_provider_parameters = []):
"""The complement to write would be read, not get - and set() can't be
used.
Articles is stored as a string in redis with SET, since SET is
O(1) and all other types are O(n)+ on retrieval. To be stored as a
string, it has to be pickled (serialized) and unpickled.
"""
data = rc(redis_connection.REDIS_CACHE_DB).get(rn(key))
if data is None:
data = data_provider(*data_provider_parameters)
write(key, data)
else:
data = pickle.loads(data)
return data
def delete(self):
if not self.exists():
return # My work here is done
pipeline = rc().pipeline()
url_title = self._title_url(self.get('title'))
pipeline.delete(rn('article:title_to_id:{0}'.format(url_title)))\
.delete(rn('article:{0}'.format(self._id)))\
.delete(rn('article:{0}:last_retrieval_at'.format(self._id)))\
.zrem(rn('articles'), self._id)\
.zrem(rn('articles:published'), self._id)\
.zrem(rn('articles:times_retrieved'), self._id)\
.zrem(rn('pipeline'), self._id)
pipeline.execute()
clear_published_articles()
self._delete(self)
def is_scheduled(self):
return rc().zscore(rn('pipeline'), self._id) is not None
def is_published(self):
return rc().zscore(rn('articles:published'), self._id) is not None
def _cb_rem(self, fields):
""" Delete fields from article hash """
for field in fields:
rc().hdel(rn('article:{0}'.format(self._id)), field)
def get_articles(max_):
# -1 denotes the last entry in redis sets, 0 to -1 = all
if max_ is None:
max_ = -1
return rc().zrevrange(rn('articles:published'), 0, max_)
def schedule(self, timestamp):
rc().zadd(rn('pipeline'), timestamp, self._id)
self.set(publish_at=timestamp)
def get_scheduled():
"""Returns a list of Article's that were written and scheduled for
publication sometime in the future
"""
article_ids = rc().zrevrange(rn('pipeline'), 0, -1)
return mget(*article_ids)
def set(self, **kwargs):
cc.hmset(rn(self._id), kwargs)
self._set_data(**kwargs)
def _get_data_all(self):
data = cc.hgetall(rn(self._id))
if len(data) == 0:
raise cache.ObjectNotFoundError(self._id)
return data
}
),
('house',
{
'color': 'red',
'location': 'Londonshire',
'owner': 'Lord Testington',
'size': '13252sqft',
'value': '13352267 pounds'
}
)
]
for key, value in data:
for subkey, subvalue in value.iteritems():
cc.hset(rn(key), subkey, subvalue)
list_ = [Test('testing'), Test('the_lord'), Test('house')]
try:
int(list_[0].get('a'))
except:
print 'First data set failed, returned ' + list_[0].get('a')
assert list_[1].get('name') == 'Lord Testing', 'Second data set failed,\
returned ' + list_[1].get('name')
assert list_[2].get('color') == 'red', 'Third data set failed, returned '\
+ list_[2].get('color')
rand1, rand2 = random.randint(90, 99), random.randint(30, 39)
list_[1].set(one=rand1, two=rand2)
assert list_[1].get('one') == rand1, 'Data set failure on second\
data set, variable 1'
assert list_[1].get('two') == rand2, 'Data set failure on second\
from hobonaut.models.redis_connection import rn, rc
from hobonaut.models import articles
import time
if __name__ == '__main__':
last_run = rc().get(rn('pipeline:last_run'))
if last_run is None:
last_run = '-inf'
now = int(time.time())
publishable_articles = rc().zrevrangebyscore(rn('pipeline'), now, last_run)
for article_id in publishable_articles:
article = articles.get(article_id)
article.publish()
rc().zrem(rn('pipeline'), article_id)
rc().set(rn('pipeline:last_run'), now)
def get():
global slogan
if slogan is None:
slogan = rc().get(rn('slogan'))
return slogan
def delete(key):
rc(redis_connection.REDIS_CACHE_DB).delete(rn(key))
def unpublish(self):
rc().zrem(rn('articles:published'), self._id)
if self.has('publish_at'):
self.rem('publish_at')
clear_published_articles()
def get_by_name(name):
id_ = rc().get(rn('author:name_to_id:{0}'.format(name)))
return get(id_)
def get_by_title(title):
id = rc().get(rn('article:title_to_id:{0}'.format(title)))
return get(id)
def save(new_slogan):
global slogan
slogan = new_slogan
rc().set(rn('slogan'), new_slogan)
rc().zadd(rn('slogans'), int(time.time()), new_slogan)
from hobonaut.models.redis_connection import rc, rn
author = {
'id': '1',
'name': 'some_author',
'email': '*****@*****.**',
'password_hash': '',
'password_salt': ''
}
rc().hmset(rn('author:{0}'.format(author['id'])), author)
rc().set(rn('author:name_to_id:{0}'.format(author['name'])), author['id'])
rc().sadd(rn('authors'), author['id'])
def _get_data(self):
self._data = rc().hgetall(rn('author:{0}'.format(self._id)))
def exists(self):
"""zscore is faster than zrank"""
return rc().zscore(rn('articles'), self._id) is not None
def get(id_):
if not rc().sismember(rn('authors'), id_):
raise AuthorNotFoundError(id_)
return Author(id_)
