class TestCache(unittest.TestCase):
def setUp(self):
self.client = Redis()
self.client.flushdb()
self.cache = Cache(self.client, "hash-key")
def test_set_and_get(self):
self.assertIsNone(self.cache.get("msg"))
self.cache.set("msg", "hello world")
self.assertEqual(self.cache.get("msg"), "hello world")
def test_is_exists(self):
self.assertFalse(self.cache.is_exists("msg"))
self.cache.set("msg", "hello world")
self.assertTrue(self.cache.is_exists("msg"))
def test_size(self):
self.assertEqual(self.cache.size(), 0)
self.cache.set("msg", "hello world")
self.assertEqual(self.cache.size(), 1)
def test_delete(self):
self.assertFalse(self.cache.delete("msg"))
self.cache.set("msg", "hello world")
self.assertTrue(self.cache.delete("msg"))
def cache_test():
"""
TEST CASE FOR CACHE SYSTEM.
:return:
"""
# Test the cache
Keys = [i for i in range(random.randint(1, 10))] # Total Entries
place = [
'Imalia',
'Brazil',
'Canada',
'Zimbabawe',
'Saturn?',
'New Zealand',
'São Paulo',
'Dubai',
'Mars?',
'Singapore',
'Los Angeles',
'Santos',
'São Tomé',
'Luxemburg',
'New Deli',
'Abu Dhabi',
'São Vicente',
]
# Cache object
cache = Cache()
# Updating Cache with entries
for i, key in enumerate(Keys):
if key in cache:
continue
else:
value = ''.join([random.choice(place)])
print('\t', value)
cache.update(key, value)
print("{0}.Iteration, #{1} cached entries".format(i + 1, cache.size()))
# Cache List
print('\n\n\t', '*' * 10, ' Places List ', '*' * 10)
for k, v in cache.view().items():
print("{0} : {1}".format(k, v))
print("Cache size: ", cache.size())
print("Cache overview: ", cache.view())
print("delete the oldest item from cache: ", cache.delete())
print("Erasing the cache: ", cache.empty())
# end of the test
print('_' * 60)
def delete(url):
"""Delete <url> from cache"""
url = urllib.unquote(url)
if url.endswith("/"):
raise Error("illegal url - can't delete a directory")
print "* del: removing file from cache..."
cache = Cache()
result = cache.delete(url)
return result
def delete(url):
"""Delete <url> from cache"""
url = urllib.unquote(url)
if url.endswith("/"):
raise Error("illegal url - can't delete a directory")
print "* del: removing file from cache..."
cache = Cache()
result = cache.delete(url)
return result
class Twitter:
"""
Module for core twitter service.
"""
def __init__(self, mc, config):
self.timelines = defaultdict(list)
self.relations = defaultdict(set)
self.tweets_cache = config['tweets_cache']
self.feed_num = config['feed_num']
self.relation_cache = Cache(mc, prefix='REL', dup=False)
self.user_tweet_cache = Cache(mc,
prefix='TWT',
limit=self.tweets_cache)
self.tweet_cache = Cache(mc)
def post_tweet(self, user_id, tweet):
# create db entry
kwargs = {
'user_id': user_id,
'content': tweet,
'time_stamp': str(int(time.time()))
}
tweet_db = model.Tweet.create(**kwargs)
# update user's top tweets
self.user_tweet_cache.update(
user_id, (tweet_db.id, tweet_db.user_id, tweet_db.time_stamp))
# cache tweets
self.tweet_cache.set(str(tweet_db.id), tweet)
return tweet_db.id
def follow(self, follower_id, followee_id):
if follower_id == followee_id:
return
# create db entry
kwargs = {'follower': follower_id, 'followee': followee_id}
model.Relation.create(**kwargs)
# update user relation cache
self.relation_cache.update(follower_id, followee_id)
def unfollow(self, follower_id, followee_id):
if follower_id == followee_id:
return
try:
# create db entry
model.Relation.get(follower=follower_id,
followee=followee_id).delete()
except DoesNotExist:
log.info('Relation entry not found %s %s' %
(follower_id, followee_id))
# update user relation cache
self.relation_cache.remove(follower_id, followee_id)
def get_feed(self, user_id):
candidates = []
for user in self.get_follow(user_id) + [user_id]:
log.info('Search tweets for user %s' % user)
# try get user top tweets from cache first
res = self.user_tweet_cache.get(user)
if res is None:
# get tweets from database
candidates += self.get_tweet(user,
limit=self.tweets_cache,
update_cache=True)
else:
# load tweets from cache
log.info('Load tweets from cache for user %s' % user)
for tweet in res:
candidates.append(
Tweet(tweet[0], tweet[1], tweet[2], self.tweet_cache))
# sort tweets
candidates.sort(key=lambda x: x.timestamp, reverse=True)
return [str(c) for c in candidates[:self.feed_num]]
def get_tweet(self, user_id, limit=None, update_cache=False):
# read tweets from db
tweets = sorted(model.Tweet.filter(user_id=user_id),
key=lambda x: x.time_stamp,
reverse=True)
candidates = []
cache_list = []
for i in range(len(tweets)):
if limit is not None and i > limit:
break
tweet = tweets[i]
log.info('Find tweets %s in db for %s' % (tweet.id, tweet.user_id))
candidates.append(
Tweet(tweet.id, tweet.user_id, tweet.time_stamp,
self.tweet_cache))
cache_list.append((tweet.id, tweet.user_id, tweet.time_stamp))
if update_cache:
# update user's top tweet cache
self.user_tweet_cache.set(user_id, cache_list)
return candidates
def delete_tweet(self, user_id, tweet_id):
try:
# delete db entry
tweet = model.Tweet.get(id=tweet_id)
except DoesNotExist:
log.warning('Tweet %s not found' % id)
# update user top tweets cache
self.user_tweet_cache.remove(
user_id, (tweet.id, tweet.user_id, tweet.time_stamp))
tweet.delete()
def get_follow(self, user_id):
# try get followee from cache
res = self.relation_cache.get(user_id)
if res is None:
# get followee from db
res = [
str(x.followee)
for x in model.Relation.filter(follower=user_id)
]
self.relation_cache.set(user_id, res)
else:
log.info('Get follow from cache for user %s' % user_id)
return res
def add_user(self, name):
# create db entry
kwargs = {'name': name}
return model.User.create(**kwargs).id
def get_user(self, id):
try:
# get user from db
return model.User.get(id=id).name
except DoesNotExist:
log.warning('User %s not found' % id)
return None
def delete_user(self, id):
try:
# delete db entry
model.User.get(id=id).delete()
except DoesNotExist:
log.warning('User %s not found' % id)
# delete relation
for relation in model.Relation.filter(followee=id).allow_filtering():
self.unfollow(relation.follower, relation.followee)
for relation in model.Relation.filter(follower=id):
relation.delete()
# delete cache
self.relation_cache.delete(id)
self.user_tweet_cache.delete(id)
for tweet in model.Tweet.filter(user_id=id):
tweet.delete()
def reset_seen():
redis = Cache()
n = redis.delete(*redis.keys('*-seen'))
return jsonify({'status_code': 200, 'data': n})
def reset_user_seen(mac_address):
redis = Cache()
n = redis.delete(*redis.keys('%s-seen' % (mac_address)) +
["__deletekey__"])
return jsonify({'status_code': 200, 'data': n})
def delete_user(mac_address):
redis = Cache()
print('Deleting entry for %s' % (mac_address))
success = redis.delete(mac_address) > 0
return jsonify({'status_code': 200, 'data': success})
class PollingServer:
def __init__(self, host, port, cache_size, db_name, workers):
self.host = host
self.port = port
self.cache = Cache(cache_size)
self.persistent = Persistent(db_name)
self.req_queue = Queue() # Unbounded queue
self.resp_queue = Queue()
self.pool = {}
self.workers = workers #number of workers in thread pool
self.polling = None
self.pending_reqs = {} #dict to store pending requests on a key
def init_helpers(self):
for i in range(self.workers):
self.pool[i] = Thread(target=io_handler, \
args=(self.req_queue, \
self.resp_queue,\
self.persistent, ))
def run_helpers(self):
for thread in self.pool.values():
thread.start()
def issue_request(self, req):
if req.key not in self.pending_reqs:
self.req_queue.put(req, block=False)
self.pending_reqs[req.key] = [req]
else:
pend = self.pending_reqs[req.key]
pend.append(req)
self.pending_reqs[req.key] = pend
def get_responses(self):
responses = []
for i in range(self.workers):
response = self.resp_queue.get(block=False)
if len(self.pending_reqs[response.key])==1:
del self.pending_reqs[response.key]
else:
self.req_queue.put(self.pending_reqs[response.key][0], block=False)
new_pending = self.pending_reqs[response.key]
new_pending.pop(0)
self.pending_reqs[response.key] = new_pending
responses.append(response)
if self.resp_queue.empty()==True:
break
return responses
def __bring_to_cache(self, key, value, dirty):
retkey, retval, succ = self.cache.insert(key, value, dirty)
if retkey and retval:
# Writeback dirty eviction
wb_req = Request('WRITEBACK', retkey, retval)
self.issue_request(wb_req)
return succ
def run_server(self):
self.polling = PollingSocket(self.host, self.port)
self.init_helpers()
self.run_helpers()
#num_req = 0
#first = True
#tot_time = 0
while True:
requests = self.polling.poll_connection()
#start = time.time()
#num_req += len(requests)
#if num_req:
# if first:
# tot_time = 0
# first = False
# if tot_time >= 1:
# with open('ep.lg', 'a') as f:
# f.write(str(num_req)+" "+str(tot_time)+"\n")
# #print(str(num_req)+" "+str(tot_time)+"\n")
# num_req = 0
# tot_time = 0
for request in requests:
#Request is a tuple of sock, request_data
data = request[1]
#Process request here
req = parse_req(data.decode('utf-8'))
req.fd = request[0]
if req.op == 'GET':
val = self.cache.get(req.key)
if val is None:
# Cache miss: non-blocking put to req_queue
# for helper_threads to consume
self.issue_request(req)
else:
resp = val
add_response(self.polling.sock_data_map, request[0], resp)
elif req.op == 'PUT':
retval = self.cache.put(req.key, req.value)
if retval is None:
#Cache miss
self.issue_request(req)
else:
resp = "ACK"
add_response(self.polling.sock_data_map, request[0], resp)
elif req.op in ['INSERT', 'DELETE']:
self.issue_request(req)
else:
resp = "-1".encode('utf-8')
add_response(self.polling.sock_data_map, request[0], resp)
# Consume from response queue (non-blocking)
try:
responses = self.get_responses()
for resp in responses:
#Ignore for write-back requests
if resp.fd is None:
continue
#resp.value is actual value for GET
#for PUT/INSERT resp.value is "ACK" message
#resp.value is "-1" for all errors
#Delete needs special check if nothing to
#delete from both cache and persistent
if resp.op != 'DELETE':
add_response(self.polling.sock_data_map, resp.fd, resp.value)
#Entry brought to cache line for GET/PUT miss and INSERT
if resp.op in ['GET', 'PUT', 'INSERT']:
if resp.value != "-1":
self.__bring_to_cache(resp.key, resp.value, False)
elif resp.op == 'DELETE':
result = self.cache.delete(req.key)
if resp.value == "-1" and result is False:
#Nothing to delete from cache/ persistent
add_response(self.polling.sock_data_map, resp.fd, "-1")
else:
add_response(self.polling.sock_data_map, resp.fd, "ACK")
else:
pass
#self.cache.show()
except:
pass
class Database(object):
def __init__(self, multi_value=False, cache_port=None, router=None, domain=None):
self._db = CommonDB(name=self.name, router=router, domain=domain)
self._multi_value = multi_value
if cache_port != None:
self._cache = Cache(cache_port)
else:
self._cache = None
@property
def name(self):
return self.__class__.__name__.lower()
def get(self, key, first=False):
val = None
cache = False
if self._cache:
cache = True
if not self._multi_value or first:
val = self._cache.get(key)
if not val:
coll = self._db.collection(key)
conn = self._db.connection(coll)
val = self._db.get(conn, key)
if self._multi_value:
if first and type(val) == list:
val = val[0]
else:
cache = False
if cache:
self._cache.put(key, val)
return val
def put(self, key, value):
if self._cache:
self._cache.delete(key)
if not self._multi_value:
val = {'$set':{VAL_NAME:value}}
else:
val = {'$addToSet':{VAL_NAME:value}}
coll = self._db.collection(key)
conn = self._db.connection(coll)
self._db.put(conn, key, val, create=True)
def delete(self, key, value=None, regex=False):
if self._cache:
self._cache.delete(key)
coll = self._db.collection(key)
conn = self._db.connection(coll)
if not value:
self._db.delete(conn, key)
else:
if not self._multi_value:
log_err(self, 'failed to delete')
raise Exception(log_get(self, 'failed to delete'))
if not regex:
self._db.put(conn, key, {'$pull':{VAL_NAME:value}})
else:
self._db.put(conn, key, {'$pull':{VAL_NAME:{'$regex':value}}})
class DataManager(DataTable):
"""
This class provides high level access to a database table.
It loads and saves data from the table into an object or a DataSet,
or a subclass thereof. The class of objects and data sets that
are returned can be set using the set_object() and set_dataset()
functions.
You can request a single object (row), or you can specify a set of criteria,
which are then translated into a WHERE clause.
"""
def __init__(self, table_name, field_dictionary):
DataTable.__init__(self, table_name, field_dictionary)
self.table = DataTable(table_name, field_dictionary)
self.reset_cache()
def reset_cache(self):
"""
Initialize this data manager's object cache.
Any contents in the cache are discarded.
"""
self.cache = Cache()
self.all_cached = 0
self.last_synched = ''
def preload(self):
"""
Handles a client request to preload all objects from the database,
fully populating the local cache and preparing for subsequent
client requests.
Preloading can result in significant performance benefits,
because data managers who have preloaded their cache can
fill more client requests from cache, avoiding expensive
database accesses.
Not all requests for preloading are honored. To be preloaded,
a datamanager's cache size must be set to CACHE_UNLIMITED.
Also, only one call is honored. Subsequent calls are
silently ignored. This makes it safe and efficient for
clients to request preloading, without needing to know
whether or not the data manager has already been preloaded.
"""
if self.all_cached == 1:
return
# FIXME: Also try to preload caches that are not unlimited,
# but are large enough to hold all existing objects.
# Use a SQL COUNT(*) function to make this determination,
# so we don't waste lots of time attempting to preload
# a cache that cannot be preloaded.
if self.cache.size <> CACHE_UNLIMITED:
return
#print 'Preloading ' + self.table.name
self.get_all()
#i18n_table = self.table.name + '_i18n'
#if self.dms.has_key(i18n_table):
# i18n_dm = self.dms[i18n_table]
# i18n_dm.preload()
def get_by_id(self, id):
"""
Returns an individual object whose primary key field matches
the specified id.
If the object is in the cache, the cached object is returned.
Objects which have more than one primary key field cannot be
reliably retrieved using this function. In this event, only the
first matching object will be returned.
"""
object = self.cache.get_by_key(id)
if object == None:
data_field = self.table.id_field()
sql = self.table.select + ' WHERE ' + data_field.field_name + '=' + data_field.attr_to_field(
id)
cursor = db.select(sql)
row = cursor.fetchone()
if row == None: return
object = self.row_to_object(row)
return object
def get_by_keys(self, filters):
"""
Returns all objects which match the supplied filters.
"""
if self.cache.filled == 1:
self.all_cached = 0
if self.all_cached == 1:
return self.get_cached_by_keys(filters)
else:
sql = self.filters_to_sql(filters)
return self.get_sql(sql)
def get_cached_by_keys(self, filters):
"""
This private function fills keyed requests directly from the
object cache.
No checking is performed to determine whether the cache contains
all objects which fit the request. Therefore, this function
should only be called by data managers whose caches are preloaded.
See the preload() function for more information on preloading.
"""
sql = self.filters_to_sql(filters)
function_text = self.filters_to_function(filters)
print 'Function text: '
print function_text
code = compile(function_text, '<string>', 'exec')
print 'Code: ' + str(code)
print 'Code has %s arguments.' % code.co_argcount
self.test_object_filters.im_func.func_code = code
#print 'Method code: ' + str(self.test_object_filters.im_func.func_code)
good_keys = filter(self.test_object_filters, self.cache.keys())
print 'Good keys: ' + str(good_keys)
dataset = self.new_dataset()
for key in good_keys:
dataset[key] = self.cache[key]
return dataset
def test_object_filters(self, key):
return 1
def filters_to_function(self, filters):
"""
Converts a list of filters into a Python function which tests
an object to see if it matches the filters.
Precompiling filter tests speeds up key filtering enormously.
The generated function accepts a single parameter, "key".
It retrieves the object with that key in the object
cache and tests for a match. If the object matches all
the filters, the generated function returns 1.
Otherwise, it returns 0.
"""
code = WOStringIO()
code.write('def test_cached_object(key):\n')
code.write(' object = self.cache[key]\n')
for filter in filters:
attribute, operator, value = filter
test_value = repr(value)
code.write(' obj_value = object.%s\n' % (attribute))
if operator.upper() == 'LIKE':
code.write(' if %s > len(%s): return 0\n' %
(len(value), obj_value))
code.write(' return (%s <> obj_value.upper()[:%s])\n' %
(test_value.upper(), len(value)))
elif operator in ['<>', '<', '<=', '=', '>=', '>']:
if operator == '=':
operator = '=='
code.write(' return (object.%s %s %s)\n' %
(attribute, operator, repr(value)))
else:
raise UnknownOperator('Unrecognized operator: %s' % (operator))
return code.get_value()
def filters_to_sql(self, filters):
"""
Converts a list of filters into the SQL statement which will
retrieve matching records from the database.
"""
wheres = []
for filter in filters:
attribute, operator, value = filter
field = self.table.fields.find_attribute(attribute)
if operator.upper() == 'LIKE':
wheres.append('upper(' + field_name + ') LIKE ' +
field.attr_to_field(value.upper() + '%'))
else:
wheres.append(field.field_name + operator +
field.attr_to_field(value))
where = ' WHERE ' + string.join(wheres, ' AND ')
return self.table.select + where
def get_all(self):
"""
Returns a set of all objects managed by this data manager.
If the data manager's cache proves sufficient to cache all objects,
the cache will subsequently be considered preloaded, i.e.,
subsequent calls to get_by_keys() will be served directly from the
cache, bypassing expensive database accesses. See the preload()
function for more information on preloading.
"""
if self.cache.filled == 1:
self.all_cached = 0
if self.all_cached == 0:
#print 'Loading all of ' + self.table.name + ' into cache.'
set = self.get_sql(self.table.select)
if self.cache.filled == 0:
self.all_cached = 1
return set
return self.get_cached()
def synch(self):
"""
Synchronize objects in the object cache with the database.
Objects which have been deleted in the database are removed
from the object cache.
Objects which are out of synch with their database record
have their attribute set to match the data in the database.
"""
#print 'Synchronizing ' + self.table.name + ' with database'
last_synched = self.last_synched # Remember this, because we're about to overwrite it.
self.last_synched = now_string()
# Delete any newly deleted objects.
sql = 'SELECT identifier FROM deleted WHERE table_name=' + wsq(
self.table.name) + ' AND deleted >= ' + wsq(last_synched)
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row == None: break
# Load keys for the deleted object
object = self.new_object()
if len(self.table.key_list) == 1:
field = self.table.fields[self.table.key_list[0]]
value = field.field_to_attr(row[0])
setattr(object, field.attribute, value)
else:
values = row[0].split()
for key in self.table.key_list:
field = self.table.fields[key]
value = field.field_to_attr(values[field.index])
setattr(object, field.attribute, value)
object.key = self.table.get_key(object)
#print 'Deleting from ' + self.table.name + ' cache: ' + str(value)
self.cache.delete(object)
# FIXME: Delete the object from all data sets which contain it!
# Update any newly updated objects.
sql = self.table.select + ' WHERE updated >= ' + wsq(last_synched)
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row == None: break
key = self.table.get_row_key(row)
if self.cache.has_key(key):
object = self.cache[key]
self.table.load_row(object, row)
#print 'Updating in ' + self.table.name + ' cache: ' + str(object.key)
else:
object = self.row_to_object(row)
self.cache.add(object)
#print 'Adding in ' + self.table.name + ' cache: ' + str(object.key)
# FIXME: Add the object to all data sets whose filters it matches.
def get_cached(self):
"""
Returns a dataset containing all objects in the object cache.
"""
#print 'Pulling ' + self.table.name + ' from cache.'
dataset = self.new_dataset()
for key in self.cache.keys():
dataset[key] = self.cache[key]
return dataset
def get_sql(self, sql):
"""
Accepts a SQL statement, instantiates the corresponding objects from the
database, and stores those objects in the data cache if possible.
"""
#print 'Cache miss, loading: ' + self.table.name
dataset = self.new_dataset()
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row == None: break
object = self.row_to_object(row)
dataset[object.key] = object
self.cache.add(object)
return dataset
def set_object_class(self, object_class):
self.object_class = object_class
def set_dataset_class(self, dataset_class):
self.dataset_class = dataset_class
def new_object(self):
object = self.object_class(self.dms, self)
for key in self.table.fields.keys():
field = self.table.fields[key]
setattr(object, field.attribute, field.get_default())
object.changed = 0
object.in_database = 0
return object
def new_dataset(self):
return self.dataset_class(self)
def row_to_object(self, row):
object = self.new_object()
self.table.load_row(object, row)
return object
def add(self, object):
self.save(object)
def save(self, object):
object.key = self.table.get_key(
object) # New objects need their key calculated.
if object.changed == 0 and object.in_database == 0:
return
if object.in_database == 0:
field_list = []
value_list = []
for key in self.table.field_list:
field = self.table.fields[key]
if field.data_type == 'created': # The database is responsible for setting the timestamp.
continue
if field.data_type == 'sequence': # When inserting, always increment the value.
new_id = db.next_id(self.name, field.field_name)
setattr(object, field.attribute, new_id)
value = field.attr_to_field(getattr(object, field.attribute))
field_list.append(field.field_name)
value_list.append(value)
sql = 'INSERT INTO %s (%s) VALUES (%s)' % (
self.table.name, string.join(
field_list, ', '), string.join(value_list, ', '))
else:
update_list = []
where_list = []
for key in self.table.field_list:
field = self.table.fields[key]
if field.data_type == 'created':
continue
if field.data_type == 'updated':
value = wsq(now_string())
else:
value = field.attr_to_field(
getattr(object, field.attribute))
update_list.append(field.field_name + '=' + value)
if field.key_field == 1:
where_list.append(field.field_name + '=' + value)
sql = 'UPDATE %s SET %s WHERE %s' % (
self.table.name, string.join(
update_list, ', '), string.join(where_list, ' AND '))
# print sql
db.runsql(sql)
db.commit()
self.cache.add(object)
object.in_database = 1
object.changed = 0
def delete(self, object):
if object.in_database == 0:
return
self.cache.delete(object)
wheres = []
for key in self.table.key_list:
data_field = self.table.fields[key]
value = data_field.attr_to_field(
getattr(object, data_field.attribute))
wheres.append(data_field.field_name + '=' + value)
where = ' WHERE ' + string.join(wheres, ' AND ')
sql = 'DELETE FROM %s %s' % (self.table.name, where)
db.runsql(sql)
db.commit()
sql = 'INSERT INTO deleted (table_name, identifier) VALUES (%s, %s)' % (
wsq(self.table.name), wsq(str(object.key)))
db.runsql(sql)
db.commit()
def delete_by_keys(self, filters):
dataset = self.get_by_keys(filters)
for key in dataset.keys():
object = dataset[key]
self.delete(object)
def clear(self, dataset):
for key in dataset.keys():
self.delete(dataset[key])
class DataManager(DataTable):
"""
This class provides high level access to a database table.
It loads and saves data from the table into an object or a DataSet,
or a subclass thereof. The class of objects and data sets that
are returned can be set using the set_object() and set_dataset()
functions.
You can request a single object (row), or you can specify a set of criteria,
which are then translated into a WHERE clause.
"""
def __init__(self, table_name, field_dictionary):
DataTable.__init__(self, table_name, field_dictionary)
self.table = DataTable(table_name, field_dictionary)
self.reset_cache()
def reset_cache(self):
"""
Initialize this data manager's object cache.
Any contents in the cache are discarded.
"""
self.cache = Cache()
self.all_cached = 0
self.last_synched = ''
def preload(self):
"""
Handles a client request to preload all objects from the database,
fully populating the local cache and preparing for subsequent
client requests.
Preloading can result in significant performance benefits,
because data managers who have preloaded their cache can
fill more client requests from cache, avoiding expensive
database accesses.
Not all requests for preloading are honored. To be preloaded,
a datamanager's cache size must be set to CACHE_UNLIMITED.
Also, only one call is honored. Subsequent calls are
silently ignored. This makes it safe and efficient for
clients to request preloading, without needing to know
whether or not the data manager has already been preloaded.
"""
if self.all_cached==1:
return
# FIXME: Also try to preload caches that are not unlimited,
# but are large enough to hold all existing objects.
# Use a SQL COUNT(*) function to make this determination,
# so we don't waste lots of time attempting to preload
# a cache that cannot be preloaded.
if self.cache.size <> CACHE_UNLIMITED:
return
#print 'Preloading ' + self.table.name
self.get_all()
#i18n_table = self.table.name + '_i18n'
#if self.dms.has_key(i18n_table):
# i18n_dm = self.dms[i18n_table]
# i18n_dm.preload()
def get_by_id(self, id):
"""
Returns an individual object whose primary key field matches
the specified id.
If the object is in the cache, the cached object is returned.
Objects which have more than one primary key field cannot be
reliably retrieved using this function. In this event, only the
first matching object will be returned.
"""
object = self.cache.get_by_key(id)
if object==None:
data_field = self.table.id_field()
sql = self.table.select + ' WHERE ' + data_field.field_name + '=' + data_field.attr_to_field(id)
cursor = db.select(sql)
row = cursor.fetchone()
if row==None: return
object = self.row_to_object(row)
return object
def get_by_keys(self, filters):
"""
Returns all objects which match the supplied filters.
"""
if self.cache.filled==1:
self.all_cached = 0
if self.all_cached==1:
return self.get_cached_by_keys(filters)
else:
sql = self.filters_to_sql(filters)
return self.get_sql(sql)
def get_cached_by_keys(self, filters):
"""
This private function fills keyed requests directly from the
object cache.
No checking is performed to determine whether the cache contains
all objects which fit the request. Therefore, this function
should only be called by data managers whose caches are preloaded.
See the preload() function for more information on preloading.
"""
sql = self.filters_to_sql(filters)
function_text = self.filters_to_function(filters)
print 'Function text: '
print function_text
code = compile(function_text, '<string>', 'exec')
print 'Code: ' + str(code)
print 'Code has %s arguments.' % code.co_argcount
self.test_object_filters.im_func.func_code = code
#print 'Method code: ' + str(self.test_object_filters.im_func.func_code)
good_keys = filter(self.test_object_filters, self.cache.keys())
print 'Good keys: ' + str(good_keys)
dataset = self.new_dataset()
for key in good_keys:
dataset[key] = self.cache[key]
return dataset
def test_object_filters(self, key):
return 1
def filters_to_function(self, filters):
"""
Converts a list of filters into a Python function which tests
an object to see if it matches the filters.
Precompiling filter tests speeds up key filtering enormously.
The generated function accepts a single parameter, "key".
It retrieves the object with that key in the object
cache and tests for a match. If the object matches all
the filters, the generated function returns 1.
Otherwise, it returns 0.
"""
code = WOStringIO()
code.write('def test_cached_object(key):\n')
code.write(' object = self.cache[key]\n')
for filter in filters:
attribute, operator, value = filter
test_value = repr(value)
code.write(' obj_value = object.%s\n' % (attribute))
if operator.upper()=='LIKE':
code.write(' if %s > len(%s): return 0\n' % (len(value), obj_value))
code.write(' return (%s <> obj_value.upper()[:%s])\n' % (test_value.upper(), len(value)))
elif operator in ['<>', '<', '<=', '=', '>=', '>']:
if operator=='=':
operator = '=='
code.write(' return (object.%s %s %s)\n' % (attribute, operator, repr(value)))
else:
raise UnknownOperator('Unrecognized operator: %s' % (operator))
return code.get_value()
def filters_to_sql(self, filters):
"""
Converts a list of filters into the SQL statement which will
retrieve matching records from the database.
"""
wheres = []
for filter in filters:
attribute, operator, value = filter
field = self.table.fields.find_attribute(attribute)
if operator.upper()=='LIKE':
wheres.append('upper(' + field_name + ') LIKE ' + field.attr_to_field(value.upper() + '%'))
else:
wheres.append(field.field_name + operator + field.attr_to_field(value))
where = ' WHERE ' + string.join(wheres, ' AND ')
return self.table.select + where
def get_all(self):
"""
Returns a set of all objects managed by this data manager.
If the data manager's cache proves sufficient to cache all objects,
the cache will subsequently be considered preloaded, i.e.,
subsequent calls to get_by_keys() will be served directly from the
cache, bypassing expensive database accesses. See the preload()
function for more information on preloading.
"""
if self.cache.filled==1:
self.all_cached = 0
if self.all_cached==0:
#print 'Loading all of ' + self.table.name + ' into cache.'
set = self.get_sql(self.table.select)
if self.cache.filled==0:
self.all_cached = 1
return set
return self.get_cached()
def synch(self):
"""
Synchronize objects in the object cache with the database.
Objects which have been deleted in the database are removed
from the object cache.
Objects which are out of synch with their database record
have their attribute set to match the data in the database.
"""
#print 'Synchronizing ' + self.table.name + ' with database'
last_synched = self.last_synched # Remember this, because we're about to overwrite it.
self.last_synched = now_string()
# Delete any newly deleted objects.
sql = 'SELECT identifier FROM deleted WHERE table_name=' + wsq(self.table.name) + ' AND deleted >= ' + wsq(last_synched)
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row==None: break
# Load keys for the deleted object
object = self.new_object()
if len(self.table.key_list)==1:
field = self.table.fields[self.table.key_list[0]]
value = field.field_to_attr(row[0])
setattr(object, field.attribute, value)
else:
values = row[0].split()
for key in self.table.key_list:
field = self.table.fields[key]
value = field.field_to_attr(values[field.index])
setattr(object, field.attribute, value)
object.key = self.table.get_key(object)
#print 'Deleting from ' + self.table.name + ' cache: ' + str(value)
self.cache.delete(object)
# FIXME: Delete the object from all data sets which contain it!
# Update any newly updated objects.
sql = self.table.select + ' WHERE updated >= ' + wsq(last_synched)
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row==None: break
key = self.table.get_row_key(row)
if self.cache.has_key(key):
object = self.cache[key]
self.table.load_row(object, row)
#print 'Updating in ' + self.table.name + ' cache: ' + str(object.key)
else:
object = self.row_to_object(row)
self.cache.add(object)
#print 'Adding in ' + self.table.name + ' cache: ' + str(object.key)
# FIXME: Add the object to all data sets whose filters it matches.
def get_cached(self):
"""
Returns a dataset containing all objects in the object cache.
"""
#print 'Pulling ' + self.table.name + ' from cache.'
dataset = self.new_dataset()
for key in self.cache.keys():
dataset[key] = self.cache[key]
return dataset
def get_sql(self, sql):
"""
Accepts a SQL statement, instantiates the corresponding objects from the
database, and stores those objects in the data cache if possible.
"""
#print 'Cache miss, loading: ' + self.table.name
dataset = self.new_dataset()
cursor = db.select(sql)
while (1):
row = cursor.fetchone()
if row==None: break
object = self.row_to_object(row)
dataset[object.key] = object
self.cache.add(object)
return dataset
def set_object_class(self, object_class):
self.object_class = object_class
def set_dataset_class(self, dataset_class):
self.dataset_class = dataset_class
def new_object(self):
object = self.object_class(self.dms, self)
for key in self.table.fields.keys():
field = self.table.fields[key]
setattr(object, field.attribute, field.get_default())
object.changed = 0
object.in_database = 0
return object
def new_dataset(self):
return self.dataset_class(self)
def row_to_object(self, row):
object = self.new_object()
self.table.load_row(object, row)
return object
def add(self, object):
self.save(object)
def save(self, object):
object.key = self.table.get_key(object) # New objects need their key calculated.
if object.changed==0 and object.in_database==0:
return
if object.in_database==0:
field_list = []
value_list = []
for key in self.table.field_list:
field = self.table.fields[key]
if field.data_type=='created': # The database is responsible for setting the timestamp.
continue
if field.data_type=='sequence': # When inserting, always increment the value.
new_id = db.next_id(self.name, field.field_name)
setattr(object, field.attribute, new_id)
value = field.attr_to_field(getattr(object, field.attribute))
field_list.append(field.field_name)
value_list.append(value)
sql = 'INSERT INTO %s (%s) VALUES (%s)' % (self.table.name, string.join(field_list, ', '), string.join(value_list, ', '))
else:
update_list = []
where_list = []
for key in self.table.field_list:
field = self.table.fields[key]
if field.data_type=='created':
continue
if field.data_type=='updated':
value = wsq(now_string())
else:
value = field.attr_to_field(getattr(object, field.attribute))
update_list.append(field.field_name + '=' + value)
if field.key_field==1:
where_list.append(field.field_name + '=' + value)
sql = 'UPDATE %s SET %s WHERE %s' % (self.table.name, string.join(update_list, ', '), string.join(where_list, ' AND '))
# print sql
db.runsql(sql)
db.commit()
self.cache.add(object)
object.in_database = 1
object.changed = 0
def delete(self, object):
if object.in_database==0:
return
self.cache.delete(object)
wheres = []
for key in self.table.key_list:
data_field = self.table.fields[key]
value = data_field.attr_to_field(getattr(object, data_field.attribute))
wheres.append(data_field.field_name + '=' + value)
where = ' WHERE ' + string.join(wheres, ' AND ')
sql = 'DELETE FROM %s %s' % (self.table.name, where)
db.runsql(sql)
db.commit()
sql = 'INSERT INTO deleted (table_name, identifier) VALUES (%s, %s)' % (wsq(self.table.name), wsq(str(object.key)))
db.runsql(sql)
db.commit()
def delete_by_keys(self, filters):
dataset = self.get_by_keys(filters)
for key in dataset.keys():
object = dataset[key]
self.delete(object)
def clear(self, dataset):
for key in dataset.keys():
self.delete(dataset[key])
