def testSanity(self):
'''Tests the sanity of Reference Property'''
from homer.core.commons import String
print "######## Creating Models ################"
@key("name")
class Person(Model):
name = String(required = True)
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = Reference(Person)
print "Persisting Person"
person = Person(name = "sasuke")
self.db.save(person)
print "Persisting Book"
book = Book(name = "Pride", author = person)
self.db.save(book)
print "Checking Conversion Routine"
k = eval(Book.author.convert(person))
self.assertTrue(k == Key(Settings.default(),"Person","sasuke"))
with self.assertRaises(BadValueError):
print "Checks if Reference accepts other kinds"
book.author = Key(Settings.default(), "Book", "House")
print "Checking Automatic Reference Read"
id = Key(Settings.default(),"Book","Pride")
# id.columns = ["name", "author"]
found = self.db.read(id, FetchMode.Property)
self.assertTrue(found.author.name == "sasuke")
self.assertTrue(found.author == person)
def testBatchPut(self):
'''Tests if puts in batches actually works'''
import time
import uuid
@key("id")
class Profile(Model):
id = String(required = True, indexed = True)
fullname = String(indexed = True)
bookmarks = Map(String, URL)
profile = Profile(id = str(uuid.uuid4()), fullname = "Iroiso Ikpokonte", bookmarks={})
profile.save()
l = []
for i in range(500):
profile = Profile(id = str(i), fullname = "Iroiso Ikpokonte", bookmarks={})
profile.bookmarks["google"] = "http://google.com"
profile.bookmarks["twitter"] = "http://twitter.com"
l.append(profile)
start = time.time()
print ''
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
with Level.All:
self.db.saveMany(Settings.default(),*l)
print "Time Taken to put 500 Profiles: %s secs" % (time.time() - start)
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
cursor = self.connection
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT COUNT(*) FROM Profile;")
count = cursor.fetchone()[0]
print "Count: ", count
self.assertTrue(count == 501)
def optionsFor(namespace):
'''Returns configuration for a namespace or the default'''
found = None
namespaces = Settings.namespaces()
if namespace not in namespaces:
default = Settings.default()
found = Settings.namespaces()[default]
else:
found = Settings.namespaces()[namespace]
return found
def saveMany(clasz, namespace, *models):
'''Write a Lot of Models in one batch, They must all belong to one keyspace'''
from homer.core.models import key, BaseModel
# PERSISTS ALL CHANGES IN *MODELS IN A SINGLE BATCH
def commit(namespace, mutations):
'''Stores all the mutations in one batch operation'''
pool = poolFor(namespace)
with using(pool) as conn:
keyspace = keyspaceFor(namespace)
conn.client.set_keyspace(keyspace)
conn.client.batch_mutate(mutations, clasz.consistency)
# BATCH ALL THE INDIVIDUAL CHANGES IN ONE TRANSFER
mutations = {}
for model in models:
assert issubclass(model.__class__, BaseModel), "parameter model:\
%s must inherit from BaseModel" % model
info = Schema.Get(model)
keyspace = keyspaceFor(info[0])
mnamespace = info[0]
assert namespace == mnamespace, "All the Models must belong to %s for this operation to complete" % namespace
kind = info[1]
if kind not in __COLUMNFAMILIES__ and Settings.debug():
Lisa.create(model)
meta = MetaModel(model)
key = model.key()
key.saved = True
mutations[meta.id()] = meta.mutations()
commit(namespace,mutations)
def testCountWithFilters(self):
'''Show that counts with filters work'''
import time
import uuid
@key("id")
class Profile(Model):
id = String(required = True, indexed = True)
fullname = String(indexed=True)
bookmarks = Map(String, URL)
profile = Profile(id = str(uuid.uuid4()), fullname = "Iroiso Ikpokonte", bookmarks={})
profile.save()
l = []
for i in range(500):
profile = Profile(id = str(i), fullname = "Iroiso", bookmarks={})
profile.bookmarks["google"] = "http://google.com"
profile.bookmarks["twitter"] = "http://twitter.com"
l.append(profile)
start = time.time()
with Level.All:
self.db.saveMany(Settings.default(),*l)
self.assertTrue(Profile.count(fullname="Iroiso") == 500)
def tearDown(self):
'''Release resources that have been allocated'''
try:
self.db.clear()
Schema.Clear()
self.connection.execute("DROP KEYSPACE %s" % Settings.keyspace())
self.connection.close()
except Exception as e:
print e
def makeColumnFamily(self, connection):
'''Creates a new column family from the 'kind' property of this BaseModel'''
try:
connection.client.set_keyspace(self.keyspace)
connection.client.system_add_column_family(self.asColumnFamily())
self.wait(connection)
except InvalidRequestException as e:
if Settings.debug():
print_exc()
def MakeModels(self, *models):
'''Tries to bootstrap all the models that have been passed in'''
from homer.core.models import Model
for model in models:
try:
logging.info("Creating Model: %s" % model)
Lisa.create(model);
except:
if Settings.debug():
print_exc()
def testCountForNonExistentModel(self):
'''Show that counts work for Models that have not been saved'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
query = CqlQuery(Book, "SELECT COUNT(*) FROM Book;")
result = query.fetchone()
self.connection.execute("USE %s" % Settings.keyspace())
self.connection.execute("SELECT COUNT(*) FROM Book;")
correct = self.connection.fetchone()[0]
self.assertTrue(result == correct)
def Put(cls, namespace, model, key):
"""Stores Meta Information for a particular class"""
from homer.options import Settings
if not namespace:
namespace = Settings.default()
kind = model.__name__
if not namespace in cls.schema:
cls.schema[namespace] = WeakValueDictionary()
if kind not in cls.schema[namespace]:
cls.schema[namespace][kind] = model
cls.keys[id(model)] = (namespace, kind, key, )
else:
raise NamespaceCollisionError("Model: %s already \
exists in the Namespace: %s" % (model, namespace))
def testDelete(self):
'''Tests if Lisa.delete() works well'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
book = Book(name = "Pride", author="Anne Rice")
self.db.save(book)
cursor = self.connection
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT name, author FROM Book WHERE KEY=Pride")
#print cursor.description
row = cursor.fetchone()
self.assertTrue(row[0] == "Pride")
k = Key(Settings.default(), 'Book', 'Pride')
self.db.delete(k)
cursor.execute("SELECT name FROM Book WHERE KEY=Pride")
row = cursor.fetchone()
print "Deleted row: %s" % row
self.assertTrue(row[0] == None)
# Make sure that Reads for Lisa return null too.
results = self.db.read(k, FetchMode.Property)
self.assertFalse(results)
def testOtherCommonTypeKeyWork(self):
'''Shows that keys of other common types work'''
@key("id")
class Message(Model):
id = Integer(indexed = True)
message = String(indexed = True)
cursor = self.connection
self.db.save(Message(id=1, message="Something broke damn"))
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT id, message FROM Message WHERE KEY='1'")
self.assertTrue(cursor.rowcount == 1)
row = cursor.fetchone()
print(row)
self.assertTrue(row[0] == '1' and row[1] == "Something broke damn")
def MakeEveryModel(self):
'''Tries to create all the models, registered on Homer'''
from homer.core.models import Schema
namespaces = Schema.schema.keys()
for namespace in namespaces:
kinds = Schema.schema[namespace].keys()
for kind in kinds:
try:
logging.info("Creating Model: %s, %s" % (namespace, kind))
clasz = Schema.ClassForModel(namespace, kind)
instance = clasz()
Lisa.create(instance);
except:
if Settings.debug():
print_exc()
def testPut(self):
'''Tests if Lisa.put() actually stores the model to Cassandra'''
@key("id")
class Profile(Model):
id = String(required = True, indexed = True)
fullname = String(indexed = True)
cursor = self.connection
profile = Profile(id = "1234", fullname = "Iroiso Ikpokonte")
self.db.save(profile)
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT id, fullname FROM Profile WHERE KEY=1234;")
self.assertTrue(cursor.rowcount == 1)
row = cursor.fetchone()
self.assertTrue(row[0] == "1234" and row[1] == "Iroiso Ikpokonte")
assert profile.key().complete() == True # Make sure the object has a complete Key
assert profile.key().saved
def testTTL(self):
'''Tests if put() supports ttl in columns'''
import time
@key("id")
class House(Model):
id = String(required = True, indexed = True)
fullname = String(indexed = True, ttl = 2)
cursor = self.connection
profile = House(id = "1234", fullname = "Iroiso Ikpokonte")
self.db.save(profile)
time.sleep(3) #=> Sleep for 3 secs and see if you can still find it in the datastore
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT fullname FROM House WHERE KEY=1234;")
row = cursor.fetchone()
self.assertTrue(row[0] == None)
def __iter__(self):
'''Execute your queries and converts data to python data models'''
# EXECUTE THE QUERY IF IT HASN'T BEEN EXECUTED
try:
if self.cursor is None:
self.execute()
except Exception as e:
logging.exception("Something wen't wrong when executing the query: %s, error: %s" % self, str(e))
if Settings.debug():
print_exc()
# FOR SOME ODD REASON CASSANDRA 1.0.0 ALWAYS RETURNS CqlResultType.ROWS,
# SO TO FIGURE OUT COUNTS I MANUALLY SEARCH THE QUERY WITH A REGEX
if re.search(self.pattern, self.query):
logging.info("Count expression found;")
yield self.cursor.fetchone()[0]
else:
logging.info("Deciphering rows as usual")
cursor = self.cursor
description = self.cursor.description
if not description:
raise StopIteration
names = [tuple[0] for tuple in description]
row = cursor.fetchone()
while row:
model = self.kind()
descs = fields(model, Property)
values = {}
for name, value in zip(names, row):
if not value: continue
if name == "KEY": continue #Ignore the KEY attribute
prop = descs.get(name, None)
if prop:
found = prop.deconvert(value)
model[name] = found
else:
k, v = model.default
k = k() if isinstance(k, type) else k
v = v() if isinstance(v, type) else v
name = k.deconvert(value)
value = v.deconvert(value)
model[name] = value
yield model
row = cursor.fetchone()
def testCount(self):
'''Shows that count based queries work'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
for i in range(500):
book = Book(name = i, author="Anne Rice")
book.save()
query = CqlQuery(Book, "SELECT COUNT(*) FROM Book;")
result = query.fetchone()
self.connection.execute("USE %s" % Settings.keyspace())
self.connection.execute("SELECT COUNT(*) FROM Book;")
correct = self.connection.fetchone()[0]
print "Results: ", (result, correct)
self.assertTrue(result == correct)
def makeIndexes(self, connection):
'''Creates Indices for all the indexed properties in the model'''
query = 'CREATE INDEX ON {kind}({name});'
for name, property in self.fields.items():
if property.saveable() and property.indexed():
try:
logging.info("Creating index on: %s" % property)
cursor = connection.cursor()
formatted = query.format(kind = self.kind, name= property.name)
cursor.execute("USE %s;" % self.keyspace)
cursor.execute(formatted)
except Exception as e:
logging.exception(e)
if Settings.debug():
print_exc()
else:
logging.info("%s is not indexable" % property)
self.wait(connection)
def testDelete(self):
'''Shows that deletes work as expected'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
book = Book(name = "Pride", author="Anne Rice")
book.save()
cursor = self.connection
cursor.execute("USE %s" % Settings.keyspace())
cursor.execute("SELECT name, author FROM Book WHERE KEY=Pride")
print cursor.description
row = cursor.fetchone()
print "Row Contents: ", row[0]
self.assertTrue(row[0] == "Pride")
Book.delete('Pride')
cursor.execute("SELECT name FROM Book WHERE KEY=Pride")
row = cursor.fetchone()
print "Deleted row: %s" % row
self.assertTrue(row[0] == None)
def testReadMode(self):
'''Tests if FetchMode works in Reads'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
isbn = String(indexed = True)
book = Book(name="Lord of the Rings", author="J.R.R Tolkein", isbn="12345")
for n in xrange(500):
book[str(n)] = n
book.save()
print "Book len:" , len(book)
k = Key(Settings.default(), "Book", "Lord of the Rings")
#k.columns = ["name", "author", "isbn", "titles"] #We'll specify the columns manually for now
b = self.db.read(k, FetchMode.All)
assert isinstance(b, Book)
print "Book len:" , len(b)
assert len(b) == len(book)
assert b == book
def save(clasz, model):
'''Write one Model to Cassandra'''
from homer.core.models import key, BaseModel
# PUT PERSISTS ALL CHANGES IN A MODEL IN A SINGLE BATCH
def commit(namespace, mutations):
'''Stores all the mutations in one batch operation'''
pool = poolFor(namespace)
with using(pool) as conn:
keyspace = keyspaceFor(namespace)
conn.client.set_keyspace(keyspace)
conn.client.batch_mutate(mutations, clasz.consistency)
assert issubclass(model.__class__, BaseModel), "%s must inherit from BaseModel" % model
info = Schema.Get(model)
namespace = info[0]
kind = info[1]
if kind not in __COLUMNFAMILIES__ and Settings.debug():
Lisa.create(model)
meta = MetaModel(model)
changes = { meta.id() : meta.mutations() }
commit(namespace, changes)
key = model.key()
key.saved = True
def testSave(self):
'''Shows that save works'''
@key("id")
class Profile(Model):
id = String(required = True, indexed = True)
fullname = String(indexed = True)
bookmarks = Map(String, URL)
cursor = self.connection
profile = Profile(id = "1234", fullname = "Iroiso Ikpokonte", bookmarks={})
profile.bookmarks["google"] = "http://google.com"
profile.bookmarks["twitter"] = "http://twitter.com"
profile.save() # Save to the datastore
cursor.execute("Use %s" % Settings.keyspace())
cursor.execute("SELECT id, fullname FROM Profile WHERE KEY=1234;")
self.assertTrue(cursor.rowcount == 1)
row = cursor.fetchone()
print "Row Contents: ", row
self.assertTrue(row[0] == u"1234" and row[1] == u"Iroiso Ikpokonte")
assert profile.key().complete() == True # Make sure the object has a complete Key
assert profile.key().saved == True
def execute(self):
'''Executes @self.query in self.keyspace and returns a cursor'''
# FIGURE OUT WHICH KEYSPACE THE MODEL BELONGS TO
if not self.keyspace:
self.namespace = Schema.Get(self.kind)[0] #Every Model is guaranteed to have a namespace at init time.
self.keyspace = keyspaceFor(self.namespace)
logging.info("Executing Query: %s in %s" % (self.query, self.keyspace))
if not self.kind.__name__ in __COLUMNFAMILIES__ and Settings.debug():
logging.info("Creating new Column Family: %s " % self.kind.__name__)
Lisa.create(self.kind())
pool = poolFor(self.namespace)
with using(pool) as conn:
logging.info("Executing %s" % self)
conn.client.set_keyspace(self.keyspace)
cursor = conn.cursor()
keywords = self.keywords
if self.convert:
logging.info("Converting parameters for query: %s" % self.query)
keywords = self.parse(keywords)
cursor.execute(self.query, dict(keywords))
self.cursor = cursor
def testRead(self):
'''Tests if Lisa.read() behaves as usual'''
@key("name")
class Book(Model):
name = String(required = True, indexed = True)
author = String(indexed = True)
isbn = String(indexed = True)
book = Book(name="Lord of the Rings", author="J.R.R Tolkein", isbn="12345")
self.db.save(book)
k = Key(Settings.default(), "Book", "Lord of the Rings")
#k.columns = ["name", "author", "isbn", "titles"] #We'll specify the columns manually for now
b = self.db.read(k, FetchMode.Property)
assert isinstance(b, Book)
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
print b.name
print b.author
print b.isbn
self.assertTrue(b == book)
self.assertTrue(b.name == "Lord of the Rings")
self.assertTrue(b.author == "J.R.R Tolkein")
self.assertTrue(b.isbn == "12345")
print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
def setUp(self):
'''Create the Pool'''
print "Creating a Pool with the default connections"
default = Settings.default()
self.pool = RoundRobinPool(Settings.namespaces().get(default))
def testCreate(self):
'''Tests if Lisa.create() actually creates a Keyspace and ColumnFamily in Cassandra'''
@key("name")
class Person(Model):
name = String("Homer Lisa", indexed = True)
twitter = URL("http://twitter.com/homer", indexed = True)
self.db.create(Person()); #=> Quantum Leap; This was the first time I tested my assumptions on Homer
self.assertRaises(Exception, lambda : self.connection.execute("CREATE KEYSPACE %s" % Settings.keyspace()))
self.assertRaises(Exception, lambda : self.connection.execute("CREATE COLUMNFAMILY Person;"))
self.assertRaises(Exception, lambda : self.connection.execute("CREATE INDEX ON Person(twitter);"))
self.assertRaises(Exception, lambda : self.connection.execute("CREATE INDEX ON Person(name);"))
def setUp(self):
default = Settings.default()
self.pool = RoundRobinPool(Settings.namespaces().get(default))
