def test_null_pool_failover(self):
listener = _TestListener()
pool = NullPool(keyspace='Keyspace1', credentials=_credentials,
listeners=[listener], use_threadlocal=False,
server_list=['localhost:9160', 'localhost:9160'])
conn = pool.get()
cf = ColumnFamily(conn, 'Standard1')
for i in range(1,5):
setattr(cf.client._connection.client, 'batch_mutate', _timeout)
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val'})
assert_equal(listener.failure_count, i)
cf.get('key')
pool.dispose()
listener.reset()
pool = NullPool(keyspace='Keyspace1', credentials=_credentials,
listeners=[listener], use_threadlocal=False,
server_list=['localhost:9160', 'localhost:9160'])
threads = []
args = (pool, 'key', {'col':'val'})
for i in range(0, 5):
threads.append(threading.Thread(target=_five_fails, args=args))
threads[-1].start()
for thread in threads:
thread.join()
assert_equal(listener.failure_count, 25)
pool.dispose()
class ActiveCode:
def __init__(self):
# Connect to Cassandra servers
client = connect(cassandra_hosts)
self.u = ColumnFamily(client, cassandra_keyspace,
'Users', super=True)
self.e = Error()
def _gen_active_code(self, length=6, chars=(letters + digits)):
return ''.join([choice(chars) for _ in xrange(length)])
def _is_exist(self, code):
try:
self.u.get('active_codes', super_column=code)
return True
except NotFoundException:
return False
def get_new_code(self):
new = self._gen_active_code()
while self._is_exist(new) is True:
new = self._gen_active_code()
# store in database
self.u.insert('active_codes', {new: {'create_time': str(datetime.now())}})
return new
def active(self, username, active_code):
if self._is_exist(active_code) is False:
return False
self.u.insert('active_codes', {active_code: {'active_time': str(datetime.now()),
'owner': str(username)}})
return True
def stats(self):
return self.u.get('active_codes')
class TestSuperColumnFamily:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.pool, 'Super2')
def tearDown(self):
for key, columns in self.cf.get_range():
self.cf.remove(key)
def test_super(self):
key = 'TestSuperColumnFamily.test_super'
columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert_equal(self.cf.get(key), columns)
assert_equal(self.cf.multiget([key]), {key: columns})
assert_equal(list(self.cf.get_range(start=key, finish=key)), [(key, columns)])
def test_super_column_argument(self):
key = 'TestSuperColumnFamily.test_super_columns_argument'
sub12 = {'sub1': 'val1', 'sub2': 'val2'}
sub34 = {'sub3': 'val3', 'sub4': 'val4'}
columns = {'1': sub12, '2': sub34}
self.cf.insert(key, columns)
assert_equal(self.cf.get(key, super_column='1'), sub12)
assert_raises(NotFoundException, self.cf.get, key, super_column='3')
assert_equal(self.cf.multiget([key], super_column='1'), {key: sub12})
assert_equal(list(self.cf.get_range(start=key, finish=key, super_column='1')), [(key, sub12)])
def _five_fails(pool, key, column):
conn = pool.get()
cf = ColumnFamily(conn, 'Standard1')
for i in range(0,5):
setattr(cf.client._connection.client, 'batch_mutate', _timeout)
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert(key, column)
cf.get(key)
class TestSuperColumnFamily:
def setUp(self):
self.client = connect_thread_local()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
super=True)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for subcolumns in columns.itervalues():
for value, timestamp in subcolumns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def test_super(self):
key = 'TestSuperColumnFamily.test_super'
columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert self.cf.get(key) == columns
assert self.cf.multiget([key]) == {key: columns}
assert list(self.cf.get_range(start=key, finish=key)) == [(key, columns)]
def test_super_column_argument(self):
key = 'TestSuperColumnFamily.test_super_columns_argument'
sub12 = {'sub1': 'val1', 'sub2': 'val2'}
sub34 = {'sub3': 'val3', 'sub4': 'val4'}
columns = {'1': sub12, '2': sub34}
self.cf.insert(key, columns)
assert self.cf.get(key, super_column='1') == sub12
assert_raises(NotFoundException, self.cf.get, key, super_column='3')
assert self.cf.multiget([key], super_column='1') == {key: sub12}
assert list(self.cf.get_range(start=key, finish=key, super_column='1')) == [(key, sub12)]
class TestSuperColumnFamily:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
super=True)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for subcolumns in columns.itervalues():
for value, timestamp in subcolumns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def test_super(self):
key = 'TestSuperColumnFamily.test_super'
columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert self.cf.get(key) == columns
assert self.cf.multiget([key]) == {key: columns}
assert list(self.cf.get_range(start=key, finish=key)) == [(key, columns)]
def test_super_column_argument(self):
key = 'TestSuperColumnFamily.test_super_columns_argument'
sub12 = {'sub1': 'val1', 'sub2': 'val2'}
sub34 = {'sub3': 'val3', 'sub4': 'val4'}
columns = {'1': sub12, '2': sub34}
self.cf.insert(key, columns)
assert self.cf.get(key, super_column='1') == sub12
assert_raises(NotFoundException, self.cf.get, key, super_column='3')
assert self.cf.multiget([key], super_column='1') == {key: sub12}
assert list(self.cf.get_range(start=key, finish=key, super_column='1')) == [(key, sub12)]
class TestDefaultValidators(unittest.TestCase):
def setUp(self):
credentials = {"username": "******", "password": "******"}
self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials)
self.cf_def_valid = ColumnFamily(self.pool, "DefaultValidator")
def tearDown(self):
for key, cols in self.cf_def_valid.get_range():
self.cf_def_valid.remove(key)
self.pool.dispose()
def test_default_validated_columns(self):
key = "key1"
col_cf = {"aaaaaa": 1L}
col_cm = {"subcol": TIME1}
col_ncf = {"aaaaaa": TIME1}
col_ncm = {"subcol": 1L}
# Both of these inserts work, as cf allows
# longs and cm for 'subcol' allows TIMEUUIDs.
self.cf_def_valid.insert(key, col_cf)
self.cf_def_valid.insert(key, col_cm)
assert self.cf_def_valid.get(key) == {"aaaaaa": 1L, "subcol": TIME1}
assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncf)
assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncm)
class Database:
def __init__(self, column_family):
"""
ColumnFamily:
- Thông tin người dùng
- Tìm kiếm
- Thông tin vé
- Nhật ký hệ thống
- Thông tin nhà cung cấp
"""
# Connect to Cassandra servers
client = connect(CASSANDRA_HOSTS)
self.db = ColumnFamily(client, CASSANDRA_KEYSPACE, column_family, super=False)
def insert(self, key, columns):
key = md5(capwords(key).lower()).hexdigest()
return self.db.insert(key, columns)
def get(self, key, columns=None):
key = md5(capwords(key).lower()).hexdigest()
return self.db.get(key=key, columns=columns)
def remove(self, key, column=None):
key = md5(capwords(key).lower()).hexdigest()
return self.db.remove(key=key, column=column)
def test_queue_failover(self):
for prefill in (True, False):
listener = _TestListener()
pool = ConnectionPool(
pool_size=1,
max_overflow=0,
recycle=10000,
prefill=prefill,
timeout=1,
keyspace='PycassaTestKeyspace',
credentials=_credentials,
listeners=[listener],
use_threadlocal=False,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(1, 5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'})
assert_equal(listener.failure_count, i)
assert_equal(cf.get('key'), {
'col': 'val%d' % i,
'col2': 'val'
})
pool.dispose()
def test_queue_threadlocal_failover(self):
stats_logger = StatsLoggerWithListStorage()
pool = ConnectionPool(pool_size=1,
max_overflow=0,
recycle=10000,
prefill=True,
timeout=0.05,
keyspace='PycassaTestKeyspace',
credentials=_credentials,
listeners=[stats_logger],
use_threadlocal=True,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(1, 5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'})
assert_equal(stats_logger.stats['failed'], i)
assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'})
pool.dispose()
stats_logger.reset()
pool = ConnectionPool(pool_size=5,
max_overflow=5,
recycle=10000,
prefill=True,
timeout=0.05,
keyspace='PycassaTestKeyspace',
credentials=_credentials,
listeners=[stats_logger],
use_threadlocal=True,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
threads = []
args = ('key', {'col': 'val', 'col2': 'val'})
for i in range(5):
threads.append(threading.Thread(target=cf.insert, args=args))
threads[-1].start()
for thread in threads:
thread.join()
assert_equal(stats_logger.stats['failed'], 5)
pool.dispose()
class Dictionary:
"""
Nhóm chức năng từ điển:
* Tra từ Anh-Việt
* Tra từ Việt-Anh
"""
def __init__(self):
# Connect to Cassandra servers
client = connect(cassandra_hosts)
self.d = ColumnFamily(client, cassandra_keyspace, 'Dictionary', super=True)
self.u = ColumnFamily(client, cassandra_keyspace, 'Users', super=True)
self.e = Error()
def _lookup(self, keyword, dict_type='en_vi'):
try:
return self.d.get(dict_type, super_column=str(keyword))
except (NotFoundException, InvalidRequestException):
return None
def lookup(self, environ):
try:
session_id = environ['request']['session_id']
except KeyError:
return self.e.authen_error("Thiếu session_id")
try:
self.u.get('session_id', super_column=session_id)
except (NotFoundException, InvalidRequestException):
return self.e.authen_error()
result = self._lookup(environ['request']['keyword'])
result2 = self._lookup(environ['request']['keyword'], 'vi_en')
result3 = self._lookup(environ['request']['keyword'], 'en_en')
if (result is None) and (result2 is None) and (result3 is None):
return self.e.not_found("Từ khóa bạn tìm không có trong từ điển")
xml = []
if result is not None:
xml.append('<result type="en_vi" keyword="%s" mean="%s" spell="%s" status_code="200"/>' \
% (xml_format(environ['request']['keyword']),
xml_format(result['nghia']),
xml_format(result['phien_am_quoc_te'])))
if result2 is not None:
xml.append('<result type="vi_en" keyword="%s" mean="%s" spell="" status_code="200"/>' \
% (xml_format(environ['request']['keyword']),
xml_format(result2['nghia'])))
return '\n\n'.join(xml)
def total_words(self, dict_type='en_vi'):
return self.d.get_count(dict_type)
class TestValidators(unittest.TestCase):
def setUp(self):
credentials = {"username": "******", "password": "******"}
self.pool = ConnectionPool(pool_size=10, keyspace="Keyspace1", credentials=credentials)
self.cf_valid_long = ColumnFamily(self.pool, "ValidatorLong")
self.cf_valid_int = ColumnFamily(self.pool, "ValidatorInt")
self.cf_valid_time = ColumnFamily(self.pool, "ValidatorTime")
self.cf_valid_lex = ColumnFamily(self.pool, "ValidatorLex")
self.cf_valid_ascii = ColumnFamily(self.pool, "ValidatorAscii")
self.cf_valid_utf8 = ColumnFamily(self.pool, "ValidatorUTF8")
self.cf_valid_bytes = ColumnFamily(self.pool, "ValidatorBytes")
self.cfs = [
self.cf_valid_long,
self.cf_valid_int,
self.cf_valid_time,
self.cf_valid_lex,
self.cf_valid_ascii,
self.cf_valid_utf8,
self.cf_valid_bytes,
]
def tearDown(self):
for cf in self.cfs:
for key, cols in cf.get_range():
cf.remove(key)
self.pool.dispose()
def test_validated_columns(self):
key = "key1"
col = {"subcol": 1L}
self.cf_valid_long.insert(key, col)
assert self.cf_valid_long.get(key) == col
col = {"subcol": 1}
self.cf_valid_int.insert(key, col)
assert self.cf_valid_int.get(key) == col
col = {"subcol": TIME1}
self.cf_valid_time.insert(key, col)
assert self.cf_valid_time.get(key) == col
col = {"subcol": uuid.UUID(bytes="aaa aaa aaa aaaa")}
self.cf_valid_lex.insert(key, col)
assert self.cf_valid_lex.get(key) == col
col = {"subcol": "aaa"}
self.cf_valid_ascii.insert(key, col)
assert self.cf_valid_ascii.get(key) == col
col = {"subcol": u"a\u0020"}
self.cf_valid_utf8.insert(key, col)
assert self.cf_valid_utf8.get(key) == col
col = {"subcol": "aaa"}
self.cf_valid_bytes.insert(key, col)
assert self.cf_valid_bytes.get(key) == col
def test_assertion_threadlocal_failover(self):
listener = _TestListener()
pool = AssertionPool(keyspace='Keyspace1', credentials=_credentials,
listeners=[listener], use_threadlocal=False,
server_list=['localhost:9160', 'localhost:9160'])
conn = pool.get()
cf = ColumnFamily(conn, 'Standard1')
for i in range(1,5):
setattr(cf.client._connection.client, 'batch_mutate', _timeout)
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val'})
assert_equal(listener.failure_count, i)
cf.get('key')
pool.dispose()
class TestSuperColumnFamilyMap:
def setUp(self):
self.client = connect_thread_local()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp,
super=True)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for subcolumns in columns.itervalues():
for value, timestamp in subcolumns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def instance(self, key, super_column):
instance = TestUTF8()
instance.key = key
instance.super_column = super_column
instance.strcol = '1'
instance.intcol = 2
instance.floatcol = 3.5
instance.datetimecol = datetime.now().replace(microsecond=0)
instance.intstrcol = 8
instance.floatstrcol = 4.6
instance.datetimestrcol = datetime.now().replace(microsecond=0)
return instance
def test_super(self):
instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1')
assert_raises(NotFoundException, self.map.get, instance.key)
self.map.insert(instance)
assert self.map.get(instance.key)[instance.super_column] == instance
assert self.map.multiget([instance.key])[instance.key][instance.super_column] == instance
assert list(self.map.get_range(start=instance.key, finish=instance.key)) == [{instance.super_column: instance}]
def get(self, colfam, key, columns=None, column_start="", column_finish="",
column_reversed=False, column_count=100, include_timestamp=False,
super_column=None, read_consistency_level=None):
cf = ColumnFamily(self.db, colfam)
try:
return cf.get(key, columns, column_start, column_finish,
column_reversed, column_count, include_timestamp,
super_column, read_consistency_level)
except NotFoundException:
return None
class TestSuperColumnFamilyMap:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp,
super=True)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for subcolumns in columns.itervalues():
for value, timestamp in subcolumns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def instance(self, key, super_column):
instance = TestUTF8()
instance.key = key
instance.super_column = super_column
instance.strcol = '1'
instance.intcol = 2
instance.floatcol = 3.5
instance.datetimecol = datetime.now().replace(microsecond=0)
instance.intstrcol = 8
instance.floatstrcol = 4.6
instance.datetimestrcol = datetime.now().replace(microsecond=0)
return instance
def test_super(self):
instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1')
assert_raises(NotFoundException, self.map.get, instance.key)
self.map.insert(instance)
assert self.map.get(instance.key)[instance.super_column] == instance
assert self.map.multiget([instance.key])[instance.key][instance.super_column] == instance
assert list(self.map.get_range(start=instance.key, finish=instance.key)) == [{instance.super_column: instance}]
def test_queue_threadlocal_failover(self):
listener = _TestListener()
pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000,
prefill=True, timeout=0.05,
keyspace='PycassaTestKeyspace', credentials=_credentials,
listeners=[listener], use_threadlocal=True,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(1,5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'})
assert_equal(listener.failure_count, i)
assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'})
pool.dispose()
listener.reset()
pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=10000,
prefill=True, timeout=0.05,
keyspace='PycassaTestKeyspace', credentials=_credentials,
listeners=[listener], use_threadlocal=True,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
threads = []
args=('key', {'col': 'val', 'col2': 'val'})
for i in range(5):
threads.append(threading.Thread(target=cf.insert, args=args))
threads[-1].start()
for thread in threads:
thread.join()
assert_equal(listener.failure_count, 5)
pool.dispose()
def test_validated_columns(self):
sys = SystemManager()
sys.create_column_family(
TEST_KS,
'Validators',
)
sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE)
sys.close()
cf = ColumnFamily(pool, 'Validators')
key = 'key1'
col = {'long': 1L}
cf.insert(key, col)
assert_equal(cf.get(key)['long'], 1L)
col = {'int': 1}
cf.insert(key, col)
assert_equal(cf.get(key)['int'], 1)
col = {'time': TIME1}
cf.insert(key, col)
assert_equal(cf.get(key)['time'], TIME1)
col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')}
cf.insert(key, col)
assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa'))
col = {'ascii': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['ascii'], 'aaa')
col = {'utf8': u'a\u0020'}
cf.insert(key, col)
assert_equal(cf.get(key)['utf8'], u'a\u0020')
col = {'bytes': 'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['bytes'], 'aaa')
cf.remove(key)
def test_default_validated_columns(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'DefaultValidator', default_validation_class=LONG_TYPE)
sys.alter_column(TEST_KS, 'DefaultValidator', 'subcol', TIME_UUID_TYPE)
sys.close()
cf = ColumnFamily(pool, 'DefaultValidator')
key = 'key1'
col_cf = {'aaaaaa': 1L}
col_cm = {'subcol': TIME1}
col_ncf = {'aaaaaa': TIME1}
col_ncm = {'subcol': 1L}
# Both of these inserts work, as cf allows
# longs and cm for 'subcol' allows TIMEUUIDs.
cf.insert(key, col_cf)
cf.insert(key, col_cm)
assert_equal(cf.get(key), {'aaaaaa': 1L, 'subcol': TIME1})
def test_validated_columns(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'Validators',)
sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE)
sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE)
sys.close()
cf = ColumnFamily(pool, 'Validators')
key = 'key1'
col = {'long':1L}
cf.insert(key, col)
assert_equal(cf.get(key)['long'], 1L)
col = {'int':1}
cf.insert(key, col)
assert_equal(cf.get(key)['int'], 1)
col = {'time':TIME1}
cf.insert(key, col)
assert_equal(cf.get(key)['time'], TIME1)
col = {'lex':uuid.UUID(bytes='aaa aaa aaa aaaa')}
cf.insert(key, col)
assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa'))
col = {'ascii':'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['ascii'], 'aaa')
col = {'utf8':u'a\u0020'}
cf.insert(key, col)
assert_equal(cf.get(key)['utf8'], u'a\u0020')
col = {'bytes':'aaa'}
cf.insert(key, col)
assert_equal(cf.get(key)['bytes'], 'aaa')
cf.remove(key)
def get_variable_data(id_variable, columns):
# Load Postgres session
session = sqlm.load_session()
# get variable by id
variable = session.query(sqlm.Variable).\
filter(sqlm.Variable.id == id_variable).\
filter(sqlm.Variable.deletion_date == None).\
first()
if not variable:
session.close()
return {'error': json_error_not_exists(sqlm.Variable)}
# build query cassandra, structured by tables
query_cassandra = {} # {table name: list of cassandra ids}
# query cassandra
pool = load_pool()
pool.timeout = 300
cf = ColumnFamily(pool, variable.timeseries_cassandra)
try:
ans_cassandra = cf.get(variable.id_cassandra, **columns)
except NotFoundException:
session.close()
pool.dispose()
return {'error': json.dumps({'error': 'No data found'})}
output = rearrange_timeseries(ans_cassandra)
session.close()
pool.dispose()
# the output is returned as a list of one list
return [output]
def test_queue_failover(self):
for prefill in (True, False):
listener = _TestListener()
pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000,
prefill=prefill, timeout=1,
keyspace='PycassaTestKeyspace', credentials=_credentials,
listeners=[listener], use_threadlocal=False,
server_list=['localhost:9160', 'localhost:9160'])
cf = ColumnFamily(pool, 'Standard1')
for i in range(1,5):
conn = pool.get()
setattr(conn, 'send_batch_mutate', conn._fail_once)
conn._should_fail = True
conn.return_to_pool()
# The first insert attempt should fail, but failover should occur
# and the insert should succeed
cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'})
assert_equal(listener.failure_count, i)
assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'})
pool.dispose()
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.pools = []
self.clients = []
self.cfs = []
for pool_cls in _pool_classes:
pool = pool_cls(keyspace='Keyspace1', credentials=credentials)
self.pools.append(pool)
client = pool.get()
self.clients.append(client)
cf = ColumnFamily(client, 'Standard%s' % pool_cls.__name__,
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
dict_class=TestDict)
self.cfs.append(cf)
self.timestamp_n = 0
for cf in self.cfs:
try:
self.timestamp_n = max(self.timestamp_n,
int(cf.get('meta')['timestamp']))
except NotFoundException:
pass
self.clear()
class TestMutator(unittest.TestCase):
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client,
'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp)
self.scf = ColumnFamily(self.client,
'Super1',
write_consistency_level=ConsistencyLevel.ONE,
super=True,
timestamp=self.timestamp)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def clear(self):
self.cf.truncate()
self.scf.truncate()
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def test_insert(self):
batch = self.cf.batch()
for key, cols in ROWS.iteritems():
batch.insert(key, cols)
batch.send()
for key, cols in ROWS.items():
assert self.cf.get(key) == cols
def test_insert_supercolumns(self):
batch = self.scf.batch()
batch.insert('one', ROWS)
batch.insert('two', ROWS)
batch.insert('three', ROWS)
batch.send()
assert self.scf.get('one') == ROWS
assert self.scf.get('two') == ROWS
assert self.scf.get('three') == ROWS
def test_queue_size(self):
batch = self.cf.batch(queue_size=2)
batch.insert('1', ROWS['1'])
batch.insert('2', ROWS['2'])
batch.insert('3', ROWS['3'])
assert self.cf.get('1') == ROWS['1']
assert_raises(NotFoundException, self.cf.get, '3')
batch.send()
for key, cols in ROWS.items():
assert self.cf.get(key) == cols
def test_remove_key(self):
batch = self.cf.batch()
batch.insert('1', ROWS['1'])
batch.remove('1')
batch.send()
assert_raises(NotFoundException, self.cf.get, '1')
def test_remove_columns(self):
batch = self.cf.batch()
batch.insert('1', {'a': '123', 'b': '123'})
batch.remove('1', ['a'])
batch.send()
assert self.cf.get('1') == {'b': '123'}
def test_remove_supercolumns(self):
batch = self.scf.batch()
batch.insert('one', ROWS)
batch.insert('two', ROWS)
batch.insert('three', ROWS)
batch.remove('two', ['b'], '2')
batch.send()
assert self.scf.get('one') == ROWS
assert self.scf.get('two')['2'] == {'a': '234'}
assert self.scf.get('three') == ROWS
def test_chained(self):
batch = self.cf.batch()
batch.insert('1',
ROWS['1']).insert('2',
ROWS['2']).insert('3',
ROWS['3']).send()
assert self.cf.get('1') == ROWS['1']
assert self.cf.get('2') == ROWS['2']
assert self.cf.get('3') == ROWS['3']
def test_contextmgr(self):
if sys.version_info < (2, 5):
raise SkipTest("No context managers in Python < 2.5")
exec """with self.cf.batch(queue_size=2) as b:
b.insert('1', ROWS['1'])
b.insert('2', ROWS['2'])
b.insert('3', ROWS['3'])
assert self.cf.get('3') == ROWS['3']"""
def test_multi_column_family(self):
batch = self.client.batch()
cf2 = self.cf
batch.insert(self.cf, '1', ROWS['1'])
batch.insert(self.cf, '2', ROWS['2'])
batch.remove(cf2, '1', ROWS['1'])
batch.send()
assert self.cf.get('2') == ROWS['2']
assert_raises(NotFoundException, self.cf.get, '1')
class TestColumnFamily:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
dict_class=TestDict)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def test_empty(self):
key = 'TestColumnFamily.test_empty'
assert_raises(NotFoundException, self.cf.get, key)
assert len(self.cf.multiget([key])) == 0
for key, columns in self.cf.get_range():
assert len(columns) == 0
def test_insert_get(self):
key = 'TestColumnFamily.test_insert_get'
columns = {'1': 'val1', '2': 'val2'}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert self.cf.get(key) == columns
def test_insert_multiget(self):
key1 = 'TestColumnFamily.test_insert_multiget1'
columns1 = {'1': 'val1', '2': 'val2'}
key2 = 'test_insert_multiget1'
columns2 = {'3': 'val1', '4': 'val2'}
missing_key = 'key3'
self.cf.insert(key1, columns1)
self.cf.insert(key2, columns2)
rows = self.cf.multiget([key1, key2, missing_key])
assert len(rows) == 2
assert rows[key1] == columns1
assert rows[key2] == columns2
assert missing_key not in rows
def test_insert_get_count(self):
key = 'TestColumnFamily.test_insert_get_count'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
assert self.cf.get_count(key) == 2
assert_equal(self.cf.get_count(key, column_start='1'), 2)
assert_equal(self.cf.get_count(key, column_finish='2'), 2)
assert_equal(self.cf.get_count(key, column_start='1', column_finish='2'), 2)
assert_equal(self.cf.get_count(key, column_start='1', column_finish='1'), 1)
assert_equal(self.cf.get_count(key, columns=['1','2']), 2)
assert_equal(self.cf.get_count(key, columns=['1']), 1)
def test_insert_multiget_count(self):
keys = ['TestColumnFamily.test_insert_multiget_count1',
'TestColumnFamily.test_insert_multiget_count2',
'TestColumnFamily.test_insert_multiget_count3']
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
result = self.cf.multiget_count(keys)
assert_equal(result[keys[0]], 2)
assert_equal(result[keys[1]], 2)
assert_equal(result[keys[2]], 2)
result = self.cf.multiget_count(keys, column_start='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_start='1', column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_start='1', column_finish='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
result = self.cf.multiget_count(keys, columns=['1','2'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, columns=['1'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
def test_insert_get_range(self):
keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)]
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
assert len(rows) == len(keys)
for i, (k, c) in enumerate(rows):
assert k == keys[i]
assert c == columns
def test_get_range_batching(self):
self.cf.truncate()
keys = []
columns = {'c': 'v'}
for i in range(100, 201):
keys.append('key%d' % i)
self.cf.insert('key%d' % i, columns)
for i in range(201, 301):
self.cf.insert('key%d' % i, columns)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=10):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=1000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=150):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=7):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
# Put the remaining keys in our list
for i in range(201, 301):
keys.append('key%d' % i)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=7):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=200):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=10000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
# Don't give a row count
count = 0
for (k,v) in self.cf.get_range(buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=77):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=200):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=10000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
self.cf.truncate()
def insert_insert_get_indexed_slices(self):
indexed_cf = ColumnFamily(self.client, 'Indexed1')
columns = {'birthdate': 1L}
keys = []
for i in range(1,4):
indexed_cf.insert('key%d' % i, columns)
keys.append('key%d')
expr = index.create_index_expression(column_name='birthdate', value=1L)
clause = index.create_index_clause([expr])
count = 0
for key,cols in indexed_cf.get_indexed_slices(clause):
assert cols == columns
assert key in keys
count += 1
assert_equal(count, 3)
def test_get_indexed_slices_batching(self):
indexed_cf = ColumnFamily(self.client, 'Indexed1')
columns = {'birthdate': 1L}
for i in range(200):
indexed_cf.insert('key%d' % i, columns)
expr = index.create_index_expression(column_name='birthdate', value=1L)
clause = index.create_index_clause([expr], count=10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000))
assert_equal(len(result), 10)
clause = index.create_index_clause([expr], count=250)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000))
assert_equal(len(result), 200)
def test_remove(self):
key = 'TestColumnFamily.test_remove'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
self.cf.remove(key, columns=['2'])
del columns['2']
assert self.cf.get(key) == {'1': 'val1'}
self.cf.remove(key)
assert_raises(NotFoundException, self.cf.get, key)
def test_dict_class(self):
key = 'TestColumnFamily.test_dict_class'
self.cf.insert(key, {'1': 'val1'})
assert isinstance(self.cf.get(key), TestDict)
class Authentication:
"""
Nhóm chức năng quản lý người dùng
Các hành động hỗ trợ:
* đăng ký
* đăng nhập
* đăng xuất
"""
def __init__(self):
# Connect to Cassandra servers
client = connect(cassandra_hosts)
self.a = ColumnFamily(client, cassandra_keyspace, "Users", super=True)
self.e = Error()
def _check(self, username):
try:
username = username.strip().lower()
self.a.get(username)
return 1
except (NotFoundException, InvalidRequestException):
return 0
def _remove(self, username):
try:
username = username.strip().lower()
self.a.remove(username)
return 1
except (NotFoundException, InvalidRequestException):
return 0
def _register(self, username, password, phone_number):
try:
username = username.strip().lower()
if self._check(username) == 1:
return False
salt = str(random())
password = sha512(password + salt).hexdigest()
self.a.insert(
username,
{
"info": {
"password": password,
"salt": salt,
"phone_number": str(phone_number),
"join_time": str(datetime.now()),
}
},
)
return True
except (NotFoundException, InvalidRequestException):
return False
def _login(self, username, password):
try:
username = username.strip().lower()
store_data = self.a.get(username, super_column="info")
store_passwd = store_data["password"]
store_salt = store_data["salt"]
store_join_time = datetime.strptime(store_data["join_time"], "%Y-%m-%d %H:%M:%S.%f")
password = sha512(password + store_salt).hexdigest()
if password == store_passwd:
session_id = md5(username + str(datetime.now())).hexdigest()
# --- check trial expire ---
delta = datetime.now() - store_join_time
if delta.days > trial_time:
return 402
# --- end check ---
self.a.insert("session_id", {session_id: {"username": username, "timestamp": str(datetime.now())}})
return session_id
return False
except (NotFoundException, InvalidRequestException):
return False
def _logout(self, session_id):
try:
self.a.get("session_id", super_column=session_id) # check if exist
self.a.remove("session_id", session_id)
return True
except (NotFoundException, InvalidRequestException):
return False
def register(self, environ):
try:
username = environ["request"]["username"]
password = environ["request"]["password"]
phone_number = environ["request"]["phone_number"]
except KeyError:
return self.e.bad_request("Hành động này yêu cầu 4 tham số: username, password, phone_number và api_key")
username = username.strip().lower()
# check params
allow_chars = lowercase + ".-_@"
for char in username:
if char not in allow_chars:
self.e.param_error("Tên đăng nhập chỉ được dùng các ký tự a-z, A-Z và .-_@")
if len(password) != 32:
return self.e.param_error("Mật khẩu phải được mã hóa md5 trước khi gửi lên")
# ----- check phone_number (dạng 841673450799) -----
if phone_number != "":
phone_number = phone_number.replace(" ", "")
phone_number = "\t" + phone_number
if phone_number.startswith("\t+"):
phone_number = phone_number.replace("\t+", "\t") # remove +
# normalize
if phone_number.startswith("\t09"):
phone_number = phone_number.replace("\t09", "849")
elif phone_number.startswith("\t01"):
phone_number = phone_number.replace("\t01", "841")
else:
return self.e.param_error("phone_number không hợp lệ")
if len(phone_number) in (11, 12):
try:
phone_number = int(phone_number)
except ValueError: # not int
return self.e.param_error("phone_number không hợp lệ")
else:
return self.e.param_error("phone_number không hợp lệ")
# ----- end phone_number check -----
result = self._register(username, password, phone_number)
if result is True:
xml = '<register status_code="201" description="Tài khoản đã được tạo thành công"/>'
return xml
else:
xml = '<register status_code="406" description="Tham số không hợp lệ"/>'
return xml
def login(self, environ):
try:
username = environ["request"]["username"]
password = environ["request"]["password"]
except KeyError:
return self.e.bad_request("Hành động này yêu cầu 3 tham số: username, password và api_key")
result = self._login(username, password)
if result is False:
xml = '<login status_code="401" description="Tên đăng nhập/mật khẩu không hợp lệ"/>'
return xml
elif result == 402:
return self.e.trial_expired()
else:
xml = '<login status_code="202" session_id="%s" description="Đăng nhập thành công"/>' % (result)
return xml
def logout(self, environ):
try:
session_id = environ["request"]["session_id"]
except KeyError:
return self.e.bad_request("Hành động này yêu cầu 2 tham số: session_id và api_key")
result = self._logout(session_id)
if result is True:
xml = '<logout status_code="200" description="Đăng xuất thành công"/>'
return xml
else:
xml = '<logout status_code="401" description="session_id không hợp lệ"/>'
return xml
class TestTimeUUIDs(unittest.TestCase):
def setUp(self):
credentials = {"username": "******", "password": "******"}
self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials)
self.cf_time = ColumnFamily(self.pool, "StdTimeUUID")
def tearDown(self):
for key, cols in self.cf_time.get_range():
self.cf_time.remove(key)
self.pool.dispose()
def test_datetime_to_uuid(self):
key = "key1"
timeline = []
timeline.append(datetime.now())
time1 = uuid1()
col1 = {time1: "0"}
self.cf_time.insert(key, col1)
time.sleep(1)
timeline.append(datetime.now())
time2 = uuid1()
col2 = {time2: "1"}
self.cf_time.insert(key, col2)
time.sleep(1)
timeline.append(datetime.now())
cols = {time1: "0", time2: "1"}
assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1)
assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2)
def test_time_to_uuid(self):
key = "key1"
timeline = []
timeline.append(time.time())
time1 = uuid1()
col1 = {time1: "0"}
self.cf_time.insert(key, col1)
time.sleep(0.1)
timeline.append(time.time())
time2 = uuid1()
col2 = {time2: "1"}
self.cf_time.insert(key, col2)
time.sleep(0.1)
timeline.append(time.time())
cols = {time1: "0", time2: "1"}
assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1)
assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2)
def test_auto_time_to_uuid1(self):
key = "key"
t = time.time()
col = {t: "foo"}
self.cf_time.insert(key, col)
uuid_res = self.cf_time.get(key).keys()[0]
timestamp = convert_uuid_to_time(uuid_res)
assert_almost_equal(timestamp, t, places=3)
class CassandraImporter:
def __init__(self):
parser = argparse.ArgumentParser(description="Process some integers.")
parser.add_argument(
"-s",
"--source",
help="Generally the prod cassandra path, list of machines: \
localhost:9162 localhost:9163",
nargs="*",
required=True,
)
parser.add_argument(
"-d",
"--destination",
help="Cassandra path where you need your data: \
localhost:9160 localhost:9161",
nargs="*",
required=True,
)
parser.add_argument("-ks", "--keyspace", help="The keyspace: myks", required=True)
parser.add_argument("-cf", "--column_family", help="The Column family: mycf", required=True)
parser.add_argument("-k", "--key", help="A specific key to be imported", required=False)
parser.add_argument("-c", "--count", help="Total count of keys to be imported", required=False)
parser.add_argument("-a", "--all", action="store_true", help="Get all. Not recommended!", required=False)
args = vars(parser.parse_args())
"""Connection setting with cassandra
The script is meant to sync data. So source and destination KS
and CF shold be the same."""
try:
source_pool = ConnectionPool(args["keyspace"], args["source"])
destination_pool = ConnectionPool(args["keyspace"], args["destination"])
self.source_cf = ColumnFamily(source_pool, args["column_family"])
self.source_cf.autopack_names = False
self.source_cf.autopack_values = False
self.source_cf.autopack_keys = False
self.source_cf.default_validation_class = pycassa.types.UTF8Type()
self.destination_cf = ColumnFamily(destination_pool, args["column_family"])
self.destination_cf.autopack_names = False
self.destination_cf.autopack_values = False
self.destination_cf.autopack_keys = False
self.destination_cf.default_validation_class = pycassa.types.UTF8Type()
except Exception as e:
print "ERROR: The keyspace or the column family does not exist or request is timing out!"
sys.exit()
# Optional data
self.count = args["count"]
if self.count:
self.count = int(self.count)
self.key = args["key"]
self.all = args["all"]
def importData(self):
data = dict()
# Get columns for a key
if self.key:
column_data = self.source_cf.get(self.key)
data[self.key] = column_data
# Get last x keys and their columns
elif self.count:
counter = 0
error_count = 0
for value in self.source_cf.get_range(column_count=0, filter_empty=False):
if counter < self.count:
try:
column_data = self.source_cf.get(value[0], column_count=100)
data[value[0]] = column_data
counter += 1
except NotFoundException:
# Ignore keys with empty columns
pass
except Exception:
error_count += 1
if error_count > 10:
# Write the read data
self.insertData(data)
print "ERROR: Remote cassandra is too slow to read, exiting after writing..."
sys.exit()
# Use this to throttle reads from cassandra
time.sleep(0.2)
else:
break
# Get All, Not recommended
elif self.all:
for value in self.source_cf.get_range(column_count=0, filter_empty=False):
column_data = self.source_cf.get(value[0])
key = value[0]
data[key] = column_data
else:
print "Please pass -c or -k or -a arguments!"
return data
def insertData(self, data):
print "Writing " + str(len(data.keys())) + " keys"
for key, value in data.iteritems():
self.destination_cf.insert(key, value)
def run(self):
self.update_progress(0)
data = self.importData()
self.update_progress(50)
self.insertData(data)
self.update_progress(100)
print "Import complete!"
def update_progress(self, progress):
print "\r[{0}] {1}%".format("#" * (progress / 10), progress)
class TestAutoPacking:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Standard2')
self.cf_long = ColumnFamily(self.client, 'StdLong')
self.cf_int = ColumnFamily(self.client, 'StdInteger')
self.cf_time = ColumnFamily(self.client, 'StdTimeUUID')
self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID')
self.cf_ascii = ColumnFamily(self.client, 'StdAscii')
self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8')
self.cf_bytes = ColumnFamily(self.client, 'StdBytes')
self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True)
self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True)
self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True)
self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True)
self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True)
self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True)
self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True)
self.cf_suplong_sublong = ColumnFamily(self.client, 'SuperLongSubLong', super=True)
self.cf_suplong_subint = ColumnFamily(self.client, 'SuperLongSubInt', super=True)
self.cf_suplong_subtime = ColumnFamily(self.client, 'SuperLongSubTime', super=True)
self.cf_suplong_sublex = ColumnFamily(self.client, 'SuperLongSubLex', super=True)
self.cf_suplong_subascii = ColumnFamily(self.client, 'SuperLongSubAscii', super=True)
self.cf_suplong_subutf8 = ColumnFamily(self.client, 'SuperLongSubUTF8', super=True)
self.cf_suplong_subbytes = ColumnFamily(self.client, 'SuperLongSubBytes', super=True)
self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong')
self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt')
self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime')
self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex')
self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii')
self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8')
self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes')
self.cfs = [self.cf_long, self.cf_int, self.cf_time, self.cf_lex,
self.cf_ascii, self.cf_utf8, self.cf_bytes,
self.cf_suplong, self.cf_supint, self.cf_suptime,
self.cf_suplex, self.cf_supascii, self.cf_suputf8,
self.cf_supbytes,
self.cf_suplong_subint, self.cf_suplong_subint,
self.cf_suplong_subtime, self.cf_suplong_sublex,
self.cf_suplong_subascii, self.cf_suplong_subutf8,
self.cf_suplong_subbytes,
self.cf_valid_long, self.cf_valid_int, self.cf_valid_time,
self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8,
self.cf_valid_bytes]
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
for cf in self.cfs:
for key, columns in cf.get_range():
cf.remove(key)
def test_basic_inserts(self):
long_col = {1111111111111111L: VALS[0]}
int_col = {1: VALS[0]}
time_col = {TIME1: VALS[0]}
lex_col = {uuid.UUID(bytes='abc abc abc abcd'): VALS[0]}
ascii_col = {'foo': VALS[0]}
utf8_col = {u'\u0020': VALS[0]}
bytes_col = {'bytes': VALS[0]}
self.cf_long.insert(KEYS[0], long_col)
self.cf_int.insert(KEYS[0], int_col)
self.cf_time.insert(KEYS[0], time_col)
self.cf_lex.insert(KEYS[0], lex_col)
self.cf_ascii.insert(KEYS[0], ascii_col)
self.cf_utf8.insert(KEYS[0], utf8_col)
self.cf_bytes.insert(KEYS[0], bytes_col)
assert self.cf_long.get(KEYS[0]) == long_col
assert self.cf_int.get(KEYS[0]) == int_col
assert self.cf_time.get(KEYS[0]) == time_col
assert self.cf_lex.get(KEYS[0]) == lex_col
assert self.cf_ascii.get(KEYS[0]) == ascii_col
assert self.cf_utf8.get(KEYS[0]) == utf8_col
assert self.cf_bytes.get(KEYS[0]) == bytes_col
self.cf_suplong.insert(KEYS[0], {123L: bytes_col})
class Course:
def __init__(self):
self.cache = KeyValueDatabase()
client = connect(cassandra_hosts)
self.u = ColumnFamily(client, cassandra_keyspace,
'Users', super=True)
self.error = Error()
def refresh(self):
self.cache.flush()
def _levels(self, lang='en'):
key = 'levels_%s' % (lang)
results = self.cache.get(key)
if results:
return results
try:
parent_folder = unicode(os.path.join(data_folder, lang))
_levels = os.listdir(parent_folder)
if _levels == []:
return self.error.not_found("Chưa có dữ liệu")
results = []
# get only folder names
for level in _levels:
if os.path.isdir(os.path.join(parent_folder, level)):
results.append(level)
results = sorted(results)
self.cache.set(key, results)
return results
except OSError:
return 404
def _lessons(self, level, lang='en'):
key = 'lessons_%s_%s' % (level, lang)
results = self.cache.get(key)
if results:
return results
try:
parent_folder = unicode(os.path.join(data_folder, lang))
parent_folder = os.path.join(parent_folder, level)
_lessons = os.listdir(parent_folder)
if _lessons == []:
return self.error.not_found("Chưa có dữ liệu")
results = []
for lesson in _lessons:
if os.path.isdir(os.path.join(parent_folder, lesson)):
results.append(lesson)
results = sorted(results)
self.cache.set(key, results)
return results
except OSError:
return 404
def _listen(self, level, lesson, lang='en'):
key = 'listen_%s_%s_%s' % (level, lesson, lang)
results = self.cache.get(key)
if results:
return results
try:
parent_folder = unicode(os.path.join(data_folder, lang, level, lesson))
audio = os.listdir(parent_folder)
if audio == []:
return self.error.not_found("Chưa có dữ liệu")
results = []
for i in audio:
title = None
description = None
if i.endswith('.inf'):
# add '\t' to detect end string when replace
i = i + '\t'
amr = os.path.join(parent_folder, i.replace('.inf\t', '.amr'))
if not os.path.exists(amr): # check amr file exist
return 404
i = i.replace('\t', '') # restore i
# file inf chứa 2 biến là title và description
cmd = open(os.path.join(parent_folder, i))
exec(cmd) # thực hiện chuỗi như là 1 lệnh
info = {'title': title,
'description': description,
'audio_file': amr.replace(data_folder, '')}
results.append(info)
results = sorted(results)
self.cache.set(key, results)
return results
except OSError:
return 404
def levels(self, environ):
try:
lang = environ['request']['lang']
if lang not in ('en', 'vi'):
return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi")
session_id = environ['request']['session_id']
except KeyError:
return self.error.authen_error()
try:
self.u.get('session_id', super_column=session_id)
except (NotFoundException, InvalidRequestException):
return self.error.authen_error()
results = self._levels(lang)
if results == 404:
return self.error.not_found("Cấu hình hệ thống sai")
xml = ['<level name="%s"/>' % result for result in results]
return '\n\t'.join(xml)
def lessons(self, environ):
try:
level = environ['request']['level']
lang = environ['request']['lang']
if lang not in ('en', 'vi'):
return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi")
session_id = environ['request']['session_id']
except KeyError:
return self.error.authen_error()
try:
self.u.get('session_id', super_column=session_id)
except (NotFoundException, InvalidRequestException):
return self.error.authen_error()
results = self._lessons(level, lang)
if results == 404:
return self.error.not_found("Cấu hình hệ thống sai")
xml = ['<lesson name="%s"/>' % result for result in results]
return '\n\t'.join(xml)
def listen(self, environ):
try:
level = environ['request']['level']
lang = environ['request']['lang']
if lang not in ('en', 'vi'):
return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi")
lesson = environ['request']['lesson']
session_id = environ['request']['session_id']
except KeyError:
return self.error.authen_error()
try:
self.u.get('session_id', super_column=session_id)
except (NotFoundException, InvalidRequestException):
return self.error.authen_error()
results = self._listen(level, lesson, lang)
if results == 404:
return self.error.not_found("Cấu hình hệ thống sai")
xml = ['<audio title="%s" description="%s" link="%s"/>' \
% (x['title'], x['description'], audio_url_prefix + x['audio_file']) \
for x in results]
return '\n\t'.join(xml)
class TestAutoPacking:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1',
credentials=credentials)
self.cf = ColumnFamily(self.client, 'Standard2')
self.cf_long = ColumnFamily(self.client, 'StdLong')
self.cf_int = ColumnFamily(self.client, 'StdInteger')
self.cf_time = ColumnFamily(self.client, 'StdTimeUUID')
self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID')
self.cf_ascii = ColumnFamily(self.client, 'StdAscii')
self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8')
self.cf_bytes = ColumnFamily(self.client, 'StdBytes')
self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True)
self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True)
self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True)
self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True)
self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True)
self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True)
self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True)
self.cf_suplong_sublong = ColumnFamily(self.client,
'SuperLongSubLong',
super=True)
self.cf_suplong_subint = ColumnFamily(self.client,
'SuperLongSubInt',
super=True)
self.cf_suplong_subtime = ColumnFamily(self.client,
'SuperLongSubTime',
super=True)
self.cf_suplong_sublex = ColumnFamily(self.client,
'SuperLongSubLex',
super=True)
self.cf_suplong_subascii = ColumnFamily(self.client,
'SuperLongSubAscii',
super=True)
self.cf_suplong_subutf8 = ColumnFamily(self.client,
'SuperLongSubUTF8',
super=True)
self.cf_suplong_subbytes = ColumnFamily(self.client,
'SuperLongSubBytes',
super=True)
self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong')
self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt')
self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime')
self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex')
self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii')
self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8')
self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes')
self.cf_def_valid = ColumnFamily(self.client, 'DefaultValidator')
self.cfs = [
self.cf_long,
self.cf_int,
self.cf_time,
self.cf_lex,
self.cf_ascii,
self.cf_utf8,
self.cf_bytes,
#
self.cf_suplong,
self.cf_supint,
self.cf_suptime,
self.cf_suplex,
self.cf_supascii,
self.cf_suputf8,
self.cf_supbytes,
#
self.cf_suplong_subint,
self.cf_suplong_subint,
self.cf_suplong_subtime,
self.cf_suplong_sublex,
self.cf_suplong_subascii,
self.cf_suplong_subutf8,
self.cf_suplong_subbytes,
#
self.cf_valid_long,
self.cf_valid_int,
self.cf_valid_time,
self.cf_valid_lex,
self.cf_valid_ascii,
self.cf_valid_utf8,
self.cf_valid_bytes,
#
self.cf_def_valid,
]
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
for cf in self.cfs:
for key, columns in cf.get_range():
cf.remove(key)
def test_basic_inserts(self):
long_col = {1111111111111111L: VALS[0]}
int_col = {1: VALS[0]}
time_col = {TIME1: VALS[0]}
lex_col = {uuid.UUID(bytes='abc abc abc abcd'): VALS[0]}
ascii_col = {'foo': VALS[0]}
utf8_col = {u'\u0020': VALS[0]}
bytes_col = {'bytes': VALS[0]}
self.cf_long.insert(KEYS[0], long_col)
self.cf_int.insert(KEYS[0], int_col)
self.cf_time.insert(KEYS[0], time_col)
self.cf_lex.insert(KEYS[0], lex_col)
self.cf_ascii.insert(KEYS[0], ascii_col)
self.cf_utf8.insert(KEYS[0], utf8_col)
self.cf_bytes.insert(KEYS[0], bytes_col)
assert self.cf_long.get(KEYS[0]) == long_col
assert self.cf_int.get(KEYS[0]) == int_col
assert self.cf_time.get(KEYS[0]) == time_col
assert self.cf_lex.get(KEYS[0]) == lex_col
assert self.cf_ascii.get(KEYS[0]) == ascii_col
assert self.cf_utf8.get(KEYS[0]) == utf8_col
assert self.cf_bytes.get(KEYS[0]) == bytes_col
self.cf_suplong.insert(KEYS[0], {123L: bytes_col})
class CassandraImporter:
def __init__(self):
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('-s', '--source',
help='Generally the prod cassandra path, list of machines: \
localhost:9162 localhost:9163', nargs='*',
required=True)
parser.add_argument('-d', '--destination',
help='Cassandra path where you need your data: \
localhost:9160 localhost:9161', nargs='*',
required=True)
parser.add_argument('-ks', '--keyspace',
help='The keyspace: myks',
required=True)
parser.add_argument('-cf', '--column_family',
help='The Column family: mycf',
required=True)
parser.add_argument('-k', '--key',
help='A specific key to be imported',
required=False)
parser.add_argument('-c', '--count',
help='Total count of keys to be imported',
required=False)
parser.add_argument('-a', '--all',
action='store_true',
help='Get all. Not recommended!',
required=False)
args = vars(parser.parse_args())
"""Connection setting with cassandra
The script is meant to sync data. So source and destination KS
and CF shold be the same."""
try:
source_pool = ConnectionPool(args["keyspace"],
args["source"])
destination_pool = ConnectionPool(args["keyspace"],
args["destination"])
self.source_cf = ColumnFamily(source_pool,
args["column_family"])
self.source_cf.autopack_names = False
self.source_cf.autopack_values = False
self.source_cf.autopack_keys = False
self.source_cf.default_validation_class = pycassa.types.UTF8Type()
self.destination_cf = ColumnFamily(destination_pool,
args["column_family"])
self.destination_cf.autopack_names = False
self.destination_cf.autopack_values = False
self.destination_cf.autopack_keys = False
self.destination_cf.default_validation_class = pycassa.types.UTF8Type()
except Exception as e:
print "ERROR: The keyspace or the column family does not exist or request is timing out!"
sys.exit()
# Optional data
self.count = args["count"]
if self.count:
self.count = int(self.count)
self.key = args["key"]
self.all = args["all"]
def importData(self):
data = dict()
# Get columns for a key
if self.key:
column_data = self.source_cf.get(self.key)
data[self.key] = column_data
# Get last x keys and their columns
elif self.count:
counter = 0
error_count = 0
for value in self.source_cf.get_range(column_count=0,
filter_empty=False):
if(counter < self.count):
try:
column_data = self.source_cf.get(value[0], column_count=100)
data[value[0]] = column_data
counter += 1
except NotFoundException:
#Ignore keys with empty columns
pass
except Exception:
error_count += 1
if error_count > 10:
# Write the read data
self.insertData(data)
print "ERROR: Remote cassandra is too slow to read, exiting after writing..."
sys.exit()
# Use this to throttle reads from cassandra
time.sleep(0.2)
else:
break
# Get All, Not recommended
elif self.all:
for value in self.source_cf.get_range(column_count=0,
filter_empty=False):
column_data = self.source_cf.get(value[0])
key = value[0]
data[key] = column_data
else:
print "Please pass -c or -k or -a arguments!"
return data
def insertData(self, data):
print "Writing " + str(len(data.keys())) + " keys"
for key, value in data.iteritems():
self.destination_cf.insert(key, value)
def run(self):
self.update_progress(0)
data = self.importData()
self.update_progress(50)
self.insertData(data)
self.update_progress(100)
print "Import complete!"
def update_progress(self, progress):
print '\r[{0}] {1}%'.format('#' * (progress / 10), progress)
class TestColumnFamilyMap:
def setUp(self):
self.client = connect()
self.client.login('Keyspace1', {
'username': '******',
'password': '******'
})
self.cf = ColumnFamily(self.client,
'Keyspace1',
'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
self.empty_map = ColumnFamilyMap(TestEmpty, self.cf, raw_columns=True)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def instance(self, key):
instance = TestUTF8()
instance.key = key
instance.strcol = '1'
instance.intcol = 2
instance.floatcol = 3.5
instance.datetimecol = datetime.now().replace(microsecond=0)
instance.intstrcol = 8
instance.floatstrcol = 4.6
instance.datetimestrcol = datetime.now().replace(microsecond=0)
return instance
def test_will_not_insert_none(self):
for column in ('strcol', 'intcol', 'floatcol', 'datetimecol',
'intstrcol', 'floatstrcol', 'datetimestrcol'):
instance = self.instance(
'TestColumnFamilyMap.test_will_not_insert_none')
setattr(instance, column, None)
assert_raises(TypeError, self.map.insert, instance)
def test_empty(self):
key = 'TestColumnFamilyMap.test_empty'
assert_raises(NotFoundException, self.map.get, key)
assert len(self.map.multiget([key])) == 0
def test_insert_get(self):
instance = self.instance('TestColumnFamilyMap.test_insert_get')
assert_raises(NotFoundException, self.map.get, instance.key)
self.map.insert(instance)
assert self.map.get(instance.key) == instance
assert self.empty_map.get(
instance.key).raw_columns['intstrcol'] == str(instance.intstrcol)
def test_insert_multiget(self):
instance1 = self.instance('TestColumnFamilyMap.test_insert_multiget1')
instance2 = self.instance('TestColumnFamilyMap.test_insert_multiget2')
missing_key = 'TestColumnFamilyMap.test_insert_multiget3'
self.map.insert(instance1)
self.map.insert(instance2)
rows = self.map.multiget([instance1.key, instance2.key, missing_key])
assert len(rows) == 2
assert rows[instance1.key] == instance1
assert rows[instance2.key] == instance2
assert missing_key not in rows
assert self.empty_map.multiget([
instance1.key
])[instance1.key].raw_columns['intstrcol'] == str(instance1.intstrcol)
def test_insert_get_count(self):
instance = self.instance('TestColumnFamilyMap.test_insert_get_count')
self.map.insert(instance)
assert self.map.get_count(instance.key) == 7
def test_insert_get_range(self):
instances = []
for i in xrange(5):
instance = self.instance(
'TestColumnFamilyMap.test_insert_get_range%s' % i)
instances.append(instance)
for instance in instances:
self.map.insert(instance)
rows = list(
self.map.get_range(start=instances[0].key,
finish=instances[-1].key))
assert len(rows) == len(instances)
assert rows == instances
assert list(
self.empty_map.get_range(
start=instances[0].key,
finish=instances[0].key))[0].raw_columns['intstrcol'] == str(
instances[0].intstrcol)
def test_remove(self):
instance = self.instance('TestColumnFamilyMap.test_remove')
self.map.insert(instance)
self.map.remove(instance)
assert_raises(NotFoundException, self.map.get, instance.key)
def test_does_not_insert_extra_column(self):
instance = self.instance(
'TestColumnFamilyMap.test_does_not_insert_extra_column')
instance.othercol = 'Test'
self.map.insert(instance)
get_instance = self.map.get(instance.key)
assert get_instance.strcol == instance.strcol
assert get_instance.intcol == instance.intcol
assert get_instance.floatcol == instance.floatcol
assert get_instance.datetimecol == instance.datetimecol
assert_raises(AttributeError, getattr, get_instance, 'othercol')
def test_has_defaults(self):
key = 'TestColumnFamilyMap.test_has_defaults'
self.cf.insert(key, {'strcol': '1'})
instance = self.map.get(key)
assert instance.intcol == TestUTF8.intcol.default
assert instance.floatcol == TestUTF8.floatcol.default
assert instance.datetimecol == TestUTF8.datetimecol.default
assert instance.intstrcol == TestUTF8.intstrcol.default
assert instance.floatstrcol == TestUTF8.floatstrcol.default
assert instance.datetimestrcol == TestUTF8.datetimestrcol.default
class TestColumnFamily:
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client,
'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2,
timestamp=self.timestamp,
dict_class=TestDict)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def test_empty(self):
key = 'TestColumnFamily.test_empty'
assert_raises(NotFoundException, self.cf.get, key)
assert len(self.cf.multiget([key])) == 0
for key, columns in self.cf.get_range():
assert len(columns) == 0
def test_insert_get(self):
key = 'TestColumnFamily.test_insert_get'
columns = {'1': 'val1', '2': 'val2'}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert self.cf.get(key) == columns
def test_insert_multiget(self):
key1 = 'TestColumnFamily.test_insert_multiget1'
columns1 = {'1': 'val1', '2': 'val2'}
key2 = 'test_insert_multiget1'
columns2 = {'3': 'val1', '4': 'val2'}
missing_key = 'key3'
self.cf.insert(key1, columns1)
self.cf.insert(key2, columns2)
rows = self.cf.multiget([key1, key2, missing_key])
assert len(rows) == 2
assert rows[key1] == columns1
assert rows[key2] == columns2
assert missing_key not in rows
def test_insert_get_count(self):
key = 'TestColumnFamily.test_insert_get_count'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
assert self.cf.get_count(key) == 2
assert_equal(self.cf.get_count(key, column_start='1'), 2)
assert_equal(self.cf.get_count(key, column_finish='2'), 2)
assert_equal(
self.cf.get_count(key, column_start='1', column_finish='2'), 2)
assert_equal(
self.cf.get_count(key, column_start='1', column_finish='1'), 1)
assert_equal(self.cf.get_count(key, columns=['1', '2']), 2)
assert_equal(self.cf.get_count(key, columns=['1']), 1)
def test_insert_multiget_count(self):
keys = [
'TestColumnFamily.test_insert_multiget_count1',
'TestColumnFamily.test_insert_multiget_count2',
'TestColumnFamily.test_insert_multiget_count3'
]
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
result = self.cf.multiget_count(keys)
assert_equal(result[keys[0]], 2)
assert_equal(result[keys[1]], 2)
assert_equal(result[keys[2]], 2)
result = self.cf.multiget_count(keys, column_start='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys,
column_start='1',
column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys,
column_start='1',
column_finish='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
result = self.cf.multiget_count(keys, columns=['1', '2'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, columns=['1'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
def test_insert_get_range(self):
keys = [
'TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)
]
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
assert len(rows) == len(keys)
for i, (k, c) in enumerate(rows):
assert k == keys[i]
assert c == columns
def test_insert_get_indexed_slices(self):
indexed_cf = ColumnFamily(self.client, 'Indexed1')
columns = {'birthdate': 1L}
key = 'key1'
indexed_cf.insert(key,
columns,
write_consistency_level=ConsistencyLevel.ONE)
key = 'key2'
indexed_cf.insert(key,
columns,
write_consistency_level=ConsistencyLevel.ONE)
key = 'key3'
indexed_cf.insert(key,
columns,
write_consistency_level=ConsistencyLevel.ONE)
expr = index.create_index_expression(column_name='birthdate', value=1L)
clause = index.create_index_clause([expr])
result = indexed_cf.get_indexed_slices(clause)
assert len(result) == 3
assert result.get('key1') == columns
assert result.get('key2') == columns
assert result.get('key3') == columns
def test_remove(self):
key = 'TestColumnFamily.test_remove'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
self.cf.remove(key, columns=['2'])
del columns['2']
assert self.cf.get(key) == {'1': 'val1'}
self.cf.remove(key)
assert_raises(NotFoundException, self.cf.get, key)
def test_dict_class(self):
key = 'TestColumnFamily.test_dict_class'
self.cf.insert(key, {'1': 'val1'})
assert isinstance(self.cf.get(key), TestDict)
class TestTimeUUIDs(unittest.TestCase):
def setUp(self):
self.cf_time = ColumnFamily(pool, 'StdTimeUUID')
def tearDown(self):
self.cf_time.remove('key1')
def test_datetime_to_uuid(self):
key = 'key1'
timeline = []
timeline.append(datetime.now())
time1 = uuid1()
col1 = {time1:'0'}
self.cf_time.insert(key, col1)
time.sleep(1)
timeline.append(datetime.now())
time2 = uuid1()
col2 = {time2:'1'}
self.cf_time.insert(key, col2)
time.sleep(1)
timeline.append(datetime.now())
cols = {time1:'0', time2:'1'}
assert_equal(self.cf_time.get(key, column_start=timeline[0]) , cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]) , col1)
assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]) , col2)
def test_time_to_uuid(self):
key = 'key1'
timeline = []
timeline.append(time.time())
time1 = uuid1()
col1 = {time1:'0'}
self.cf_time.insert(key, col1)
time.sleep(0.1)
timeline.append(time.time())
time2 = uuid1()
col2 = {time2:'1'}
self.cf_time.insert(key, col2)
time.sleep(0.1)
timeline.append(time.time())
cols = {time1:'0', time2:'1'}
assert_equal(self.cf_time.get(key, column_start=timeline[0]) , cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols)
assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]) , col1)
assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]) , col2)
def test_auto_time_to_uuid1(self):
key = 'key1'
t = time.time()
col = {t: 'foo'}
self.cf_time.insert(key, col)
uuid_res = self.cf_time.get(key).keys()[0]
timestamp = convert_uuid_to_time(uuid_res)
assert_almost_equal(timestamp, t, places=3)
class TestColumnFamily:
def setUp(self):
self.client = connect()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
dict_class=TestDict)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def test_empty(self):
key = 'TestColumnFamily.test_empty'
assert_raises(NotFoundException, self.cf.get, key)
assert len(self.cf.multiget([key])) == 0
for key, columns in self.cf.get_range():
assert len(columns) == 0
def test_insert_get(self):
key = 'TestColumnFamily.test_insert_get'
columns = {'1': 'val1', '2': 'val2'}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert self.cf.get(key) == columns
def test_insert_multiget(self):
key1 = 'TestColumnFamily.test_insert_multiget1'
columns1 = {'1': 'val1', '2': 'val2'}
key2 = 'test_insert_multiget1'
columns2 = {'3': 'val1', '4': 'val2'}
missing_key = 'key3'
self.cf.insert(key1, columns1)
self.cf.insert(key2, columns2)
rows = self.cf.multiget([key1, key2, missing_key])
assert len(rows) == 2
assert rows[key1] == columns1
assert rows[key2] == columns2
assert missing_key not in rows
def test_insert_get_count(self):
key = 'TestColumnFamily.test_insert_get_count'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
assert self.cf.get_count(key) == 2
def test_insert_get_range(self):
keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)]
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
assert len(rows) == len(keys)
for i, (k, c) in enumerate(rows):
assert k == keys[i]
assert c == columns
def test_remove(self):
key = 'TestColumnFamily.test_remove'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
self.cf.remove(key, columns=['2'])
del columns['2']
assert self.cf.get(key) == {'1': 'val1'}
self.cf.remove(key)
assert_raises(NotFoundException, self.cf.get, key)
def test_dict_class(self):
key = 'TestColumnFamily.test_dict_class'
self.cf.insert(key, {'1': 'val1'})
assert isinstance(self.cf.get(key), TestDict)
class CassandraCache(CacheUtils):
permanent = True
"""A cache that uses a Cassandra ColumnFamily. Uses only the
column-name 'value'"""
def __init__(self, column_family, client,
read_consistency_level = CL_ONE,
write_consistency_level = CL_QUORUM):
self.column_family = column_family
self.client = client
self.read_consistency_level = read_consistency_level
self.write_consistency_level = write_consistency_level
self.cf = ColumnFamily(self.client,
self.column_family,
read_consistency_level = read_consistency_level,
write_consistency_level = write_consistency_level)
def _rcl(self, alternative):
return (alternative if alternative is not None
else self.cf.read_consistency_level)
def _wcl(self, alternative):
return (alternative if alternative is not None
else self.cf.write_consistency_level)
def get(self, key, default = None, read_consistency_level = None):
try:
rcl = self._rcl(read_consistency_level)
row = self.cf.get(key, columns=['value'],
read_consistency_level = rcl)
return pickle.loads(row['value'])
except (CassandraNotFound, KeyError):
return default
def simple_get_multi(self, keys, read_consistency_level = None):
rcl = self._rcl(read_consistency_level)
rows = self.cf.multiget(list(keys),
columns=['value'],
read_consistency_level = rcl)
return dict((key, pickle.loads(row['value']))
for (key, row) in rows.iteritems())
def set(self, key, val,
write_consistency_level = None,
time = None):
if val == NoneResult:
# NoneResult caching is for other parts of the chain
return
wcl = self._wcl(write_consistency_level)
ret = self.cf.insert(key, {'value': pickle.dumps(val)},
write_consistency_level = wcl,
ttl = time)
self._warm([key])
return ret
def set_multi(self, keys, prefix='',
write_consistency_level = None,
time = None):
if not isinstance(keys, dict):
# allow iterables yielding tuples
keys = dict(keys)
wcl = self._wcl(write_consistency_level)
ret = {}
with self.cf.batch(write_consistency_level = wcl):
for key, val in keys.iteritems():
if val != NoneResult:
ret[key] = self.cf.insert('%s%s' % (prefix, key),
{'value': pickle.dumps(val)},
ttl = time or None)
self._warm(keys.keys())
return ret
def _warm(self, keys):
import random
if False and random.random() > 0.98:
print 'Warming', keys
self.cf.multiget(keys)
def delete(self, key, write_consistency_level = None):
wcl = self._wcl(write_consistency_level)
self.cf.remove(key, write_consistency_level = wcl)
class TestColumnFamilyMap:
def setUp(self):
self.client = connect()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
self.empty_map = ColumnFamilyMap(TestEmpty, self.cf, raw_columns=True)
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def tearDown(self):
self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})
# Since the timestamp passed to Cassandra will be in the same second
# with the default timestamp function, causing problems with removing
# and inserting (Cassandra doesn't know which is later), we supply our own
def timestamp(self):
self.timestamp_n += 1
return self.timestamp_n
def clear(self):
for key, columns in self.cf.get_range(include_timestamp=True):
for value, timestamp in columns.itervalues():
self.timestamp_n = max(self.timestamp_n, timestamp)
self.cf.remove(key)
def instance(self, key):
instance = TestUTF8()
instance.key = key
instance.strcol = '1'
instance.intcol = 2
instance.floatcol = 3.5
instance.datetimecol = datetime.now().replace(microsecond=0)
instance.intstrcol = 8
instance.floatstrcol = 4.6
instance.datetimestrcol = datetime.now().replace(microsecond=0)
return instance
def test_will_not_insert_none(self):
for column in ('strcol', 'intcol', 'floatcol', 'datetimecol',
'intstrcol', 'floatstrcol', 'datetimestrcol'):
instance = self.instance('TestColumnFamilyMap.test_will_not_insert_none')
setattr(instance, column, None)
assert_raises(TypeError, self.map.insert, instance)
def test_empty(self):
key = 'TestColumnFamilyMap.test_empty'
assert_raises(NotFoundException, self.map.get, key)
assert len(self.map.multiget([key])) == 0
def test_insert_get(self):
instance = self.instance('TestColumnFamilyMap.test_insert_get')
assert_raises(NotFoundException, self.map.get, instance.key)
self.map.insert(instance)
assert self.map.get(instance.key) == instance
assert self.empty_map.get(instance.key).raw_columns['intstrcol'] == str(instance.intstrcol)
def test_insert_multiget(self):
instance1 = self.instance('TestColumnFamilyMap.test_insert_multiget1')
instance2 = self.instance('TestColumnFamilyMap.test_insert_multiget2')
missing_key = 'TestColumnFamilyMap.test_insert_multiget3'
self.map.insert(instance1)
self.map.insert(instance2)
rows = self.map.multiget([instance1.key, instance2.key, missing_key])
assert len(rows) == 2
assert rows[instance1.key] == instance1
assert rows[instance2.key] == instance2
assert missing_key not in rows
assert self.empty_map.multiget([instance1.key])[instance1.key].raw_columns['intstrcol'] == str(instance1.intstrcol)
def test_insert_get_count(self):
instance = self.instance('TestColumnFamilyMap.test_insert_get_count')
self.map.insert(instance)
assert self.map.get_count(instance.key) == 7
def test_insert_get_range(self):
instances = []
for i in xrange(5):
instance = self.instance('TestColumnFamilyMap.test_insert_get_range%s' % i)
instances.append(instance)
for instance in instances:
self.map.insert(instance)
rows = list(self.map.get_range(start=instances[0].key, finish=instances[-1].key))
assert len(rows) == len(instances)
assert rows == instances
assert list(self.empty_map.get_range(start=instances[0].key, finish=instances[0].key))[0].raw_columns['intstrcol'] == str(instances[0].intstrcol)
def test_remove(self):
instance = self.instance('TestColumnFamilyMap.test_remove')
self.map.insert(instance)
self.map.remove(instance)
assert_raises(NotFoundException, self.map.get, instance.key)
def test_does_not_insert_extra_column(self):
instance = self.instance('TestColumnFamilyMap.test_does_not_insert_extra_column')
instance.othercol = 'Test'
self.map.insert(instance)
get_instance = self.map.get(instance.key)
assert get_instance.strcol == instance.strcol
assert get_instance.intcol == instance.intcol
assert get_instance.floatcol == instance.floatcol
assert get_instance.datetimecol == instance.datetimecol
assert_raises(AttributeError, getattr, get_instance, 'othercol')
def test_has_defaults(self):
key = 'TestColumnFamilyMap.test_has_defaults'
self.cf.insert(key, {'strcol': '1'})
instance = self.map.get(key)
assert instance.intcol == TestUTF8.intcol.default
assert instance.floatcol == TestUTF8.floatcol.default
assert instance.datetimecol == TestUTF8.datetimecol.default
assert instance.intstrcol == TestUTF8.intstrcol.default
assert instance.floatstrcol == TestUTF8.floatstrcol.default
assert instance.datetimestrcol == TestUTF8.datetimestrcol.default
class CassandraCache(CacheUtils):
permanent = True
"""A cache that uses a Cassandra ColumnFamily. Uses only the
column-name 'value'"""
def __init__(self,
column_family,
client,
read_consistency_level=CL_ONE,
write_consistency_level=CL_QUORUM):
self.column_family = column_family
self.client = client
self.read_consistency_level = read_consistency_level
self.write_consistency_level = write_consistency_level
self.cf = ColumnFamily(self.client,
self.column_family,
read_consistency_level=read_consistency_level,
write_consistency_level=write_consistency_level)
def _rcl(self, alternative):
return (alternative
if alternative is not None else self.cf.read_consistency_level)
def _wcl(self, alternative):
return (alternative if alternative is not None else
self.cf.write_consistency_level)
def get(self, key, default=None, read_consistency_level=None):
try:
rcl = self._rcl(read_consistency_level)
row = self.cf.get(key,
columns=['value'],
read_consistency_level=rcl)
return pickle.loads(row['value'])
except (CassandraNotFound, KeyError):
return default
def simple_get_multi(self, keys, read_consistency_level=None):
rcl = self._rcl(read_consistency_level)
rows = self.cf.multiget(list(keys),
columns=['value'],
read_consistency_level=rcl)
return dict((key, pickle.loads(row['value']))
for (key, row) in rows.iteritems())
def set(self, key, val, write_consistency_level=None, time=None):
if val == NoneResult:
# NoneResult caching is for other parts of the chain
return
wcl = self._wcl(write_consistency_level)
ret = self.cf.insert(key, {'value': pickle.dumps(val)},
write_consistency_level=wcl,
ttl=time)
self._warm([key])
return ret
def set_multi(self,
keys,
prefix='',
write_consistency_level=None,
time=None):
if not isinstance(keys, dict):
# allow iterables yielding tuples
keys = dict(keys)
wcl = self._wcl(write_consistency_level)
ret = {}
with self.cf.batch(write_consistency_level=wcl):
for key, val in keys.iteritems():
if val != NoneResult:
ret[key] = self.cf.insert('%s%s' % (prefix, key),
{'value': pickle.dumps(val)},
ttl=time)
self._warm(keys.keys())
return ret
def _warm(self, keys):
import random
if False and random.random() > 0.98:
print 'Warming', keys
self.cf.multiget(keys)
def delete(self, key, write_consistency_level=None):
wcl = self._wcl(write_consistency_level)
self.cf.remove(key, write_consistency_level=wcl)
class TestMutator(unittest.TestCase):
def setUp(self):
credentials = {"username": "******", "password": "******"}
self.pool = ConnectionPool(keyspace="Keyspace1", credentials=credentials)
self.cf = ColumnFamily(self.pool, "Standard2")
self.scf = ColumnFamily(self.pool, "Super1")
def tearDown(self):
for key, cols in self.cf.get_range():
self.cf.remove(key)
for key, cols in self.scf.get_range():
self.scf.remove(key)
def test_insert(self):
batch = self.cf.batch()
for key, cols in ROWS.iteritems():
batch.insert(key, cols)
batch.send()
for key, cols in ROWS.items():
assert self.cf.get(key) == cols
def test_insert_supercolumns(self):
batch = self.scf.batch()
batch.insert("one", ROWS)
batch.insert("two", ROWS)
batch.insert("three", ROWS)
batch.send()
assert self.scf.get("one") == ROWS
assert self.scf.get("two") == ROWS
assert self.scf.get("three") == ROWS
def test_queue_size(self):
batch = self.cf.batch(queue_size=2)
batch.insert("1", ROWS["1"])
batch.insert("2", ROWS["2"])
batch.insert("3", ROWS["3"])
assert self.cf.get("1") == ROWS["1"]
assert_raises(NotFoundException, self.cf.get, "3")
batch.send()
for key, cols in ROWS.items():
assert self.cf.get(key) == cols
def test_remove_key(self):
batch = self.cf.batch()
batch.insert("1", ROWS["1"])
batch.remove("1")
batch.send()
assert_raises(NotFoundException, self.cf.get, "1")
def test_remove_columns(self):
batch = self.cf.batch()
batch.insert("1", {"a": "123", "b": "123"})
batch.remove("1", ["a"])
batch.send()
assert self.cf.get("1") == {"b": "123"}
def test_remove_supercolumns(self):
batch = self.scf.batch()
batch.insert("one", ROWS)
batch.insert("two", ROWS)
batch.insert("three", ROWS)
batch.remove("two", ["b"], "2")
batch.send()
assert self.scf.get("one") == ROWS
assert self.scf.get("two")["2"] == {"a": "234"}
assert self.scf.get("three") == ROWS
def test_chained(self):
batch = self.cf.batch()
batch.insert("1", ROWS["1"]).insert("2", ROWS["2"]).insert("3", ROWS["3"]).send()
assert self.cf.get("1") == ROWS["1"]
assert self.cf.get("2") == ROWS["2"]
assert self.cf.get("3") == ROWS["3"]
def test_contextmgr(self):
if sys.version_info < (2, 5):
raise SkipTest("No context managers in Python < 2.5")
exec """with self.cf.batch(queue_size=2) as b:
b.insert('1', ROWS['1'])
b.insert('2', ROWS['2'])
b.insert('3', ROWS['3'])
assert self.cf.get('3') == ROWS['3']"""
def test_multi_column_family(self):
batch = batch_mod.Mutator(self.pool)
cf2 = self.cf
batch.insert(self.cf, "1", ROWS["1"])
batch.insert(self.cf, "2", ROWS["2"])
batch.remove(cf2, "1", ROWS["1"])
batch.send()
assert self.cf.get("2") == ROWS["2"]
assert_raises(NotFoundException, self.cf.get, "1")
class TestTimeUUIDs(unittest.TestCase):
def setUp(self):
self.cf_time = ColumnFamily(pool, 'StdTimeUUID')
def tearDown(self):
self.cf_time.remove('key1')
def test_datetime_to_uuid(self):
key = 'key1'
timeline = []
timeline.append(datetime.now())
time1 = uuid1()
col1 = {time1: '0'}
self.cf_time.insert(key, col1)
time.sleep(1)
timeline.append(datetime.now())
time2 = uuid1()
col2 = {time2: '1'}
self.cf_time.insert(key, col2)
time.sleep(1)
timeline.append(datetime.now())
cols = {time1: '0', time2: '1'}
assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[1]), col1)
assert_equal(
self.cf_time.get(key,
column_start=timeline[1],
column_finish=timeline[2]), col2)
def test_time_to_uuid(self):
key = 'key1'
timeline = []
timeline.append(time.time())
time1 = uuid1()
col1 = {time1: '0'}
self.cf_time.insert(key, col1)
time.sleep(0.1)
timeline.append(time.time())
time2 = uuid1()
col2 = {time2: '1'}
self.cf_time.insert(key, col2)
time.sleep(0.1)
timeline.append(time.time())
cols = {time1: '0', time2: '1'}
assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[2]), cols)
assert_equal(
self.cf_time.get(key,
column_start=timeline[0],
column_finish=timeline[1]), col1)
assert_equal(
self.cf_time.get(key,
column_start=timeline[1],
column_finish=timeline[2]), col2)
def test_auto_time_to_uuid1(self):
key = 'key1'
t = time.time()
col = {t: 'foo'}
self.cf_time.insert(key, col)
uuid_res = self.cf_time.get(key).keys()[0]
timestamp = convert_uuid_to_time(uuid_res)
assert_almost_equal(timestamp, t, places=3)
class TestColumnFamily(unittest.TestCase):
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.pool, 'Standard2', dict_class=TestDict)
def tearDown(self):
for key, columns in self.cf.get_range():
self.cf.remove(key)
def test_empty(self):
key = 'TestColumnFamily.test_empty'
assert_raises(NotFoundException, self.cf.get, key)
assert_equal(len(self.cf.multiget([key])), 0)
for key, columns in self.cf.get_range():
assert_equal(len(columns), 0)
def test_insert_get(self):
key = 'TestColumnFamily.test_insert_get'
columns = {'1': 'val1', '2': 'val2'}
assert_raises(NotFoundException, self.cf.get, key)
self.cf.insert(key, columns)
assert_equal(self.cf.get(key), columns)
def test_insert_multiget(self):
key1 = 'TestColumnFamily.test_insert_multiget1'
columns1 = {'1': 'val1', '2': 'val2'}
key2 = 'test_insert_multiget1'
columns2 = {'3': 'val1', '4': 'val2'}
missing_key = 'key3'
self.cf.insert(key1, columns1)
self.cf.insert(key2, columns2)
rows = self.cf.multiget([key1, key2, missing_key])
assert_equal(len(rows), 2)
assert_equal(rows[key1], columns1)
assert_equal(rows[key2], columns2)
assert_true(missing_key not in rows)
def test_insert_get_count(self):
key = 'TestColumnFamily.test_insert_get_count'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
assert_equal(self.cf.get_count(key), 2)
assert_equal(self.cf.get_count(key, column_start='1'), 2)
assert_equal(self.cf.get_count(key, column_finish='2'), 2)
assert_equal(self.cf.get_count(key, column_start='1', column_finish='2'), 2)
assert_equal(self.cf.get_count(key, column_start='1', column_finish='1'), 1)
assert_equal(self.cf.get_count(key, columns=['1','2']), 2)
assert_equal(self.cf.get_count(key, columns=['1']), 1)
def test_insert_multiget_count(self):
keys = ['TestColumnFamily.test_insert_multiget_count1',
'TestColumnFamily.test_insert_multiget_count2',
'TestColumnFamily.test_insert_multiget_count3']
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
result = self.cf.multiget_count(keys)
assert_equal(result[keys[0]], 2)
assert_equal(result[keys[1]], 2)
assert_equal(result[keys[2]], 2)
result = self.cf.multiget_count(keys, column_start='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_start='1', column_finish='2')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, column_start='1', column_finish='1')
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
result = self.cf.multiget_count(keys, columns=['1','2'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 2)
result = self.cf.multiget_count(keys, columns=['1'])
assert_equal(len(result), 3)
assert_equal(result[keys[0]], 1)
def test_insert_get_range(self):
keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)]
columns = {'1': 'val1', '2': 'val2'}
for key in keys:
self.cf.insert(key, columns)
rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
assert_equal(len(rows), len(keys))
for i, (k, c) in enumerate(rows):
assert_equal(k, keys[i])
assert_equal(c, columns)
def test_get_range_batching(self):
self.cf.truncate()
keys = []
columns = {'c': 'v'}
for i in range(100, 201):
keys.append('key%d' % i)
self.cf.insert('key%d' % i, columns)
for i in range(201, 301):
self.cf.insert('key%d' % i, columns)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=10):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=1000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=150):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=7):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
count = 0
for (k,v) in self.cf.get_range(row_count=100, buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 100)
# Put the remaining keys in our list
for i in range(201, 301):
keys.append('key%d' % i)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=7):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=200):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(row_count=10000, buffer_size=10000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
# Don't give a row count
count = 0
for (k,v) in self.cf.get_range(buffer_size=2):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=77):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=200):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
count = 0
for (k,v) in self.cf.get_range(buffer_size=10000):
assert_true(k in keys, 'key "%s" should be in keys' % k)
count += 1
assert_equal(count, 201)
self.cf.truncate()
def insert_insert_get_indexed_slices(self):
indexed_cf = ColumnFamily(self.pool, 'Indexed1')
columns = {'birthdate': 1L}
keys = []
for i in range(1,4):
indexed_cf.insert('key%d' % i, columns)
keys.append('key%d')
expr = index.create_index_expression(column_name='birthdate', value=1L)
clause = index.create_index_clause([expr])
count = 0
for key,cols in indexed_cf.get_indexed_slices(clause):
assert_equal(cols, columns)
assert key in keys
count += 1
assert_equal(count, 3)
def test_get_indexed_slices_batching(self):
indexed_cf = ColumnFamily(self.pool, 'Indexed1')
columns = {'birthdate': 1L}
for i in range(200):
indexed_cf.insert('key%d' % i, columns)
expr = index.create_index_expression(column_name='birthdate', value=1L)
clause = index.create_index_clause([expr], count=10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200))
assert_equal(len(result), 10)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000))
assert_equal(len(result), 10)
clause = index.create_index_clause([expr], count=250)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200))
assert_equal(len(result), 200)
result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000))
assert_equal(len(result), 200)
def test_remove(self):
key = 'TestColumnFamily.test_remove'
columns = {'1': 'val1', '2': 'val2'}
self.cf.insert(key, columns)
self.cf.remove(key, columns=['2'])
del columns['2']
assert_equal(self.cf.get(key), {'1': 'val1'})
self.cf.remove(key)
assert_raises(NotFoundException, self.cf.get, key)
def test_dict_class(self):
key = 'TestColumnFamily.test_dict_class'
self.cf.insert(key, {'1': 'val1'})
assert isinstance(self.cf.get(key), TestDict)
