def test_null_pool_failover(self): listener = _TestListener() pool = NullPool(keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(1,5): setattr(cf.client._connection.client, 'batch_mutate', _timeout) # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val'}) assert_equal(listener.failure_count, i) cf.get('key') pool.dispose() listener.reset() pool = NullPool(keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) threads = [] args = (pool, 'key', {'col':'val'}) for i in range(0, 5): threads.append(threading.Thread(target=_five_fails, args=args)) threads[-1].start() for thread in threads: thread.join() assert_equal(listener.failure_count, 25) pool.dispose()
class ActiveCode: def __init__(self): # Connect to Cassandra servers client = connect(cassandra_hosts) self.u = ColumnFamily(client, cassandra_keyspace, 'Users', super=True) self.e = Error() def _gen_active_code(self, length=6, chars=(letters + digits)): return ''.join([choice(chars) for _ in xrange(length)]) def _is_exist(self, code): try: self.u.get('active_codes', super_column=code) return True except NotFoundException: return False def get_new_code(self): new = self._gen_active_code() while self._is_exist(new) is True: new = self._gen_active_code() # store in database self.u.insert('active_codes', {new: {'create_time': str(datetime.now())}}) return new def active(self, username, active_code): if self._is_exist(active_code) is False: return False self.u.insert('active_codes', {active_code: {'active_time': str(datetime.now()), 'owner': str(username)}}) return True def stats(self): return self.u.get('active_codes')
class TestSuperColumnFamily: def setUp(self): credentials = {'username': '******', 'password': '******'} self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.pool, 'Super2') def tearDown(self): for key, columns in self.cf.get_range(): self.cf.remove(key) def test_super(self): key = 'TestSuperColumnFamily.test_super' columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert_equal(self.cf.get(key), columns) assert_equal(self.cf.multiget([key]), {key: columns}) assert_equal(list(self.cf.get_range(start=key, finish=key)), [(key, columns)]) def test_super_column_argument(self): key = 'TestSuperColumnFamily.test_super_columns_argument' sub12 = {'sub1': 'val1', 'sub2': 'val2'} sub34 = {'sub3': 'val3', 'sub4': 'val4'} columns = {'1': sub12, '2': sub34} self.cf.insert(key, columns) assert_equal(self.cf.get(key, super_column='1'), sub12) assert_raises(NotFoundException, self.cf.get, key, super_column='3') assert_equal(self.cf.multiget([key], super_column='1'), {key: sub12}) assert_equal(list(self.cf.get_range(start=key, finish=key, super_column='1')), [(key, sub12)])
def _five_fails(pool, key, column): conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(0,5): setattr(cf.client._connection.client, 'batch_mutate', _timeout) # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert(key, column) cf.get(key)
class TestSuperColumnFamily: def setUp(self): self.client = connect_thread_local() self.client.login('Keyspace1', {'username': '******', 'password': '******'}) self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2', write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, super=True) try: self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for subcolumns in columns.itervalues(): for value, timestamp in subcolumns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def test_super(self): key = 'TestSuperColumnFamily.test_super' columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert self.cf.get(key) == columns assert self.cf.multiget([key]) == {key: columns} assert list(self.cf.get_range(start=key, finish=key)) == [(key, columns)] def test_super_column_argument(self): key = 'TestSuperColumnFamily.test_super_columns_argument' sub12 = {'sub1': 'val1', 'sub2': 'val2'} sub34 = {'sub3': 'val3', 'sub4': 'val4'} columns = {'1': sub12, '2': sub34} self.cf.insert(key, columns) assert self.cf.get(key, super_column='1') == sub12 assert_raises(NotFoundException, self.cf.get, key, super_column='3') assert self.cf.multiget([key], super_column='1') == {key: sub12} assert list(self.cf.get_range(start=key, finish=key, super_column='1')) == [(key, sub12)]
class TestSuperColumnFamily: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect_thread_local('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Super2', write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, super=True) try: self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for subcolumns in columns.itervalues(): for value, timestamp in subcolumns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def test_super(self): key = 'TestSuperColumnFamily.test_super' columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert self.cf.get(key) == columns assert self.cf.multiget([key]) == {key: columns} assert list(self.cf.get_range(start=key, finish=key)) == [(key, columns)] def test_super_column_argument(self): key = 'TestSuperColumnFamily.test_super_columns_argument' sub12 = {'sub1': 'val1', 'sub2': 'val2'} sub34 = {'sub3': 'val3', 'sub4': 'val4'} columns = {'1': sub12, '2': sub34} self.cf.insert(key, columns) assert self.cf.get(key, super_column='1') == sub12 assert_raises(NotFoundException, self.cf.get, key, super_column='3') assert self.cf.multiget([key], super_column='1') == {key: sub12} assert list(self.cf.get_range(start=key, finish=key, super_column='1')) == [(key, sub12)]
class TestDefaultValidators(unittest.TestCase): def setUp(self): credentials = {"username": "******", "password": "******"} self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials) self.cf_def_valid = ColumnFamily(self.pool, "DefaultValidator") def tearDown(self): for key, cols in self.cf_def_valid.get_range(): self.cf_def_valid.remove(key) self.pool.dispose() def test_default_validated_columns(self): key = "key1" col_cf = {"aaaaaa": 1L} col_cm = {"subcol": TIME1} col_ncf = {"aaaaaa": TIME1} col_ncm = {"subcol": 1L} # Both of these inserts work, as cf allows # longs and cm for 'subcol' allows TIMEUUIDs. self.cf_def_valid.insert(key, col_cf) self.cf_def_valid.insert(key, col_cm) assert self.cf_def_valid.get(key) == {"aaaaaa": 1L, "subcol": TIME1} assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncf) assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncm)
class Database: def __init__(self, column_family): """ ColumnFamily: - Thông tin người dùng - Tìm kiếm - Thông tin vé - Nhật ký hệ thống - Thông tin nhà cung cấp """ # Connect to Cassandra servers client = connect(CASSANDRA_HOSTS) self.db = ColumnFamily(client, CASSANDRA_KEYSPACE, column_family, super=False) def insert(self, key, columns): key = md5(capwords(key).lower()).hexdigest() return self.db.insert(key, columns) def get(self, key, columns=None): key = md5(capwords(key).lower()).hexdigest() return self.db.get(key=key, columns=columns) def remove(self, key, column=None): key = md5(capwords(key).lower()).hexdigest() return self.db.remove(key=key, column=column)
def test_queue_failover(self): for prefill in (True, False): listener = _TestListener() pool = ConnectionPool( pool_size=1, max_overflow=0, recycle=10000, prefill=prefill, timeout=1, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1, 5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(listener.failure_count, i) assert_equal(cf.get('key'), { 'col': 'val%d' % i, 'col2': 'val' }) pool.dispose()
def test_queue_threadlocal_failover(self): stats_logger = StatsLoggerWithListStorage() pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1, 5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(stats_logger.stats['failed'], i) assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'}) pool.dispose() stats_logger.reset() pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() threads = [] args = ('key', {'col': 'val', 'col2': 'val'}) for i in range(5): threads.append(threading.Thread(target=cf.insert, args=args)) threads[-1].start() for thread in threads: thread.join() assert_equal(stats_logger.stats['failed'], 5) pool.dispose()
class Dictionary: """ Nhóm chức năng từ điển: * Tra từ Anh-Việt * Tra từ Việt-Anh """ def __init__(self): # Connect to Cassandra servers client = connect(cassandra_hosts) self.d = ColumnFamily(client, cassandra_keyspace, 'Dictionary', super=True) self.u = ColumnFamily(client, cassandra_keyspace, 'Users', super=True) self.e = Error() def _lookup(self, keyword, dict_type='en_vi'): try: return self.d.get(dict_type, super_column=str(keyword)) except (NotFoundException, InvalidRequestException): return None def lookup(self, environ): try: session_id = environ['request']['session_id'] except KeyError: return self.e.authen_error("Thiếu session_id") try: self.u.get('session_id', super_column=session_id) except (NotFoundException, InvalidRequestException): return self.e.authen_error() result = self._lookup(environ['request']['keyword']) result2 = self._lookup(environ['request']['keyword'], 'vi_en') result3 = self._lookup(environ['request']['keyword'], 'en_en') if (result is None) and (result2 is None) and (result3 is None): return self.e.not_found("Từ khóa bạn tìm không có trong từ điển") xml = [] if result is not None: xml.append('<result type="en_vi" keyword="%s" mean="%s" spell="%s" status_code="200"/>' \ % (xml_format(environ['request']['keyword']), xml_format(result['nghia']), xml_format(result['phien_am_quoc_te']))) if result2 is not None: xml.append('<result type="vi_en" keyword="%s" mean="%s" spell="" status_code="200"/>' \ % (xml_format(environ['request']['keyword']), xml_format(result2['nghia']))) return '\n\n'.join(xml) def total_words(self, dict_type='en_vi'): return self.d.get_count(dict_type)
class TestValidators(unittest.TestCase): def setUp(self): credentials = {"username": "******", "password": "******"} self.pool = ConnectionPool(pool_size=10, keyspace="Keyspace1", credentials=credentials) self.cf_valid_long = ColumnFamily(self.pool, "ValidatorLong") self.cf_valid_int = ColumnFamily(self.pool, "ValidatorInt") self.cf_valid_time = ColumnFamily(self.pool, "ValidatorTime") self.cf_valid_lex = ColumnFamily(self.pool, "ValidatorLex") self.cf_valid_ascii = ColumnFamily(self.pool, "ValidatorAscii") self.cf_valid_utf8 = ColumnFamily(self.pool, "ValidatorUTF8") self.cf_valid_bytes = ColumnFamily(self.pool, "ValidatorBytes") self.cfs = [ self.cf_valid_long, self.cf_valid_int, self.cf_valid_time, self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8, self.cf_valid_bytes, ] def tearDown(self): for cf in self.cfs: for key, cols in cf.get_range(): cf.remove(key) self.pool.dispose() def test_validated_columns(self): key = "key1" col = {"subcol": 1L} self.cf_valid_long.insert(key, col) assert self.cf_valid_long.get(key) == col col = {"subcol": 1} self.cf_valid_int.insert(key, col) assert self.cf_valid_int.get(key) == col col = {"subcol": TIME1} self.cf_valid_time.insert(key, col) assert self.cf_valid_time.get(key) == col col = {"subcol": uuid.UUID(bytes="aaa aaa aaa aaaa")} self.cf_valid_lex.insert(key, col) assert self.cf_valid_lex.get(key) == col col = {"subcol": "aaa"} self.cf_valid_ascii.insert(key, col) assert self.cf_valid_ascii.get(key) == col col = {"subcol": u"a\u0020"} self.cf_valid_utf8.insert(key, col) assert self.cf_valid_utf8.get(key) == col col = {"subcol": "aaa"} self.cf_valid_bytes.insert(key, col) assert self.cf_valid_bytes.get(key) == col
def test_assertion_threadlocal_failover(self): listener = _TestListener() pool = AssertionPool(keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(1,5): setattr(cf.client._connection.client, 'batch_mutate', _timeout) # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val'}) assert_equal(listener.failure_count, i) cf.get('key') pool.dispose()
class TestSuperColumnFamilyMap: def setUp(self): self.client = connect_thread_local() self.client.login('Keyspace1', {'username': '******', 'password': '******'}) self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp, super=True) self.map = ColumnFamilyMap(TestUTF8, self.cf) try: self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for subcolumns in columns.itervalues(): for value, timestamp in subcolumns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def instance(self, key, super_column): instance = TestUTF8() instance.key = key instance.super_column = super_column instance.strcol = '1' instance.intcol = 2 instance.floatcol = 3.5 instance.datetimecol = datetime.now().replace(microsecond=0) instance.intstrcol = 8 instance.floatstrcol = 4.6 instance.datetimestrcol = datetime.now().replace(microsecond=0) return instance def test_super(self): instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1') assert_raises(NotFoundException, self.map.get, instance.key) self.map.insert(instance) assert self.map.get(instance.key)[instance.super_column] == instance assert self.map.multiget([instance.key])[instance.key][instance.super_column] == instance assert list(self.map.get_range(start=instance.key, finish=instance.key)) == [{instance.super_column: instance}]
def get(self, colfam, key, columns=None, column_start="", column_finish="", column_reversed=False, column_count=100, include_timestamp=False, super_column=None, read_consistency_level=None): cf = ColumnFamily(self.db, colfam) try: return cf.get(key, columns, column_start, column_finish, column_reversed, column_count, include_timestamp, super_column, read_consistency_level) except NotFoundException: return None
class TestSuperColumnFamilyMap: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect_thread_local('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Super2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp, super=True) self.map = ColumnFamilyMap(TestUTF8, self.cf) try: self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for subcolumns in columns.itervalues(): for value, timestamp in subcolumns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def instance(self, key, super_column): instance = TestUTF8() instance.key = key instance.super_column = super_column instance.strcol = '1' instance.intcol = 2 instance.floatcol = 3.5 instance.datetimecol = datetime.now().replace(microsecond=0) instance.intstrcol = 8 instance.floatstrcol = 4.6 instance.datetimestrcol = datetime.now().replace(microsecond=0) return instance def test_super(self): instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1') assert_raises(NotFoundException, self.map.get, instance.key) self.map.insert(instance) assert self.map.get(instance.key)[instance.super_column] == instance assert self.map.multiget([instance.key])[instance.key][instance.super_column] == instance assert list(self.map.get_range(start=instance.key, finish=instance.key)) == [{instance.super_column: instance}]
def test_queue_threadlocal_failover(self): listener = _TestListener() pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1,5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(listener.failure_count, i) assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'}) pool.dispose() listener.reset() pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() threads = [] args=('key', {'col': 'val', 'col2': 'val'}) for i in range(5): threads.append(threading.Thread(target=cf.insert, args=args)) threads[-1].start() for thread in threads: thread.join() assert_equal(listener.failure_count, 5) pool.dispose()
def test_validated_columns(self): sys = SystemManager() sys.create_column_family( TEST_KS, 'Validators', ) sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE) sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE) sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE) sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE) sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE) sys.close() cf = ColumnFamily(pool, 'Validators') key = 'key1' col = {'long': 1L} cf.insert(key, col) assert_equal(cf.get(key)['long'], 1L) col = {'int': 1} cf.insert(key, col) assert_equal(cf.get(key)['int'], 1) col = {'time': TIME1} cf.insert(key, col) assert_equal(cf.get(key)['time'], TIME1) col = {'lex': uuid.UUID(bytes='aaa aaa aaa aaaa')} cf.insert(key, col) assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa')) col = {'ascii': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['ascii'], 'aaa') col = {'utf8': u'a\u0020'} cf.insert(key, col) assert_equal(cf.get(key)['utf8'], u'a\u0020') col = {'bytes': 'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['bytes'], 'aaa') cf.remove(key)
def test_default_validated_columns(self): sys = SystemManager() sys.create_column_family(TEST_KS, 'DefaultValidator', default_validation_class=LONG_TYPE) sys.alter_column(TEST_KS, 'DefaultValidator', 'subcol', TIME_UUID_TYPE) sys.close() cf = ColumnFamily(pool, 'DefaultValidator') key = 'key1' col_cf = {'aaaaaa': 1L} col_cm = {'subcol': TIME1} col_ncf = {'aaaaaa': TIME1} col_ncm = {'subcol': 1L} # Both of these inserts work, as cf allows # longs and cm for 'subcol' allows TIMEUUIDs. cf.insert(key, col_cf) cf.insert(key, col_cm) assert_equal(cf.get(key), {'aaaaaa': 1L, 'subcol': TIME1})
def test_validated_columns(self): sys = SystemManager() sys.create_column_family(TEST_KS, 'Validators',) sys.alter_column(TEST_KS, 'Validators', 'long', LONG_TYPE) sys.alter_column(TEST_KS, 'Validators', 'int', INT_TYPE) sys.alter_column(TEST_KS, 'Validators', 'time', TIME_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'lex', LEXICAL_UUID_TYPE) sys.alter_column(TEST_KS, 'Validators', 'ascii', ASCII_TYPE) sys.alter_column(TEST_KS, 'Validators', 'utf8', UTF8_TYPE) sys.alter_column(TEST_KS, 'Validators', 'bytes', BYTES_TYPE) sys.close() cf = ColumnFamily(pool, 'Validators') key = 'key1' col = {'long':1L} cf.insert(key, col) assert_equal(cf.get(key)['long'], 1L) col = {'int':1} cf.insert(key, col) assert_equal(cf.get(key)['int'], 1) col = {'time':TIME1} cf.insert(key, col) assert_equal(cf.get(key)['time'], TIME1) col = {'lex':uuid.UUID(bytes='aaa aaa aaa aaaa')} cf.insert(key, col) assert_equal(cf.get(key)['lex'], uuid.UUID(bytes='aaa aaa aaa aaaa')) col = {'ascii':'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['ascii'], 'aaa') col = {'utf8':u'a\u0020'} cf.insert(key, col) assert_equal(cf.get(key)['utf8'], u'a\u0020') col = {'bytes':'aaa'} cf.insert(key, col) assert_equal(cf.get(key)['bytes'], 'aaa') cf.remove(key)
def get_variable_data(id_variable, columns): # Load Postgres session session = sqlm.load_session() # get variable by id variable = session.query(sqlm.Variable).\ filter(sqlm.Variable.id == id_variable).\ filter(sqlm.Variable.deletion_date == None).\ first() if not variable: session.close() return {'error': json_error_not_exists(sqlm.Variable)} # build query cassandra, structured by tables query_cassandra = {} # {table name: list of cassandra ids} # query cassandra pool = load_pool() pool.timeout = 300 cf = ColumnFamily(pool, variable.timeseries_cassandra) try: ans_cassandra = cf.get(variable.id_cassandra, **columns) except NotFoundException: session.close() pool.dispose() return {'error': json.dumps({'error': 'No data found'})} output = rearrange_timeseries(ans_cassandra) session.close() pool.dispose() # the output is returned as a list of one list return [output]
def test_queue_failover(self): for prefill in (True, False): listener = _TestListener() pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000, prefill=prefill, timeout=1, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1,5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(listener.failure_count, i) assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'}) pool.dispose()
def setUp(self): credentials = {'username': '******', 'password': '******'} self.pools = [] self.clients = [] self.cfs = [] for pool_cls in _pool_classes: pool = pool_cls(keyspace='Keyspace1', credentials=credentials) self.pools.append(pool) client = pool.get() self.clients.append(client) cf = ColumnFamily(client, 'Standard%s' % pool_cls.__name__, write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, dict_class=TestDict) self.cfs.append(cf) self.timestamp_n = 0 for cf in self.cfs: try: self.timestamp_n = max(self.timestamp_n, int(cf.get('meta')['timestamp'])) except NotFoundException: pass self.clear()
class TestMutator(unittest.TestCase): def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp) self.scf = ColumnFamily(self.client, 'Super1', write_consistency_level=ConsistencyLevel.ONE, super=True, timestamp=self.timestamp) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def clear(self): self.cf.truncate() self.scf.truncate() def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def test_insert(self): batch = self.cf.batch() for key, cols in ROWS.iteritems(): batch.insert(key, cols) batch.send() for key, cols in ROWS.items(): assert self.cf.get(key) == cols def test_insert_supercolumns(self): batch = self.scf.batch() batch.insert('one', ROWS) batch.insert('two', ROWS) batch.insert('three', ROWS) batch.send() assert self.scf.get('one') == ROWS assert self.scf.get('two') == ROWS assert self.scf.get('three') == ROWS def test_queue_size(self): batch = self.cf.batch(queue_size=2) batch.insert('1', ROWS['1']) batch.insert('2', ROWS['2']) batch.insert('3', ROWS['3']) assert self.cf.get('1') == ROWS['1'] assert_raises(NotFoundException, self.cf.get, '3') batch.send() for key, cols in ROWS.items(): assert self.cf.get(key) == cols def test_remove_key(self): batch = self.cf.batch() batch.insert('1', ROWS['1']) batch.remove('1') batch.send() assert_raises(NotFoundException, self.cf.get, '1') def test_remove_columns(self): batch = self.cf.batch() batch.insert('1', {'a': '123', 'b': '123'}) batch.remove('1', ['a']) batch.send() assert self.cf.get('1') == {'b': '123'} def test_remove_supercolumns(self): batch = self.scf.batch() batch.insert('one', ROWS) batch.insert('two', ROWS) batch.insert('three', ROWS) batch.remove('two', ['b'], '2') batch.send() assert self.scf.get('one') == ROWS assert self.scf.get('two')['2'] == {'a': '234'} assert self.scf.get('three') == ROWS def test_chained(self): batch = self.cf.batch() batch.insert('1', ROWS['1']).insert('2', ROWS['2']).insert('3', ROWS['3']).send() assert self.cf.get('1') == ROWS['1'] assert self.cf.get('2') == ROWS['2'] assert self.cf.get('3') == ROWS['3'] def test_contextmgr(self): if sys.version_info < (2, 5): raise SkipTest("No context managers in Python < 2.5") exec """with self.cf.batch(queue_size=2) as b: b.insert('1', ROWS['1']) b.insert('2', ROWS['2']) b.insert('3', ROWS['3']) assert self.cf.get('3') == ROWS['3']""" def test_multi_column_family(self): batch = self.client.batch() cf2 = self.cf batch.insert(self.cf, '1', ROWS['1']) batch.insert(self.cf, '2', ROWS['2']) batch.remove(cf2, '1', ROWS['1']) batch.send() assert self.cf.get('2') == ROWS['2'] assert_raises(NotFoundException, self.cf.get, '1')
class TestColumnFamily: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2', write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, dict_class=TestDict) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def test_empty(self): key = 'TestColumnFamily.test_empty' assert_raises(NotFoundException, self.cf.get, key) assert len(self.cf.multiget([key])) == 0 for key, columns in self.cf.get_range(): assert len(columns) == 0 def test_insert_get(self): key = 'TestColumnFamily.test_insert_get' columns = {'1': 'val1', '2': 'val2'} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert self.cf.get(key) == columns def test_insert_multiget(self): key1 = 'TestColumnFamily.test_insert_multiget1' columns1 = {'1': 'val1', '2': 'val2'} key2 = 'test_insert_multiget1' columns2 = {'3': 'val1', '4': 'val2'} missing_key = 'key3' self.cf.insert(key1, columns1) self.cf.insert(key2, columns2) rows = self.cf.multiget([key1, key2, missing_key]) assert len(rows) == 2 assert rows[key1] == columns1 assert rows[key2] == columns2 assert missing_key not in rows def test_insert_get_count(self): key = 'TestColumnFamily.test_insert_get_count' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) assert self.cf.get_count(key) == 2 assert_equal(self.cf.get_count(key, column_start='1'), 2) assert_equal(self.cf.get_count(key, column_finish='2'), 2) assert_equal(self.cf.get_count(key, column_start='1', column_finish='2'), 2) assert_equal(self.cf.get_count(key, column_start='1', column_finish='1'), 1) assert_equal(self.cf.get_count(key, columns=['1','2']), 2) assert_equal(self.cf.get_count(key, columns=['1']), 1) def test_insert_multiget_count(self): keys = ['TestColumnFamily.test_insert_multiget_count1', 'TestColumnFamily.test_insert_multiget_count2', 'TestColumnFamily.test_insert_multiget_count3'] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) result = self.cf.multiget_count(keys) assert_equal(result[keys[0]], 2) assert_equal(result[keys[1]], 2) assert_equal(result[keys[2]], 2) result = self.cf.multiget_count(keys, column_start='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) result = self.cf.multiget_count(keys, columns=['1','2']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, columns=['1']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) def test_insert_get_range(self): keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) rows = list(self.cf.get_range(start=keys[0], finish=keys[-1])) assert len(rows) == len(keys) for i, (k, c) in enumerate(rows): assert k == keys[i] assert c == columns def test_get_range_batching(self): self.cf.truncate() keys = [] columns = {'c': 'v'} for i in range(100, 201): keys.append('key%d' % i) self.cf.insert('key%d' % i, columns) for i in range(201, 301): self.cf.insert('key%d' % i, columns) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=10): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=1000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=150): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=7): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) # Put the remaining keys in our list for i in range(201, 301): keys.append('key%d' % i) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=7): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=200): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=10000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) # Don't give a row count count = 0 for (k,v) in self.cf.get_range(buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=77): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=200): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=10000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) self.cf.truncate() def insert_insert_get_indexed_slices(self): indexed_cf = ColumnFamily(self.client, 'Indexed1') columns = {'birthdate': 1L} keys = [] for i in range(1,4): indexed_cf.insert('key%d' % i, columns) keys.append('key%d') expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr]) count = 0 for key,cols in indexed_cf.get_indexed_slices(clause): assert cols == columns assert key in keys count += 1 assert_equal(count, 3) def test_get_indexed_slices_batching(self): indexed_cf = ColumnFamily(self.client, 'Indexed1') columns = {'birthdate': 1L} for i in range(200): indexed_cf.insert('key%d' % i, columns) expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr], count=10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 10) clause = index.create_index_clause([expr], count=250) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 200) def test_remove(self): key = 'TestColumnFamily.test_remove' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) self.cf.remove(key, columns=['2']) del columns['2'] assert self.cf.get(key) == {'1': 'val1'} self.cf.remove(key) assert_raises(NotFoundException, self.cf.get, key) def test_dict_class(self): key = 'TestColumnFamily.test_dict_class' self.cf.insert(key, {'1': 'val1'}) assert isinstance(self.cf.get(key), TestDict)
class Authentication: """ Nhóm chức năng quản lý người dùng Các hành động hỗ trợ: * đăng ký * đăng nhập * đăng xuất """ def __init__(self): # Connect to Cassandra servers client = connect(cassandra_hosts) self.a = ColumnFamily(client, cassandra_keyspace, "Users", super=True) self.e = Error() def _check(self, username): try: username = username.strip().lower() self.a.get(username) return 1 except (NotFoundException, InvalidRequestException): return 0 def _remove(self, username): try: username = username.strip().lower() self.a.remove(username) return 1 except (NotFoundException, InvalidRequestException): return 0 def _register(self, username, password, phone_number): try: username = username.strip().lower() if self._check(username) == 1: return False salt = str(random()) password = sha512(password + salt).hexdigest() self.a.insert( username, { "info": { "password": password, "salt": salt, "phone_number": str(phone_number), "join_time": str(datetime.now()), } }, ) return True except (NotFoundException, InvalidRequestException): return False def _login(self, username, password): try: username = username.strip().lower() store_data = self.a.get(username, super_column="info") store_passwd = store_data["password"] store_salt = store_data["salt"] store_join_time = datetime.strptime(store_data["join_time"], "%Y-%m-%d %H:%M:%S.%f") password = sha512(password + store_salt).hexdigest() if password == store_passwd: session_id = md5(username + str(datetime.now())).hexdigest() # --- check trial expire --- delta = datetime.now() - store_join_time if delta.days > trial_time: return 402 # --- end check --- self.a.insert("session_id", {session_id: {"username": username, "timestamp": str(datetime.now())}}) return session_id return False except (NotFoundException, InvalidRequestException): return False def _logout(self, session_id): try: self.a.get("session_id", super_column=session_id) # check if exist self.a.remove("session_id", session_id) return True except (NotFoundException, InvalidRequestException): return False def register(self, environ): try: username = environ["request"]["username"] password = environ["request"]["password"] phone_number = environ["request"]["phone_number"] except KeyError: return self.e.bad_request("Hành động này yêu cầu 4 tham số: username, password, phone_number và api_key") username = username.strip().lower() # check params allow_chars = lowercase + ".-_@" for char in username: if char not in allow_chars: self.e.param_error("Tên đăng nhập chỉ được dùng các ký tự a-z, A-Z và .-_@") if len(password) != 32: return self.e.param_error("Mật khẩu phải được mã hóa md5 trước khi gửi lên") # ----- check phone_number (dạng 841673450799) ----- if phone_number != "": phone_number = phone_number.replace(" ", "") phone_number = "\t" + phone_number if phone_number.startswith("\t+"): phone_number = phone_number.replace("\t+", "\t") # remove + # normalize if phone_number.startswith("\t09"): phone_number = phone_number.replace("\t09", "849") elif phone_number.startswith("\t01"): phone_number = phone_number.replace("\t01", "841") else: return self.e.param_error("phone_number không hợp lệ") if len(phone_number) in (11, 12): try: phone_number = int(phone_number) except ValueError: # not int return self.e.param_error("phone_number không hợp lệ") else: return self.e.param_error("phone_number không hợp lệ") # ----- end phone_number check ----- result = self._register(username, password, phone_number) if result is True: xml = '<register status_code="201" description="Tài khoản đã được tạo thành công"/>' return xml else: xml = '<register status_code="406" description="Tham số không hợp lệ"/>' return xml def login(self, environ): try: username = environ["request"]["username"] password = environ["request"]["password"] except KeyError: return self.e.bad_request("Hành động này yêu cầu 3 tham số: username, password và api_key") result = self._login(username, password) if result is False: xml = '<login status_code="401" description="Tên đăng nhập/mật khẩu không hợp lệ"/>' return xml elif result == 402: return self.e.trial_expired() else: xml = '<login status_code="202" session_id="%s" description="Đăng nhập thành công"/>' % (result) return xml def logout(self, environ): try: session_id = environ["request"]["session_id"] except KeyError: return self.e.bad_request("Hành động này yêu cầu 2 tham số: session_id và api_key") result = self._logout(session_id) if result is True: xml = '<logout status_code="200" description="Đăng xuất thành công"/>' return xml else: xml = '<logout status_code="401" description="session_id không hợp lệ"/>' return xml
class TestTimeUUIDs(unittest.TestCase): def setUp(self): credentials = {"username": "******", "password": "******"} self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials) self.cf_time = ColumnFamily(self.pool, "StdTimeUUID") def tearDown(self): for key, cols in self.cf_time.get_range(): self.cf_time.remove(key) self.pool.dispose() def test_datetime_to_uuid(self): key = "key1" timeline = [] timeline.append(datetime.now()) time1 = uuid1() col1 = {time1: "0"} self.cf_time.insert(key, col1) time.sleep(1) timeline.append(datetime.now()) time2 = uuid1() col2 = {time2: "1"} self.cf_time.insert(key, col2) time.sleep(1) timeline.append(datetime.now()) cols = {time1: "0", time2: "1"} assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1) assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2) def test_time_to_uuid(self): key = "key1" timeline = [] timeline.append(time.time()) time1 = uuid1() col1 = {time1: "0"} self.cf_time.insert(key, col1) time.sleep(0.1) timeline.append(time.time()) time2 = uuid1() col2 = {time2: "1"} self.cf_time.insert(key, col2) time.sleep(0.1) timeline.append(time.time()) cols = {time1: "0", time2: "1"} assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1) assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2) def test_auto_time_to_uuid1(self): key = "key" t = time.time() col = {t: "foo"} self.cf_time.insert(key, col) uuid_res = self.cf_time.get(key).keys()[0] timestamp = convert_uuid_to_time(uuid_res) assert_almost_equal(timestamp, t, places=3)
class CassandraImporter: def __init__(self): parser = argparse.ArgumentParser(description="Process some integers.") parser.add_argument( "-s", "--source", help="Generally the prod cassandra path, list of machines: \ localhost:9162 localhost:9163", nargs="*", required=True, ) parser.add_argument( "-d", "--destination", help="Cassandra path where you need your data: \ localhost:9160 localhost:9161", nargs="*", required=True, ) parser.add_argument("-ks", "--keyspace", help="The keyspace: myks", required=True) parser.add_argument("-cf", "--column_family", help="The Column family: mycf", required=True) parser.add_argument("-k", "--key", help="A specific key to be imported", required=False) parser.add_argument("-c", "--count", help="Total count of keys to be imported", required=False) parser.add_argument("-a", "--all", action="store_true", help="Get all. Not recommended!", required=False) args = vars(parser.parse_args()) """Connection setting with cassandra The script is meant to sync data. So source and destination KS and CF shold be the same.""" try: source_pool = ConnectionPool(args["keyspace"], args["source"]) destination_pool = ConnectionPool(args["keyspace"], args["destination"]) self.source_cf = ColumnFamily(source_pool, args["column_family"]) self.source_cf.autopack_names = False self.source_cf.autopack_values = False self.source_cf.autopack_keys = False self.source_cf.default_validation_class = pycassa.types.UTF8Type() self.destination_cf = ColumnFamily(destination_pool, args["column_family"]) self.destination_cf.autopack_names = False self.destination_cf.autopack_values = False self.destination_cf.autopack_keys = False self.destination_cf.default_validation_class = pycassa.types.UTF8Type() except Exception as e: print "ERROR: The keyspace or the column family does not exist or request is timing out!" sys.exit() # Optional data self.count = args["count"] if self.count: self.count = int(self.count) self.key = args["key"] self.all = args["all"] def importData(self): data = dict() # Get columns for a key if self.key: column_data = self.source_cf.get(self.key) data[self.key] = column_data # Get last x keys and their columns elif self.count: counter = 0 error_count = 0 for value in self.source_cf.get_range(column_count=0, filter_empty=False): if counter < self.count: try: column_data = self.source_cf.get(value[0], column_count=100) data[value[0]] = column_data counter += 1 except NotFoundException: # Ignore keys with empty columns pass except Exception: error_count += 1 if error_count > 10: # Write the read data self.insertData(data) print "ERROR: Remote cassandra is too slow to read, exiting after writing..." sys.exit() # Use this to throttle reads from cassandra time.sleep(0.2) else: break # Get All, Not recommended elif self.all: for value in self.source_cf.get_range(column_count=0, filter_empty=False): column_data = self.source_cf.get(value[0]) key = value[0] data[key] = column_data else: print "Please pass -c or -k or -a arguments!" return data def insertData(self, data): print "Writing " + str(len(data.keys())) + " keys" for key, value in data.iteritems(): self.destination_cf.insert(key, value) def run(self): self.update_progress(0) data = self.importData() self.update_progress(50) self.insertData(data) self.update_progress(100) print "Import complete!" def update_progress(self, progress): print "\r[{0}] {1}%".format("#" * (progress / 10), progress)
class TestAutoPacking: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect_thread_local('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2') self.cf_long = ColumnFamily(self.client, 'StdLong') self.cf_int = ColumnFamily(self.client, 'StdInteger') self.cf_time = ColumnFamily(self.client, 'StdTimeUUID') self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID') self.cf_ascii = ColumnFamily(self.client, 'StdAscii') self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8') self.cf_bytes = ColumnFamily(self.client, 'StdBytes') self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True) self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True) self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True) self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True) self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True) self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True) self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True) self.cf_suplong_sublong = ColumnFamily(self.client, 'SuperLongSubLong', super=True) self.cf_suplong_subint = ColumnFamily(self.client, 'SuperLongSubInt', super=True) self.cf_suplong_subtime = ColumnFamily(self.client, 'SuperLongSubTime', super=True) self.cf_suplong_sublex = ColumnFamily(self.client, 'SuperLongSubLex', super=True) self.cf_suplong_subascii = ColumnFamily(self.client, 'SuperLongSubAscii', super=True) self.cf_suplong_subutf8 = ColumnFamily(self.client, 'SuperLongSubUTF8', super=True) self.cf_suplong_subbytes = ColumnFamily(self.client, 'SuperLongSubBytes', super=True) self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong') self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt') self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime') self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex') self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii') self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8') self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes') self.cfs = [self.cf_long, self.cf_int, self.cf_time, self.cf_lex, self.cf_ascii, self.cf_utf8, self.cf_bytes, self.cf_suplong, self.cf_supint, self.cf_suptime, self.cf_suplex, self.cf_supascii, self.cf_suputf8, self.cf_supbytes, self.cf_suplong_subint, self.cf_suplong_subint, self.cf_suplong_subtime, self.cf_suplong_sublex, self.cf_suplong_subascii, self.cf_suplong_subutf8, self.cf_suplong_subbytes, self.cf_valid_long, self.cf_valid_int, self.cf_valid_time, self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8, self.cf_valid_bytes] try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) for cf in self.cfs: for key, columns in cf.get_range(): cf.remove(key) def test_basic_inserts(self): long_col = {1111111111111111L: VALS[0]} int_col = {1: VALS[0]} time_col = {TIME1: VALS[0]} lex_col = {uuid.UUID(bytes='abc abc abc abcd'): VALS[0]} ascii_col = {'foo': VALS[0]} utf8_col = {u'\u0020': VALS[0]} bytes_col = {'bytes': VALS[0]} self.cf_long.insert(KEYS[0], long_col) self.cf_int.insert(KEYS[0], int_col) self.cf_time.insert(KEYS[0], time_col) self.cf_lex.insert(KEYS[0], lex_col) self.cf_ascii.insert(KEYS[0], ascii_col) self.cf_utf8.insert(KEYS[0], utf8_col) self.cf_bytes.insert(KEYS[0], bytes_col) assert self.cf_long.get(KEYS[0]) == long_col assert self.cf_int.get(KEYS[0]) == int_col assert self.cf_time.get(KEYS[0]) == time_col assert self.cf_lex.get(KEYS[0]) == lex_col assert self.cf_ascii.get(KEYS[0]) == ascii_col assert self.cf_utf8.get(KEYS[0]) == utf8_col assert self.cf_bytes.get(KEYS[0]) == bytes_col self.cf_suplong.insert(KEYS[0], {123L: bytes_col})
class Course: def __init__(self): self.cache = KeyValueDatabase() client = connect(cassandra_hosts) self.u = ColumnFamily(client, cassandra_keyspace, 'Users', super=True) self.error = Error() def refresh(self): self.cache.flush() def _levels(self, lang='en'): key = 'levels_%s' % (lang) results = self.cache.get(key) if results: return results try: parent_folder = unicode(os.path.join(data_folder, lang)) _levels = os.listdir(parent_folder) if _levels == []: return self.error.not_found("Chưa có dữ liệu") results = [] # get only folder names for level in _levels: if os.path.isdir(os.path.join(parent_folder, level)): results.append(level) results = sorted(results) self.cache.set(key, results) return results except OSError: return 404 def _lessons(self, level, lang='en'): key = 'lessons_%s_%s' % (level, lang) results = self.cache.get(key) if results: return results try: parent_folder = unicode(os.path.join(data_folder, lang)) parent_folder = os.path.join(parent_folder, level) _lessons = os.listdir(parent_folder) if _lessons == []: return self.error.not_found("Chưa có dữ liệu") results = [] for lesson in _lessons: if os.path.isdir(os.path.join(parent_folder, lesson)): results.append(lesson) results = sorted(results) self.cache.set(key, results) return results except OSError: return 404 def _listen(self, level, lesson, lang='en'): key = 'listen_%s_%s_%s' % (level, lesson, lang) results = self.cache.get(key) if results: return results try: parent_folder = unicode(os.path.join(data_folder, lang, level, lesson)) audio = os.listdir(parent_folder) if audio == []: return self.error.not_found("Chưa có dữ liệu") results = [] for i in audio: title = None description = None if i.endswith('.inf'): # add '\t' to detect end string when replace i = i + '\t' amr = os.path.join(parent_folder, i.replace('.inf\t', '.amr')) if not os.path.exists(amr): # check amr file exist return 404 i = i.replace('\t', '') # restore i # file inf chứa 2 biến là title và description cmd = open(os.path.join(parent_folder, i)) exec(cmd) # thực hiện chuỗi như là 1 lệnh info = {'title': title, 'description': description, 'audio_file': amr.replace(data_folder, '')} results.append(info) results = sorted(results) self.cache.set(key, results) return results except OSError: return 404 def levels(self, environ): try: lang = environ['request']['lang'] if lang not in ('en', 'vi'): return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi") session_id = environ['request']['session_id'] except KeyError: return self.error.authen_error() try: self.u.get('session_id', super_column=session_id) except (NotFoundException, InvalidRequestException): return self.error.authen_error() results = self._levels(lang) if results == 404: return self.error.not_found("Cấu hình hệ thống sai") xml = ['<level name="%s"/>' % result for result in results] return '\n\t'.join(xml) def lessons(self, environ): try: level = environ['request']['level'] lang = environ['request']['lang'] if lang not in ('en', 'vi'): return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi") session_id = environ['request']['session_id'] except KeyError: return self.error.authen_error() try: self.u.get('session_id', super_column=session_id) except (NotFoundException, InvalidRequestException): return self.error.authen_error() results = self._lessons(level, lang) if results == 404: return self.error.not_found("Cấu hình hệ thống sai") xml = ['<lesson name="%s"/>' % result for result in results] return '\n\t'.join(xml) def listen(self, environ): try: level = environ['request']['level'] lang = environ['request']['lang'] if lang not in ('en', 'vi'): return self.error.param_error("Hệ thống chỉ hỗ trợ 2 ngôn ngữ là en và vi") lesson = environ['request']['lesson'] session_id = environ['request']['session_id'] except KeyError: return self.error.authen_error() try: self.u.get('session_id', super_column=session_id) except (NotFoundException, InvalidRequestException): return self.error.authen_error() results = self._listen(level, lesson, lang) if results == 404: return self.error.not_found("Cấu hình hệ thống sai") xml = ['<audio title="%s" description="%s" link="%s"/>' \ % (x['title'], x['description'], audio_url_prefix + x['audio_file']) \ for x in results] return '\n\t'.join(xml)
class TestAutoPacking: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect_thread_local('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2') self.cf_long = ColumnFamily(self.client, 'StdLong') self.cf_int = ColumnFamily(self.client, 'StdInteger') self.cf_time = ColumnFamily(self.client, 'StdTimeUUID') self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID') self.cf_ascii = ColumnFamily(self.client, 'StdAscii') self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8') self.cf_bytes = ColumnFamily(self.client, 'StdBytes') self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True) self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True) self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True) self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True) self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True) self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True) self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True) self.cf_suplong_sublong = ColumnFamily(self.client, 'SuperLongSubLong', super=True) self.cf_suplong_subint = ColumnFamily(self.client, 'SuperLongSubInt', super=True) self.cf_suplong_subtime = ColumnFamily(self.client, 'SuperLongSubTime', super=True) self.cf_suplong_sublex = ColumnFamily(self.client, 'SuperLongSubLex', super=True) self.cf_suplong_subascii = ColumnFamily(self.client, 'SuperLongSubAscii', super=True) self.cf_suplong_subutf8 = ColumnFamily(self.client, 'SuperLongSubUTF8', super=True) self.cf_suplong_subbytes = ColumnFamily(self.client, 'SuperLongSubBytes', super=True) self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong') self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt') self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime') self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex') self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii') self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8') self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes') self.cf_def_valid = ColumnFamily(self.client, 'DefaultValidator') self.cfs = [ self.cf_long, self.cf_int, self.cf_time, self.cf_lex, self.cf_ascii, self.cf_utf8, self.cf_bytes, # self.cf_suplong, self.cf_supint, self.cf_suptime, self.cf_suplex, self.cf_supascii, self.cf_suputf8, self.cf_supbytes, # self.cf_suplong_subint, self.cf_suplong_subint, self.cf_suplong_subtime, self.cf_suplong_sublex, self.cf_suplong_subascii, self.cf_suplong_subutf8, self.cf_suplong_subbytes, # self.cf_valid_long, self.cf_valid_int, self.cf_valid_time, self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8, self.cf_valid_bytes, # self.cf_def_valid, ] try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) for cf in self.cfs: for key, columns in cf.get_range(): cf.remove(key) def test_basic_inserts(self): long_col = {1111111111111111L: VALS[0]} int_col = {1: VALS[0]} time_col = {TIME1: VALS[0]} lex_col = {uuid.UUID(bytes='abc abc abc abcd'): VALS[0]} ascii_col = {'foo': VALS[0]} utf8_col = {u'\u0020': VALS[0]} bytes_col = {'bytes': VALS[0]} self.cf_long.insert(KEYS[0], long_col) self.cf_int.insert(KEYS[0], int_col) self.cf_time.insert(KEYS[0], time_col) self.cf_lex.insert(KEYS[0], lex_col) self.cf_ascii.insert(KEYS[0], ascii_col) self.cf_utf8.insert(KEYS[0], utf8_col) self.cf_bytes.insert(KEYS[0], bytes_col) assert self.cf_long.get(KEYS[0]) == long_col assert self.cf_int.get(KEYS[0]) == int_col assert self.cf_time.get(KEYS[0]) == time_col assert self.cf_lex.get(KEYS[0]) == lex_col assert self.cf_ascii.get(KEYS[0]) == ascii_col assert self.cf_utf8.get(KEYS[0]) == utf8_col assert self.cf_bytes.get(KEYS[0]) == bytes_col self.cf_suplong.insert(KEYS[0], {123L: bytes_col})
class CassandraImporter: def __init__(self): parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('-s', '--source', help='Generally the prod cassandra path, list of machines: \ localhost:9162 localhost:9163', nargs='*', required=True) parser.add_argument('-d', '--destination', help='Cassandra path where you need your data: \ localhost:9160 localhost:9161', nargs='*', required=True) parser.add_argument('-ks', '--keyspace', help='The keyspace: myks', required=True) parser.add_argument('-cf', '--column_family', help='The Column family: mycf', required=True) parser.add_argument('-k', '--key', help='A specific key to be imported', required=False) parser.add_argument('-c', '--count', help='Total count of keys to be imported', required=False) parser.add_argument('-a', '--all', action='store_true', help='Get all. Not recommended!', required=False) args = vars(parser.parse_args()) """Connection setting with cassandra The script is meant to sync data. So source and destination KS and CF shold be the same.""" try: source_pool = ConnectionPool(args["keyspace"], args["source"]) destination_pool = ConnectionPool(args["keyspace"], args["destination"]) self.source_cf = ColumnFamily(source_pool, args["column_family"]) self.source_cf.autopack_names = False self.source_cf.autopack_values = False self.source_cf.autopack_keys = False self.source_cf.default_validation_class = pycassa.types.UTF8Type() self.destination_cf = ColumnFamily(destination_pool, args["column_family"]) self.destination_cf.autopack_names = False self.destination_cf.autopack_values = False self.destination_cf.autopack_keys = False self.destination_cf.default_validation_class = pycassa.types.UTF8Type() except Exception as e: print "ERROR: The keyspace or the column family does not exist or request is timing out!" sys.exit() # Optional data self.count = args["count"] if self.count: self.count = int(self.count) self.key = args["key"] self.all = args["all"] def importData(self): data = dict() # Get columns for a key if self.key: column_data = self.source_cf.get(self.key) data[self.key] = column_data # Get last x keys and their columns elif self.count: counter = 0 error_count = 0 for value in self.source_cf.get_range(column_count=0, filter_empty=False): if(counter < self.count): try: column_data = self.source_cf.get(value[0], column_count=100) data[value[0]] = column_data counter += 1 except NotFoundException: #Ignore keys with empty columns pass except Exception: error_count += 1 if error_count > 10: # Write the read data self.insertData(data) print "ERROR: Remote cassandra is too slow to read, exiting after writing..." sys.exit() # Use this to throttle reads from cassandra time.sleep(0.2) else: break # Get All, Not recommended elif self.all: for value in self.source_cf.get_range(column_count=0, filter_empty=False): column_data = self.source_cf.get(value[0]) key = value[0] data[key] = column_data else: print "Please pass -c or -k or -a arguments!" return data def insertData(self, data): print "Writing " + str(len(data.keys())) + " keys" for key, value in data.iteritems(): self.destination_cf.insert(key, value) def run(self): self.update_progress(0) data = self.importData() self.update_progress(50) self.insertData(data) self.update_progress(100) print "Import complete!" def update_progress(self, progress): print '\r[{0}] {1}%'.format('#' * (progress / 10), progress)
class TestColumnFamilyMap: def setUp(self): self.client = connect() self.client.login('Keyspace1', { 'username': '******', 'password': '******' }) self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp) self.map = ColumnFamilyMap(TestUTF8, self.cf) self.empty_map = ColumnFamilyMap(TestEmpty, self.cf, raw_columns=True) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def instance(self, key): instance = TestUTF8() instance.key = key instance.strcol = '1' instance.intcol = 2 instance.floatcol = 3.5 instance.datetimecol = datetime.now().replace(microsecond=0) instance.intstrcol = 8 instance.floatstrcol = 4.6 instance.datetimestrcol = datetime.now().replace(microsecond=0) return instance def test_will_not_insert_none(self): for column in ('strcol', 'intcol', 'floatcol', 'datetimecol', 'intstrcol', 'floatstrcol', 'datetimestrcol'): instance = self.instance( 'TestColumnFamilyMap.test_will_not_insert_none') setattr(instance, column, None) assert_raises(TypeError, self.map.insert, instance) def test_empty(self): key = 'TestColumnFamilyMap.test_empty' assert_raises(NotFoundException, self.map.get, key) assert len(self.map.multiget([key])) == 0 def test_insert_get(self): instance = self.instance('TestColumnFamilyMap.test_insert_get') assert_raises(NotFoundException, self.map.get, instance.key) self.map.insert(instance) assert self.map.get(instance.key) == instance assert self.empty_map.get( instance.key).raw_columns['intstrcol'] == str(instance.intstrcol) def test_insert_multiget(self): instance1 = self.instance('TestColumnFamilyMap.test_insert_multiget1') instance2 = self.instance('TestColumnFamilyMap.test_insert_multiget2') missing_key = 'TestColumnFamilyMap.test_insert_multiget3' self.map.insert(instance1) self.map.insert(instance2) rows = self.map.multiget([instance1.key, instance2.key, missing_key]) assert len(rows) == 2 assert rows[instance1.key] == instance1 assert rows[instance2.key] == instance2 assert missing_key not in rows assert self.empty_map.multiget([ instance1.key ])[instance1.key].raw_columns['intstrcol'] == str(instance1.intstrcol) def test_insert_get_count(self): instance = self.instance('TestColumnFamilyMap.test_insert_get_count') self.map.insert(instance) assert self.map.get_count(instance.key) == 7 def test_insert_get_range(self): instances = [] for i in xrange(5): instance = self.instance( 'TestColumnFamilyMap.test_insert_get_range%s' % i) instances.append(instance) for instance in instances: self.map.insert(instance) rows = list( self.map.get_range(start=instances[0].key, finish=instances[-1].key)) assert len(rows) == len(instances) assert rows == instances assert list( self.empty_map.get_range( start=instances[0].key, finish=instances[0].key))[0].raw_columns['intstrcol'] == str( instances[0].intstrcol) def test_remove(self): instance = self.instance('TestColumnFamilyMap.test_remove') self.map.insert(instance) self.map.remove(instance) assert_raises(NotFoundException, self.map.get, instance.key) def test_does_not_insert_extra_column(self): instance = self.instance( 'TestColumnFamilyMap.test_does_not_insert_extra_column') instance.othercol = 'Test' self.map.insert(instance) get_instance = self.map.get(instance.key) assert get_instance.strcol == instance.strcol assert get_instance.intcol == instance.intcol assert get_instance.floatcol == instance.floatcol assert get_instance.datetimecol == instance.datetimecol assert_raises(AttributeError, getattr, get_instance, 'othercol') def test_has_defaults(self): key = 'TestColumnFamilyMap.test_has_defaults' self.cf.insert(key, {'strcol': '1'}) instance = self.map.get(key) assert instance.intcol == TestUTF8.intcol.default assert instance.floatcol == TestUTF8.floatcol.default assert instance.datetimecol == TestUTF8.datetimecol.default assert instance.intstrcol == TestUTF8.intstrcol.default assert instance.floatstrcol == TestUTF8.floatstrcol.default assert instance.datetimestrcol == TestUTF8.datetimestrcol.default
class TestColumnFamily: def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2', write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, dict_class=TestDict) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def test_empty(self): key = 'TestColumnFamily.test_empty' assert_raises(NotFoundException, self.cf.get, key) assert len(self.cf.multiget([key])) == 0 for key, columns in self.cf.get_range(): assert len(columns) == 0 def test_insert_get(self): key = 'TestColumnFamily.test_insert_get' columns = {'1': 'val1', '2': 'val2'} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert self.cf.get(key) == columns def test_insert_multiget(self): key1 = 'TestColumnFamily.test_insert_multiget1' columns1 = {'1': 'val1', '2': 'val2'} key2 = 'test_insert_multiget1' columns2 = {'3': 'val1', '4': 'val2'} missing_key = 'key3' self.cf.insert(key1, columns1) self.cf.insert(key2, columns2) rows = self.cf.multiget([key1, key2, missing_key]) assert len(rows) == 2 assert rows[key1] == columns1 assert rows[key2] == columns2 assert missing_key not in rows def test_insert_get_count(self): key = 'TestColumnFamily.test_insert_get_count' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) assert self.cf.get_count(key) == 2 assert_equal(self.cf.get_count(key, column_start='1'), 2) assert_equal(self.cf.get_count(key, column_finish='2'), 2) assert_equal( self.cf.get_count(key, column_start='1', column_finish='2'), 2) assert_equal( self.cf.get_count(key, column_start='1', column_finish='1'), 1) assert_equal(self.cf.get_count(key, columns=['1', '2']), 2) assert_equal(self.cf.get_count(key, columns=['1']), 1) def test_insert_multiget_count(self): keys = [ 'TestColumnFamily.test_insert_multiget_count1', 'TestColumnFamily.test_insert_multiget_count2', 'TestColumnFamily.test_insert_multiget_count3' ] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) result = self.cf.multiget_count(keys) assert_equal(result[keys[0]], 2) assert_equal(result[keys[1]], 2) assert_equal(result[keys[2]], 2) result = self.cf.multiget_count(keys, column_start='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) result = self.cf.multiget_count(keys, columns=['1', '2']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, columns=['1']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) def test_insert_get_range(self): keys = [ 'TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5) ] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) rows = list(self.cf.get_range(start=keys[0], finish=keys[-1])) assert len(rows) == len(keys) for i, (k, c) in enumerate(rows): assert k == keys[i] assert c == columns def test_insert_get_indexed_slices(self): indexed_cf = ColumnFamily(self.client, 'Indexed1') columns = {'birthdate': 1L} key = 'key1' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) key = 'key2' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) key = 'key3' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr]) result = indexed_cf.get_indexed_slices(clause) assert len(result) == 3 assert result.get('key1') == columns assert result.get('key2') == columns assert result.get('key3') == columns def test_remove(self): key = 'TestColumnFamily.test_remove' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) self.cf.remove(key, columns=['2']) del columns['2'] assert self.cf.get(key) == {'1': 'val1'} self.cf.remove(key) assert_raises(NotFoundException, self.cf.get, key) def test_dict_class(self): key = 'TestColumnFamily.test_dict_class' self.cf.insert(key, {'1': 'val1'}) assert isinstance(self.cf.get(key), TestDict)
class TestTimeUUIDs(unittest.TestCase): def setUp(self): self.cf_time = ColumnFamily(pool, 'StdTimeUUID') def tearDown(self): self.cf_time.remove('key1') def test_datetime_to_uuid(self): key = 'key1' timeline = [] timeline.append(datetime.now()) time1 = uuid1() col1 = {time1:'0'} self.cf_time.insert(key, col1) time.sleep(1) timeline.append(datetime.now()) time2 = uuid1() col2 = {time2:'1'} self.cf_time.insert(key, col2) time.sleep(1) timeline.append(datetime.now()) cols = {time1:'0', time2:'1'} assert_equal(self.cf_time.get(key, column_start=timeline[0]) , cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]) , col1) assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]) , col2) def test_time_to_uuid(self): key = 'key1' timeline = [] timeline.append(time.time()) time1 = uuid1() col1 = {time1:'0'} self.cf_time.insert(key, col1) time.sleep(0.1) timeline.append(time.time()) time2 = uuid1() col2 = {time2:'1'} self.cf_time.insert(key, col2) time.sleep(0.1) timeline.append(time.time()) cols = {time1:'0', time2:'1'} assert_equal(self.cf_time.get(key, column_start=timeline[0]) , cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]) , cols) assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]) , col1) assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]) , col2) def test_auto_time_to_uuid1(self): key = 'key1' t = time.time() col = {t: 'foo'} self.cf_time.insert(key, col) uuid_res = self.cf_time.get(key).keys()[0] timestamp = convert_uuid_to_time(uuid_res) assert_almost_equal(timestamp, t, places=3)
class TestColumnFamily: def setUp(self): self.client = connect() self.client.login('Keyspace1', {'username': '******', 'password': '******'}) self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2', write_consistency_level=ConsistencyLevel.ONE, buffer_size=2, timestamp=self.timestamp, dict_class=TestDict) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def test_empty(self): key = 'TestColumnFamily.test_empty' assert_raises(NotFoundException, self.cf.get, key) assert len(self.cf.multiget([key])) == 0 for key, columns in self.cf.get_range(): assert len(columns) == 0 def test_insert_get(self): key = 'TestColumnFamily.test_insert_get' columns = {'1': 'val1', '2': 'val2'} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert self.cf.get(key) == columns def test_insert_multiget(self): key1 = 'TestColumnFamily.test_insert_multiget1' columns1 = {'1': 'val1', '2': 'val2'} key2 = 'test_insert_multiget1' columns2 = {'3': 'val1', '4': 'val2'} missing_key = 'key3' self.cf.insert(key1, columns1) self.cf.insert(key2, columns2) rows = self.cf.multiget([key1, key2, missing_key]) assert len(rows) == 2 assert rows[key1] == columns1 assert rows[key2] == columns2 assert missing_key not in rows def test_insert_get_count(self): key = 'TestColumnFamily.test_insert_get_count' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) assert self.cf.get_count(key) == 2 def test_insert_get_range(self): keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) rows = list(self.cf.get_range(start=keys[0], finish=keys[-1])) assert len(rows) == len(keys) for i, (k, c) in enumerate(rows): assert k == keys[i] assert c == columns def test_remove(self): key = 'TestColumnFamily.test_remove' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) self.cf.remove(key, columns=['2']) del columns['2'] assert self.cf.get(key) == {'1': 'val1'} self.cf.remove(key) assert_raises(NotFoundException, self.cf.get, key) def test_dict_class(self): key = 'TestColumnFamily.test_dict_class' self.cf.insert(key, {'1': 'val1'}) assert isinstance(self.cf.get(key), TestDict)
class CassandraCache(CacheUtils): permanent = True """A cache that uses a Cassandra ColumnFamily. Uses only the column-name 'value'""" def __init__(self, column_family, client, read_consistency_level = CL_ONE, write_consistency_level = CL_QUORUM): self.column_family = column_family self.client = client self.read_consistency_level = read_consistency_level self.write_consistency_level = write_consistency_level self.cf = ColumnFamily(self.client, self.column_family, read_consistency_level = read_consistency_level, write_consistency_level = write_consistency_level) def _rcl(self, alternative): return (alternative if alternative is not None else self.cf.read_consistency_level) def _wcl(self, alternative): return (alternative if alternative is not None else self.cf.write_consistency_level) def get(self, key, default = None, read_consistency_level = None): try: rcl = self._rcl(read_consistency_level) row = self.cf.get(key, columns=['value'], read_consistency_level = rcl) return pickle.loads(row['value']) except (CassandraNotFound, KeyError): return default def simple_get_multi(self, keys, read_consistency_level = None): rcl = self._rcl(read_consistency_level) rows = self.cf.multiget(list(keys), columns=['value'], read_consistency_level = rcl) return dict((key, pickle.loads(row['value'])) for (key, row) in rows.iteritems()) def set(self, key, val, write_consistency_level = None, time = None): if val == NoneResult: # NoneResult caching is for other parts of the chain return wcl = self._wcl(write_consistency_level) ret = self.cf.insert(key, {'value': pickle.dumps(val)}, write_consistency_level = wcl, ttl = time) self._warm([key]) return ret def set_multi(self, keys, prefix='', write_consistency_level = None, time = None): if not isinstance(keys, dict): # allow iterables yielding tuples keys = dict(keys) wcl = self._wcl(write_consistency_level) ret = {} with self.cf.batch(write_consistency_level = wcl): for key, val in keys.iteritems(): if val != NoneResult: ret[key] = self.cf.insert('%s%s' % (prefix, key), {'value': pickle.dumps(val)}, ttl = time or None) self._warm(keys.keys()) return ret def _warm(self, keys): import random if False and random.random() > 0.98: print 'Warming', keys self.cf.multiget(keys) def delete(self, key, write_consistency_level = None): wcl = self._wcl(write_consistency_level) self.cf.remove(key, write_consistency_level = wcl)
class TestColumnFamilyMap: def setUp(self): self.client = connect() self.client.login('Keyspace1', {'username': '******', 'password': '******'}) self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp) self.map = ColumnFamilyMap(TestUTF8, self.cf) self.empty_map = ColumnFamilyMap(TestEmpty, self.cf, raw_columns=True) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear() def tearDown(self): self.cf.insert('meta', {'timestamp': str(self.timestamp_n)}) # Since the timestamp passed to Cassandra will be in the same second # with the default timestamp function, causing problems with removing # and inserting (Cassandra doesn't know which is later), we supply our own def timestamp(self): self.timestamp_n += 1 return self.timestamp_n def clear(self): for key, columns in self.cf.get_range(include_timestamp=True): for value, timestamp in columns.itervalues(): self.timestamp_n = max(self.timestamp_n, timestamp) self.cf.remove(key) def instance(self, key): instance = TestUTF8() instance.key = key instance.strcol = '1' instance.intcol = 2 instance.floatcol = 3.5 instance.datetimecol = datetime.now().replace(microsecond=0) instance.intstrcol = 8 instance.floatstrcol = 4.6 instance.datetimestrcol = datetime.now().replace(microsecond=0) return instance def test_will_not_insert_none(self): for column in ('strcol', 'intcol', 'floatcol', 'datetimecol', 'intstrcol', 'floatstrcol', 'datetimestrcol'): instance = self.instance('TestColumnFamilyMap.test_will_not_insert_none') setattr(instance, column, None) assert_raises(TypeError, self.map.insert, instance) def test_empty(self): key = 'TestColumnFamilyMap.test_empty' assert_raises(NotFoundException, self.map.get, key) assert len(self.map.multiget([key])) == 0 def test_insert_get(self): instance = self.instance('TestColumnFamilyMap.test_insert_get') assert_raises(NotFoundException, self.map.get, instance.key) self.map.insert(instance) assert self.map.get(instance.key) == instance assert self.empty_map.get(instance.key).raw_columns['intstrcol'] == str(instance.intstrcol) def test_insert_multiget(self): instance1 = self.instance('TestColumnFamilyMap.test_insert_multiget1') instance2 = self.instance('TestColumnFamilyMap.test_insert_multiget2') missing_key = 'TestColumnFamilyMap.test_insert_multiget3' self.map.insert(instance1) self.map.insert(instance2) rows = self.map.multiget([instance1.key, instance2.key, missing_key]) assert len(rows) == 2 assert rows[instance1.key] == instance1 assert rows[instance2.key] == instance2 assert missing_key not in rows assert self.empty_map.multiget([instance1.key])[instance1.key].raw_columns['intstrcol'] == str(instance1.intstrcol) def test_insert_get_count(self): instance = self.instance('TestColumnFamilyMap.test_insert_get_count') self.map.insert(instance) assert self.map.get_count(instance.key) == 7 def test_insert_get_range(self): instances = [] for i in xrange(5): instance = self.instance('TestColumnFamilyMap.test_insert_get_range%s' % i) instances.append(instance) for instance in instances: self.map.insert(instance) rows = list(self.map.get_range(start=instances[0].key, finish=instances[-1].key)) assert len(rows) == len(instances) assert rows == instances assert list(self.empty_map.get_range(start=instances[0].key, finish=instances[0].key))[0].raw_columns['intstrcol'] == str(instances[0].intstrcol) def test_remove(self): instance = self.instance('TestColumnFamilyMap.test_remove') self.map.insert(instance) self.map.remove(instance) assert_raises(NotFoundException, self.map.get, instance.key) def test_does_not_insert_extra_column(self): instance = self.instance('TestColumnFamilyMap.test_does_not_insert_extra_column') instance.othercol = 'Test' self.map.insert(instance) get_instance = self.map.get(instance.key) assert get_instance.strcol == instance.strcol assert get_instance.intcol == instance.intcol assert get_instance.floatcol == instance.floatcol assert get_instance.datetimecol == instance.datetimecol assert_raises(AttributeError, getattr, get_instance, 'othercol') def test_has_defaults(self): key = 'TestColumnFamilyMap.test_has_defaults' self.cf.insert(key, {'strcol': '1'}) instance = self.map.get(key) assert instance.intcol == TestUTF8.intcol.default assert instance.floatcol == TestUTF8.floatcol.default assert instance.datetimecol == TestUTF8.datetimecol.default assert instance.intstrcol == TestUTF8.intstrcol.default assert instance.floatstrcol == TestUTF8.floatstrcol.default assert instance.datetimestrcol == TestUTF8.datetimestrcol.default
class CassandraCache(CacheUtils): permanent = True """A cache that uses a Cassandra ColumnFamily. Uses only the column-name 'value'""" def __init__(self, column_family, client, read_consistency_level=CL_ONE, write_consistency_level=CL_QUORUM): self.column_family = column_family self.client = client self.read_consistency_level = read_consistency_level self.write_consistency_level = write_consistency_level self.cf = ColumnFamily(self.client, self.column_family, read_consistency_level=read_consistency_level, write_consistency_level=write_consistency_level) def _rcl(self, alternative): return (alternative if alternative is not None else self.cf.read_consistency_level) def _wcl(self, alternative): return (alternative if alternative is not None else self.cf.write_consistency_level) def get(self, key, default=None, read_consistency_level=None): try: rcl = self._rcl(read_consistency_level) row = self.cf.get(key, columns=['value'], read_consistency_level=rcl) return pickle.loads(row['value']) except (CassandraNotFound, KeyError): return default def simple_get_multi(self, keys, read_consistency_level=None): rcl = self._rcl(read_consistency_level) rows = self.cf.multiget(list(keys), columns=['value'], read_consistency_level=rcl) return dict((key, pickle.loads(row['value'])) for (key, row) in rows.iteritems()) def set(self, key, val, write_consistency_level=None, time=None): if val == NoneResult: # NoneResult caching is for other parts of the chain return wcl = self._wcl(write_consistency_level) ret = self.cf.insert(key, {'value': pickle.dumps(val)}, write_consistency_level=wcl, ttl=time) self._warm([key]) return ret def set_multi(self, keys, prefix='', write_consistency_level=None, time=None): if not isinstance(keys, dict): # allow iterables yielding tuples keys = dict(keys) wcl = self._wcl(write_consistency_level) ret = {} with self.cf.batch(write_consistency_level=wcl): for key, val in keys.iteritems(): if val != NoneResult: ret[key] = self.cf.insert('%s%s' % (prefix, key), {'value': pickle.dumps(val)}, ttl=time) self._warm(keys.keys()) return ret def _warm(self, keys): import random if False and random.random() > 0.98: print 'Warming', keys self.cf.multiget(keys) def delete(self, key, write_consistency_level=None): wcl = self._wcl(write_consistency_level) self.cf.remove(key, write_consistency_level=wcl)
class TestMutator(unittest.TestCase): def setUp(self): credentials = {"username": "******", "password": "******"} self.pool = ConnectionPool(keyspace="Keyspace1", credentials=credentials) self.cf = ColumnFamily(self.pool, "Standard2") self.scf = ColumnFamily(self.pool, "Super1") def tearDown(self): for key, cols in self.cf.get_range(): self.cf.remove(key) for key, cols in self.scf.get_range(): self.scf.remove(key) def test_insert(self): batch = self.cf.batch() for key, cols in ROWS.iteritems(): batch.insert(key, cols) batch.send() for key, cols in ROWS.items(): assert self.cf.get(key) == cols def test_insert_supercolumns(self): batch = self.scf.batch() batch.insert("one", ROWS) batch.insert("two", ROWS) batch.insert("three", ROWS) batch.send() assert self.scf.get("one") == ROWS assert self.scf.get("two") == ROWS assert self.scf.get("three") == ROWS def test_queue_size(self): batch = self.cf.batch(queue_size=2) batch.insert("1", ROWS["1"]) batch.insert("2", ROWS["2"]) batch.insert("3", ROWS["3"]) assert self.cf.get("1") == ROWS["1"] assert_raises(NotFoundException, self.cf.get, "3") batch.send() for key, cols in ROWS.items(): assert self.cf.get(key) == cols def test_remove_key(self): batch = self.cf.batch() batch.insert("1", ROWS["1"]) batch.remove("1") batch.send() assert_raises(NotFoundException, self.cf.get, "1") def test_remove_columns(self): batch = self.cf.batch() batch.insert("1", {"a": "123", "b": "123"}) batch.remove("1", ["a"]) batch.send() assert self.cf.get("1") == {"b": "123"} def test_remove_supercolumns(self): batch = self.scf.batch() batch.insert("one", ROWS) batch.insert("two", ROWS) batch.insert("three", ROWS) batch.remove("two", ["b"], "2") batch.send() assert self.scf.get("one") == ROWS assert self.scf.get("two")["2"] == {"a": "234"} assert self.scf.get("three") == ROWS def test_chained(self): batch = self.cf.batch() batch.insert("1", ROWS["1"]).insert("2", ROWS["2"]).insert("3", ROWS["3"]).send() assert self.cf.get("1") == ROWS["1"] assert self.cf.get("2") == ROWS["2"] assert self.cf.get("3") == ROWS["3"] def test_contextmgr(self): if sys.version_info < (2, 5): raise SkipTest("No context managers in Python < 2.5") exec """with self.cf.batch(queue_size=2) as b: b.insert('1', ROWS['1']) b.insert('2', ROWS['2']) b.insert('3', ROWS['3']) assert self.cf.get('3') == ROWS['3']""" def test_multi_column_family(self): batch = batch_mod.Mutator(self.pool) cf2 = self.cf batch.insert(self.cf, "1", ROWS["1"]) batch.insert(self.cf, "2", ROWS["2"]) batch.remove(cf2, "1", ROWS["1"]) batch.send() assert self.cf.get("2") == ROWS["2"] assert_raises(NotFoundException, self.cf.get, "1")
class TestTimeUUIDs(unittest.TestCase): def setUp(self): self.cf_time = ColumnFamily(pool, 'StdTimeUUID') def tearDown(self): self.cf_time.remove('key1') def test_datetime_to_uuid(self): key = 'key1' timeline = [] timeline.append(datetime.now()) time1 = uuid1() col1 = {time1: '0'} self.cf_time.insert(key, col1) time.sleep(1) timeline.append(datetime.now()) time2 = uuid1() col2 = {time2: '1'} self.cf_time.insert(key, col2) time.sleep(1) timeline.append(datetime.now()) cols = {time1: '0', time2: '1'} assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1) assert_equal( self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2) def test_time_to_uuid(self): key = 'key1' timeline = [] timeline.append(time.time()) time1 = uuid1() col1 = {time1: '0'} self.cf_time.insert(key, col1) time.sleep(0.1) timeline.append(time.time()) time2 = uuid1() col2 = {time2: '1'} self.cf_time.insert(key, col2) time.sleep(0.1) timeline.append(time.time()) cols = {time1: '0', time2: '1'} assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols) assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols) assert_equal( self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1) assert_equal( self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2) def test_auto_time_to_uuid1(self): key = 'key1' t = time.time() col = {t: 'foo'} self.cf_time.insert(key, col) uuid_res = self.cf_time.get(key).keys()[0] timestamp = convert_uuid_to_time(uuid_res) assert_almost_equal(timestamp, t, places=3)
class TestColumnFamily(unittest.TestCase): def setUp(self): credentials = {'username': '******', 'password': '******'} self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.pool, 'Standard2', dict_class=TestDict) def tearDown(self): for key, columns in self.cf.get_range(): self.cf.remove(key) def test_empty(self): key = 'TestColumnFamily.test_empty' assert_raises(NotFoundException, self.cf.get, key) assert_equal(len(self.cf.multiget([key])), 0) for key, columns in self.cf.get_range(): assert_equal(len(columns), 0) def test_insert_get(self): key = 'TestColumnFamily.test_insert_get' columns = {'1': 'val1', '2': 'val2'} assert_raises(NotFoundException, self.cf.get, key) self.cf.insert(key, columns) assert_equal(self.cf.get(key), columns) def test_insert_multiget(self): key1 = 'TestColumnFamily.test_insert_multiget1' columns1 = {'1': 'val1', '2': 'val2'} key2 = 'test_insert_multiget1' columns2 = {'3': 'val1', '4': 'val2'} missing_key = 'key3' self.cf.insert(key1, columns1) self.cf.insert(key2, columns2) rows = self.cf.multiget([key1, key2, missing_key]) assert_equal(len(rows), 2) assert_equal(rows[key1], columns1) assert_equal(rows[key2], columns2) assert_true(missing_key not in rows) def test_insert_get_count(self): key = 'TestColumnFamily.test_insert_get_count' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) assert_equal(self.cf.get_count(key), 2) assert_equal(self.cf.get_count(key, column_start='1'), 2) assert_equal(self.cf.get_count(key, column_finish='2'), 2) assert_equal(self.cf.get_count(key, column_start='1', column_finish='2'), 2) assert_equal(self.cf.get_count(key, column_start='1', column_finish='1'), 1) assert_equal(self.cf.get_count(key, columns=['1','2']), 2) assert_equal(self.cf.get_count(key, columns=['1']), 1) def test_insert_multiget_count(self): keys = ['TestColumnFamily.test_insert_multiget_count1', 'TestColumnFamily.test_insert_multiget_count2', 'TestColumnFamily.test_insert_multiget_count3'] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) result = self.cf.multiget_count(keys) assert_equal(result[keys[0]], 2) assert_equal(result[keys[1]], 2) assert_equal(result[keys[2]], 2) result = self.cf.multiget_count(keys, column_start='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='2') assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, column_start='1', column_finish='1') assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) result = self.cf.multiget_count(keys, columns=['1','2']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 2) result = self.cf.multiget_count(keys, columns=['1']) assert_equal(len(result), 3) assert_equal(result[keys[0]], 1) def test_insert_get_range(self): keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)] columns = {'1': 'val1', '2': 'val2'} for key in keys: self.cf.insert(key, columns) rows = list(self.cf.get_range(start=keys[0], finish=keys[-1])) assert_equal(len(rows), len(keys)) for i, (k, c) in enumerate(rows): assert_equal(k, keys[i]) assert_equal(c, columns) def test_get_range_batching(self): self.cf.truncate() keys = [] columns = {'c': 'v'} for i in range(100, 201): keys.append('key%d' % i) self.cf.insert('key%d' % i, columns) for i in range(201, 301): self.cf.insert('key%d' % i, columns) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=10): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=1000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=150): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=7): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) count = 0 for (k,v) in self.cf.get_range(row_count=100, buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 100) # Put the remaining keys in our list for i in range(201, 301): keys.append('key%d' % i) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=7): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=200): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(row_count=10000, buffer_size=10000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) # Don't give a row count count = 0 for (k,v) in self.cf.get_range(buffer_size=2): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=77): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=200): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) count = 0 for (k,v) in self.cf.get_range(buffer_size=10000): assert_true(k in keys, 'key "%s" should be in keys' % k) count += 1 assert_equal(count, 201) self.cf.truncate() def insert_insert_get_indexed_slices(self): indexed_cf = ColumnFamily(self.pool, 'Indexed1') columns = {'birthdate': 1L} keys = [] for i in range(1,4): indexed_cf.insert('key%d' % i, columns) keys.append('key%d') expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr]) count = 0 for key,cols in indexed_cf.get_indexed_slices(clause): assert_equal(cols, columns) assert key in keys count += 1 assert_equal(count, 3) def test_get_indexed_slices_batching(self): indexed_cf = ColumnFamily(self.pool, 'Indexed1') columns = {'birthdate': 1L} for i in range(200): indexed_cf.insert('key%d' % i, columns) expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr], count=10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 10) clause = index.create_index_clause([expr], count=250) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 200) def test_remove(self): key = 'TestColumnFamily.test_remove' columns = {'1': 'val1', '2': 'val2'} self.cf.insert(key, columns) self.cf.remove(key, columns=['2']) del columns['2'] assert_equal(self.cf.get(key), {'1': 'val1'}) self.cf.remove(key) assert_raises(NotFoundException, self.cf.get, key) def test_dict_class(self): key = 'TestColumnFamily.test_dict_class' self.cf.insert(key, {'1': 'val1'}) assert isinstance(self.cf.get(key), TestDict)