def test_queue_pool_recycle(self): listener = _TestListener() pool = QueuePool(pool_size=5, max_overflow=5, recycle=1, prefill=True, pool_timeout=0.5, timeout=1, keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False) conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(10): cf.insert('key', {'col': 'val'}) conn.return_to_pool() assert_equal(listener.recycle_count, 1) pool.dispose() listener.reset() # Try with threadlocal=True pool = QueuePool(pool_size=5, max_overflow=5, recycle=10, prefill=True, pool_timeout=0.5, timeout=1, keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=True) conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(10): cf.insert('key', {'col': 'val'}) conn.return_to_pool() assert_equal(listener.recycle_count, 1)
def test_queue_threadlocal_failover(self): stats_logger = StatsLoggerWithListStorage() pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1, 5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(stats_logger.stats['failed'], i) assert_equal(cf.get('key'), {'col': 'val%d' % i, 'col2': 'val'}) pool.dispose() stats_logger.reset() pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=10000, prefill=True, timeout=0.05, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() threads = [] args = ('key', {'col': 'val', 'col2': 'val'}) for i in range(5): threads.append(threading.Thread(target=cf.insert, args=args)) threads[-1].start() for thread in threads: thread.join() assert_equal(stats_logger.stats['failed'], 5) pool.dispose()
def create_cfs(self): """ Creates the Cassandra Column Families (if not exist) """ sys_mgr = None pool = None try: sys_mgr = SystemManager() pool = ConnectionPool(settings.KEYSPACE, server_list=settings.CASSANDRA_HOSTS) for cf_name in [ CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY ]: try: cf = ColumnFamily(pool, cf_name) except: logger.info("create_cfs(): Creating column family %s", cf_name) sys_mgr.create_column_family( settings.KEYSPACE, cf_name, comparator_type=TimeUUIDType()) cf = ColumnFamily(pool, cf_name) cf.get_count(str(uuid.uuid4())) finally: if pool: pool.dispose() if sys_mgr: sys_mgr.close()
def test_queue_pool_recycle(self): listener = _TestListener() pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=1, prefill=True, pool_timeout=0.5, timeout=1, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=False) cf = ColumnFamily(pool, 'Standard1') columns = {'col1': 'val', 'col2': 'val'} for i in range(10): cf.insert('key', columns) assert_equal(listener.recycle_count, 5) pool.dispose() listener.reset() # Try with threadlocal=True pool = ConnectionPool(pool_size=5, max_overflow=5, recycle=1, prefill=False, pool_timeout=0.5, timeout=1, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=True) cf = ColumnFamily(pool, 'Standard1') for i in range(10): cf.insert('key', columns) pool.dispose() assert_equal(listener.recycle_count, 5)
def setup_module(): global pool, cf, scf credentials = {'username': '******', 'password': '******'} pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials) cf = ColumnFamily(pool, 'Standard1') scf = ColumnFamily(pool, 'Super1')
def setup_module(): global pool, cf, scf, indexed_cf, sys_man credentials = {'username': '******', 'password': '******'} pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials) cf = ColumnFamily(pool, 'Standard1', dict_class=TestDict) scf = ColumnFamily(pool, 'Super1', dict_class=dict) indexed_cf = ColumnFamily(pool, 'Indexed1') sys_man = SystemManager()
def setup_module(): global pool, cf, scf, counter_cf, super_counter_cf, sysman credentials = {'username': '******', 'password': '******'} pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials) cf = ColumnFamily(pool, 'Standard1') scf = ColumnFamily(pool, 'Super1') sysman = SystemManager() counter_cf = ColumnFamily(pool, 'Counter1') super_counter_cf = ColumnFamily(pool, 'SuperCounter1')
def __init__(cls, name, bases, dct): type.__init__(cls, name, bases, dct) if cls._use_db: if cls._type_prefix is None: # default to the class name cls._type_prefix = name if '_' in cls._type_prefix: raise TdbException("Cannot have _ in type prefix %r (for %r)" % (cls._type_prefix, name)) if cls._type_prefix in thing_types: raise InvariantException("Redefining type %r?" % (cls._type_prefix)) # if we weren't given a specific _cf_name, we can use the # classes's name cf_name = cls._cf_name or name thing_types[cls._type_prefix] = cls cls._read_consistency_level = read_consistency_level cls._write_consistency_level = write_consistency_level try: cls._cf = ColumnFamily( cassandra, cf_name, read_consistency_level=read_consistency_level, write_consistency_level=write_consistency_level) except NotFoundException: if not db_create_tables: raise manager = get_manager() log.warning("Creating Cassandra Column Family %s" % (cf_name, )) with make_lock('cassandra_schema'): manager.create_column_family( keyspace, cf_name, comparator_type=cls._compare_with) log.warning("Created Cassandra Column Family %s" % (cf_name, )) # try again to look it up cls._cf = ColumnFamily( cassandra, cf_name, read_consistency_level=read_consistency_level, write_consistency_level=write_consistency_level) cls._kind = name
def setup_module(): global pool, cf, scf, indexed_cf, counter_cf, counter_scf, sys_man, have_counters credentials = {'username': '******', 'password': '******'} pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials) cf = ColumnFamily(pool, 'Standard1', dict_class=TestDict) scf = ColumnFamily(pool, 'Super1', dict_class=dict) indexed_cf = ColumnFamily(pool, 'Indexed1') sys_man = SystemManager() have_counters = sys_man._conn.version != CASSANDRA_07 if have_counters: counter_cf = ColumnFamily(pool, 'Counter1') counter_scf = ColumnFamily(pool, 'SuperCounter1')
def setup_module(): global pool, cf, indexed_cf, pool_stub, indexed_cf_stub, cf_stub credentials = {'username': '******', 'password': '******'} pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials, timeout=1.0) cf = ColumnFamily(pool, 'Standard1', dict_class=TestDict) indexed_cf = ColumnFamily(pool, 'Indexed1') pool_stub = ConnectionPoolStub(keyspace='PycassaTestKeyspace', credentials=credentials, timeout=1.0) cf_stub = ColumnFamilyStub(pool_stub, 'Standard1', dict_class=TestDict) indexed_cf_stub = ColumnFamilyStub(pool_stub, 'Indexed1')
def test_null_pool_failover(self): listener = _TestListener() pool = NullPool(keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(1,5): setattr(cf.client._connection.client, 'batch_mutate', _timeout) # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val'}) assert_equal(listener.failure_count, i) cf.get('key') pool.dispose() listener.reset() pool = NullPool(keyspace='Keyspace1', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) threads = [] args = (pool, 'key', {'col':'val'}) for i in range(0, 5): threads.append(threading.Thread(target=_five_fails, args=args)) threads[-1].start() for thread in threads: thread.join() assert_equal(listener.failure_count, 25) pool.dispose()
def __call__(self, uuids=None, cassandra_servers=None, force=False, **kwargs): super(CheckBadRefs, self).__call__(**kwargs) self.force = force pool = ConnectionPool('config_db_uuid', server_list=cassandra_servers) uuid_cf = ColumnFamily(pool, 'obj_uuid_table') if uuids: def uuids_g(): for uuid in uuids: yield uuid else: def uuids_g(): for k, v in uuid_cf.get_range(column_count=1, filter_empty=True): yield k for uuid in uuids_g(): values = dict(uuid_cf.xget(uuid)) res = self._get_current_resource(uuid, values) bad_refs = self._check_resource_refs(uuid, values) if not res or bad_refs: printo(self._props_to_json(values)) if not res and not self.check: if self.force or continue_prompt(message="Delete ?"): self._delete(uuid_cf, uuid)
def _cassandra_init_conn_pools(self): socket_factory = self._make_socket_factory() for ks, cf_dict in itertools.chain(list(self._rw_keyspaces.items()), list(self._ro_keyspaces.items())): keyspace = '%s%s' % (self._db_prefix, ks) pool = pycassa.ConnectionPool(keyspace, self._server_list, max_overflow=5, use_threadlocal=True, prefill=True, pool_size=self._pool_size, pool_timeout=120, max_retries=15, timeout=5, credentials=self._credential, socket_factory=socket_factory) for cf_name in cf_dict: cf_kwargs = cf_dict[cf_name].get('cf_args', {}) self._cf_dict[cf_name] = ColumnFamily( pool, cf_name, read_consistency_level=ConsistencyLevel.QUORUM, write_consistency_level=ConsistencyLevel.QUORUM, dict_class=dict, **cf_kwargs) ConnectionState.update(conn_type=ConnType.DATABASE, name='Cassandra', status=ConnectionStatus.UP, message='', server_addrs=self._server_list) self._conn_state = ConnectionStatus.UP msg = 'Cassandra connection ESTABLISHED' self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def remove(self, colfam, key, columns=None): cf = ColumnFamily(self.db, colfam) if columns is not None: return cf.remove(key, columns) else: return cf.remove(key)
def test_packing_enabled(self): self.cf = ColumnFamily(pool, 'Standard1') self.cf.insert('key', {'col': 'val'}) assert_raises(TypeError, self.cf.insert, args=('key', {123: 'val'})) assert_raises(TypeError, self.cf.insert, args=('key', {'col': 123})) assert_raises(TypeError, self.cf.insert, args=('key', {123: 123})) self.cf.remove('key')
def test_failure_connection_info(self): stats_logger = StatsLoggerRequestInfo() pool = ConnectionPool(pool_size=1, max_overflow=0, recycle=10000, prefill=True, max_retries=0, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=True, server_list=['localhost:9160']) cf = ColumnFamily(pool, 'Counter1') # Corrupt the connection conn = pool.get() setattr(conn, 'send_get', conn._fail_once) conn._should_fail = True conn.return_to_pool() assert_raises(MaximumRetryException, cf.get, 'greunt', columns=['col']) assert_true('request' in stats_logger.failure_dict['connection'].info) request = stats_logger.failure_dict['connection'].info['request'] assert_equal(request['method'], 'get') assert_equal(request['args'], ('greunt', ColumnPath('Counter1', None, 'col'), 1)) assert_equal(request['kwargs'], {})
def test_insert_get_indexed_slices(self): indexed_cf = ColumnFamily(self.client, 'Indexed1') columns = {'birthdate': 1L} key = 'key1' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) key = 'key2' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) key = 'key3' indexed_cf.insert(key, columns, write_consistency_level=ConsistencyLevel.ONE) expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr]) result = indexed_cf.get_indexed_slices(clause) assert len(result) == 3 assert result.get('key1') == columns assert result.get('key2') == columns assert result.get('key3') == columns
def _cassandra_init_conn_pools(self): for ks, cf_list in self._keyspaces.items(): pool = pycassa.ConnectionPool(ks, self._server_list, max_overflow=-1, use_threadlocal=True, prefill=True, pool_size=20, pool_timeout=120, max_retries=-1, timeout=5) rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM for (cf, _) in cf_list: self._cf_dict[cf] = ColumnFamily( pool, cf, read_consistency_level=rd_consistency, write_consistency_level=wr_consistency) ConnectionState.update(conn_type=ConnectionType.DATABASE, name='Cassandra', status=ConnectionStatus.UP, message='', server_addrs=self._server_list) self._conn_state = ConnectionStatus.UP msg = 'Cassandra connection ESTABLISHED' self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def test_queue_failure_with_no_retries(self): stats_logger = StatsLoggerWithListStorage() pool = ConnectionPool( pool_size=5, max_overflow=5, recycle=10000, prefill=True, max_retries=3, # allow 3 retries keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[stats_logger], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) # Corrupt all of the connections for i in range(5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() cf = ColumnFamily(pool, 'Counter1') assert_raises(MaximumRetryException, cf.insert, 'key', { 'col': 2, 'col2': 2 }) assert_equal(stats_logger.stats['failed'], 1) # didn't retry at all pool.dispose()
def test_get_indexed_slices_batching(self): indexed_cf = ColumnFamily(pool, 'Indexed1') columns = {'birthdate': 1L} for i in range(200): indexed_cf.insert('key%d' % i, columns) expr = index.create_index_expression(column_name='birthdate', value=1L) clause = index.create_index_clause([expr], count=10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 10) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 10) clause = index.create_index_clause([expr], count=250) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=2)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=10)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=77)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=200)) assert_equal(len(result), 200) result = list(indexed_cf.get_indexed_slices(clause, buffer_size=1000)) assert_equal(len(result), 200)
def test_packing_disabled(self): self.cf = ColumnFamily(pool, 'Standard1', autopack_names=False, autopack_values=False) self.cf.insert('key', {'col': 'val'}) assert_raises(TypeError, self.cf.insert, args=('key', {123: 'val'})) assert_raises(TypeError, self.cf.insert, args=('key', {'col': 123})) assert_raises(TypeError, self.cf.insert, args=('key', {123: 123})) self.cf.remove('key')
def test_queue_threadlocal_retry_limit(self): listener = _TestListener() pool = ConnectionPool( pool_size=5, max_overflow=5, recycle=10000, prefill=True, max_retries=3, # allow 3 retries keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=True, server_list=['localhost:9160', 'localhost:9160']) # Corrupt all of the connections for i in range(5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() cf = ColumnFamily(pool, 'Standard1') assert_raises(MaximumRetryException, cf.insert, 'key', { 'col': 'val', 'col2': 'val' }) assert_equal(listener.failure_count, 4) # On the 4th failure, didn't retry pool.dispose()
def test_queue_failover(self): for prefill in (True, False): listener = _TestListener() pool = ConnectionPool( pool_size=1, max_overflow=0, recycle=10000, prefill=prefill, timeout=1, keyspace='PycassaTestKeyspace', credentials=_credentials, listeners=[listener], use_threadlocal=False, server_list=['localhost:9160', 'localhost:9160']) cf = ColumnFamily(pool, 'Standard1') for i in range(1, 5): conn = pool.get() setattr(conn, 'send_batch_mutate', conn._fail_once) conn._should_fail = True conn.return_to_pool() # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert('key', {'col': 'val%d' % i, 'col2': 'val'}) assert_equal(listener.failure_count, i) assert_equal(cf.get('key'), { 'col': 'val%d' % i, 'col2': 'val' }) pool.dispose()
def test_basic_pools(self): pool = ConnectionPool('PycassaTestKeyspace', credentials=_credentials) pool.dispose() pool = pool.recreate() cf = ColumnFamily(pool, 'Standard1') cf.insert('key1', {'col':'val'}) pool.status() pool.dispose()
def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp) self.scf = ColumnFamily(self.client, 'Super1', write_consistency_level=ConsistencyLevel.ONE, super=True, timestamp=self.timestamp) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear()
def setup_class(cls): sys = SystemManager() sys.create_column_family(TEST_KS, 'StdLong', comparator_type=LONG_TYPE) sys.create_column_family(TEST_KS, 'StdInteger', comparator_type=INT_TYPE) sys.create_column_family(TEST_KS, 'StdTimeUUID', comparator_type=TIME_UUID_TYPE) sys.create_column_family(TEST_KS, 'StdLexicalUUID', comparator_type=LEXICAL_UUID_TYPE) sys.create_column_family(TEST_KS, 'StdAscii', comparator_type=ASCII_TYPE) sys.create_column_family(TEST_KS, 'StdUTF8', comparator_type=UTF8_TYPE) sys.create_column_family(TEST_KS, 'StdBytes', comparator_type=BYTES_TYPE) sys.close() cls.cf_long = ColumnFamily(pool, 'StdLong') cls.cf_int = ColumnFamily(pool, 'StdInteger') cls.cf_time = ColumnFamily(pool, 'StdTimeUUID') cls.cf_lex = ColumnFamily(pool, 'StdLexicalUUID') cls.cf_ascii = ColumnFamily(pool, 'StdAscii') cls.cf_utf8 = ColumnFamily(pool, 'StdUTF8') cls.cf_bytes = ColumnFamily(pool, 'StdBytes') cls.cfs = [ cls.cf_long, cls.cf_int, cls.cf_time, cls.cf_lex, cls.cf_ascii, cls.cf_utf8, cls.cf_bytes ]
def setup_class(cls): sys = SystemManager() sys.create_column_family(TEST_KS, 'SuperLongSubLong', super=True, comparator_type=LONG_TYPE, subcomparator_type=LONG_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubInt', super=True, comparator_type=LONG_TYPE, subcomparator_type=INT_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubTime', super=True, comparator_type=LONG_TYPE, subcomparator_type=TIME_UUID_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubLex', super=True, comparator_type=LONG_TYPE, subcomparator_type=LEXICAL_UUID_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubAscii', super=True, comparator_type=LONG_TYPE, subcomparator_type=ASCII_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubUTF8', super=True, comparator_type=LONG_TYPE, subcomparator_type=UTF8_TYPE) sys.create_column_family(TEST_KS, 'SuperLongSubBytes', super=True, comparator_type=LONG_TYPE, subcomparator_type=BYTES_TYPE) sys.close() cls.cf_suplong_sublong = ColumnFamily(pool, 'SuperLongSubLong') cls.cf_suplong_subint = ColumnFamily(pool, 'SuperLongSubInt') cls.cf_suplong_subtime = ColumnFamily(pool, 'SuperLongSubTime') cls.cf_suplong_sublex = ColumnFamily(pool, 'SuperLongSubLex') cls.cf_suplong_subascii = ColumnFamily(pool, 'SuperLongSubAscii') cls.cf_suplong_subutf8 = ColumnFamily(pool, 'SuperLongSubUTF8') cls.cf_suplong_subbytes = ColumnFamily(pool, 'SuperLongSubBytes') cls.cfs = [cls.cf_suplong_subint, cls.cf_suplong_subint, cls.cf_suplong_subtime, cls.cf_suplong_sublex, cls.cf_suplong_subascii, cls.cf_suplong_subutf8, cls.cf_suplong_subbytes]
def setUp(self): credentials = {'username': '******', 'password': '******'} self.client = connect('Keyspace1', credentials=credentials) self.cf = ColumnFamily(self.client, 'Standard2', write_consistency_level=ConsistencyLevel.ONE, timestamp=self.timestamp, autopack_names=False, autopack_values=False) self.indexed_cf = ColumnFamily(self.client, 'Indexed1', autopack_names=False, autopack_values=False) self.map = ColumnFamilyMap(TestUTF8, self.cf) self.indexed_map = ColumnFamilyMap(TestIndex, self.indexed_cf) self.empty_map = ColumnFamilyMap(TestEmpty, self.cf, raw_columns=True) try: self.timestamp_n = int(self.cf.get('meta')['timestamp']) except NotFoundException: self.timestamp_n = 0 self.clear()
def _five_fails(pool, key, column): conn = pool.get() cf = ColumnFamily(conn, 'Standard1') for i in range(0,5): setattr(cf.client._connection.client, 'batch_mutate', _timeout) # The first insert attempt should fail, but failover should occur # and the insert should succeed cf.insert(key, column) cf.get(key)
def test_basic_pools(self): for pool_cls in _pools: print "Pool class: %s" % pool_cls.__name__ pool = pool_cls(keyspace='Keyspace1', credentials=_credentials) pool.dispose() pool = pool.recreate() conn = pool.get() cf = ColumnFamily(conn, 'Standard1') cf.insert('key1', {'col':'val'}) pool.status() pool.return_conn(conn)