def CreateCF(connStr, keySpace, cfs, columesTypeList = None): sysmgt = SystemManager(connStr) list = sysmgt.list_keyspaces() if keySpace in list: #print('pyagent is existed') pass else: sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'}) cfDict = sysmgt.get_keyspace_column_families(keySpace) allCF = cfDict.keys() for i in range(len(cfs)): columnFamily = cfs[i] #print(columnFamily) if columesTypeList != None: columes = columesTypeList[i] else: columes = None if columnFamily in allCF: #print(columnFamily + ' is existed') pass else: #print(columnFamily + ' is creating') sysmgt.create_column_family(keySpace, columnFamily, super=False, comparator_type=UTF8_TYPE, key_validation_class=ASCII_TYPE, default_validation_class=UTF8_TYPE, column_validation_classes=columes, gc_grace_seconds=1000) sysmgt.close()
class Cluster(object): def __init__(self, name): self.keyspaces = OrderedDict() self.name = name self._sm_client = None cmcache.append('clusters', self) def setclient(self, servers_list=[connection.DEFAULT_SERVER]): for server in servers_list: try: self._sm_client = SystemManager(server) return except: pass if self._sm_client is None: raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list)) def registerKeyspace(self, name, keyspc): self.keyspaces[name] = keyspc if self._sm_client is None: self.setclient() # try to actually register this keyspace to the system via if not name in self._sm_client.list_keyspaces(): self._sm_client.create_keyspace(name, keyspec.strategy_class) def __str__(self): return self.name
def get_connection(): """ Creates a connection to Cassandra. Returs: pool """ cassandra_host = os.environ.get('CASSANDRA_HOST', 'localhost') sys_mgr = SystemManager() try: sys_mgr.describe_ring(KEYSPACE) except: sys_mgr.create_keyspace(KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'}) pool = ConnectionPool(KEYSPACE, server_list=[cassandra_host]) for cf_name in [CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY]: try: cf = ColumnFamily(pool, cf_name) except: sys_mgr.create_column_family(KEYSPACE, cf_name, comparator_type=TimeUUIDType()) cf = ColumnFamily(pool, cf_name) cf.get_count(str(uuid.uuid4())) sys_mgr.close() return pool
def __init__(self, engine, **kw): super(CassandraStore, self).__init__(engine, **kw) spliturl = urlsplit(engine) _, keyspace, column_family = spliturl.path.split('/') try: self._store = pycassa.ColumnFamily( pycassa.ConnectionPool(keyspace, [spliturl.hostname]), column_family, ) except pycassa.InvalidRequestException: from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8 system_manager = SystemManager(spliturl[1]) system_manager.create_keyspace( keyspace, pycassa.system_manager.SIMPLE_STRATEGY, dict(replication_factor=native(kw.get('replication', 1))), ) system_manager.create_column_family(keyspace, column_family) self._store = pycassa.ColumnFamily( pycassa.ConnectionPool(keyspace, [spliturl.netloc]), column_family, ) except pycassa.NotFoundException: from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8 system_manager = SystemManager(spliturl[1]) system_manager.create_column_family(keyspace, column_family) self._store = pycassa.ColumnFamily( pycassa.ConnectionPool(keyspace, [spliturl.netloc]), column_family, )
def initializeTableLayout(keyspace, server_list, replicationStrategy, strategyOptions, localDCName, credentials): sys_manager = SystemManager(server_list[0], credentials=credentials) # Make sure the the keyspace exists if keyspace not in sys_manager.list_keyspaces(): sys_manager.create_keyspace(keyspace, replicationStrategy, strategyOptions) cf_defs = sys_manager.get_keyspace_column_families(keyspace) # Create UTF8 CF's for tablename in ["global_nodes", "metadata"]: if tablename not in cf_defs.keys(): createUTF8ColumnFamily(sys_manager, keyspace, tablename) if localDCName: dcNodes = "dc_%s_nodes" % (localDCName,) if dcNodes not in cf_defs.keys(): createUTF8ColumnFamily(sys_manager, keyspace, dcNodes) else: # TODO Log we do not have the DC name pass if "node_slices" not in cf_defs.keys(): sys_manager.create_column_family( keyspace, "node_slices", super=False, comparator_type=pycassa_types.LongType(), key_validation_class=pycassa_types.UTF8Type(), default_validation_class=pycassa_types.LongType() )
def ensure_cassandra_cf(self): s = SystemManager() if self.keyspace not in s.list_keyspaces(): s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace): s.create_column_family(self.keyspace, self.cassandra_columns_family) self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def _cassandra_ensure_keyspace(self, server_list, keyspace_name, cf_info_list): # Retry till cassandra is up server_idx = 0 num_dbnodes = len(self._server_list) connected = False while not connected: try: cass_server = self._server_list[server_idx] sys_mgr = SystemManager(cass_server) connected = True except Exception as e: # TODO do only for # thrift.transport.TTransport.TTransportException server_idx = (server_idx + 1) % num_dbnodes time.sleep(3) if self._reset_config: try: sys_mgr.drop_keyspace(keyspace_name) except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) try: sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY, {'replication_factor': str(num_dbnodes)}) except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) gc_grace_sec = 0 if num_dbnodes > 1: gc_grace_sec = 60 for cf_info in cf_info_list: try: (cf_name, comparator_type) = cf_info if comparator_type: sys_mgr.create_column_family( keyspace_name, cf_name, comparator_type=comparator_type, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type') else: sys_mgr.create_column_family( keyspace_name, cf_name, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type') except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) sys_mgr.alter_column_family( keyspace_name, cf_name, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type')
def _cassandra_ensure_keyspace(self, server_list, keyspace_name, cf_info_list): # Retry till cassandra is up server_idx = 0 num_dbnodes = len(self._server_list) connected = False while not connected: try: cass_server = self._server_list[server_idx] sys_mgr = SystemManager(cass_server) connected = True except Exception as e: # TODO do only for # thrift.transport.TTransport.TTransportException server_idx = (server_idx + 1) % num_dbnodes time.sleep(3) if self._reset_config: try: sys_mgr.drop_keyspace(keyspace_name) except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) try: sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY, {'replication_factor': str(num_dbnodes)}) except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) gc_grace_sec = 0 if num_dbnodes > 1: gc_grace_sec = 60 for cf_info in cf_info_list: try: (cf_name, comparator_type) = cf_info if comparator_type: sys_mgr.create_column_family( keyspace_name, cf_name, comparator_type=comparator_type, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type') else: sys_mgr.create_column_family(keyspace_name, cf_name, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type') except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) sys_mgr.alter_column_family(keyspace_name, cf_name, gc_grace_seconds=gc_grace_sec, default_validation_class='UTF8Type')
def ensure_cassandra_cf(self): s = SystemManager() if self.keyspace not in s.list_keyspaces(): s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) if self.cassandra_columns_family not in s.get_keyspace_column_families( self.keyspace): s.create_column_family(self.keyspace, self.cassandra_columns_family) self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def test_big_batched_writes(): ## this is an m1.xlarge doing nothing but supporting this test server = 'localhost:9160' keyspace = 'testkeyspace_' + getpass.getuser().replace('-', '_') family = 'testcf' sm = SystemManager(server) try: sm.drop_keyspace(keyspace) except pycassa.InvalidRequestException: pass sm.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) sm.create_column_family(keyspace, family, super=False, key_validation_class = LEXICAL_UUID_TYPE, default_validation_class = LEXICAL_UUID_TYPE, column_name_class = ASCII_TYPE) sm.alter_column(keyspace, family, 'test', ASCII_TYPE) sm.close() pool = ConnectionPool(keyspace, [server], max_retries=10, pool_timeout=0, pool_size=10, timeout=120) pool.fill() pool.add_listener( Listener() ) ## assert that we are using framed transport conn = pool._q.get() assert isinstance(conn.transport, thrift.transport.TTransport.TFramedTransport) pool._q.put(conn) try: for num_rows in range(14, 20): ## write some data to cassandra using increasing data sizes one_mb = ' ' * 2**20 rows = [] for i in xrange(num_rows): key = uuid.uuid4() rows.append((key, dict(test=one_mb))) testcf = pycassa.ColumnFamily(pool, family) with testcf.batch() as batch: for (key, data_dict) in rows: data_size = len(data_dict.values()[0]) logger.critical('adding %r with %.6f MB' % (key, float(data_size)/2**20)) batch.insert(key, data_dict) logger.critical('%d rows written' % num_rows) finally: sm = SystemManager(server) try: sm.drop_keyspace(keyspace) except pycassa.InvalidRequestException: pass sm.close() logger.critical('clearing test keyspace: %r' % keyspace)
def __init__(self, engine, **kw): super(CassandraStore, self).__init__(engine, **kw) spliturl = urlparse.urlsplit(engine) _, keyspace, column_family = spliturl[2].split('/') try: self._pool = pycassa.connect(keyspace, [spliturl[1]]) self._store = pycassa.ColumnFamily(self._pool, column_family) except pycassa.InvalidRequestException: from pycassa.system_manager import SystemManager system_manager = SystemManager(spliturl[1]) system_manager.create_keyspace(keyspace, kw.get('replication', 1)) system_manager.create_column_family(keyspace, column_family) self._pool = pycassa.connect(keyspace, [spliturl[1]]) self._store = pycassa.ColumnFamily(self._pool, column_family)
def create_keyspace(self): """ Creates the Cassandra Keyspace (if not exist) """ sys_mgr = None try: sys_mgr = SystemManager() try: sys_mgr.describe_ring(settings.KEYSPACE) except: logger.info("create_keyspace(): Creating keyspace %s", settings.KEYSPACE) sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'}) finally: if sys_mgr: sys_mgr.close()
def __init__(self, engine, **kw): super(CassandraStore, self).__init__(engine, **kw) spliturl = urlparse.urlsplit(engine) _, keyspace, column_family = spliturl[2].split('/') try: self._pool = pycassa.connect(keyspace, [spliturl[1]]) self._store = pycassa.ColumnFamily(self._pool, column_family) except pycassa.InvalidRequestException: from pycassa.system_manager import SystemManager system_manager = SystemManager(spliturl[1]) system_manager.create_keyspace( keyspace, kw.get('replication', 1) ) system_manager.create_column_family(keyspace, column_family) self._pool = pycassa.connect(keyspace, [spliturl[1]]) self._store = pycassa.ColumnFamily(self._pool, column_family)
def cassandra_reset(): from feedly import settings hostname = settings.FEEDLY_CASSANDRA_HOSTS[0] keyspace = "test_feedly" sys = SystemManager(hostname) # sys.drop_keyspace(keyspace) if keyspace not in sys.list_keyspaces(): sys.create_keyspace(keyspace, SIMPLE_STRATEGY, {"replication_factor": "1"}) sys.create_column_family(keyspace, "activity", comparator_type=UTF8_TYPE) sys.create_column_family(keyspace, "timeline", comparator_type=IntegerType(reversed=True))
class CassandraDbSysManager: def __init__(self): try: logger.info("Connection to " + DB_IP + ":" + DB_PORT + " ...") self.sysDB = SystemManager(DB_IP + ":" + DB_PORT) logger.info("Connection to the systemManager established.") except Exception as exep: logger.warning("Could not connect to the DB '" + DB_IP + ":" + DB_PORT + "'. | " + str(exep)) def create_keyspace(self, KeyspaceName, replica_factor): try: logger.info( "Creating the Keyspace '" + KeyspaceName + "' with replica_factor '" + str(replica_factor) + "'") self.sysDB.create_keyspace(KeyspaceName, replica_factor) logger.info("Keyspace created.") except Exception as exep: logger.warning("Could not create the keyspace '" + KeyspaceName + "'. | " + str(exep)) def drop_keyspace(self, KeyspaceName): try: logger.info("Dropping the Keyspace '" + KeyspaceName + "'") self.sysDB.drop_keyspace(KeyspaceName) logger.info("Keyspace dropped.") except Exception as exep: logger.warning("Could not drop the keyspace '" + KeyspaceName + "'. | " + str(exep)) def create_column_family(self, KeyspaceName, columnFamily): try: logger.info("Creating the Column family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'") self.sysDB.create_column_family(KeyspaceName, columnFamily, super=False, comparator_type=pycassa.system_manager.ASCII_TYPE) logger.info("Column Family created.") except Exception as exep: logger.warning( "Could not create the Column Family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'. | " + str( exep)) def drop_column_family(self, KeyspaceName, columnFamily): try: logger.info("Dropping the Column Family '" + columnFamily + "' from the Keyspace '" + KeyspaceName + "'") self.sysDB.drop_column_family(KeyspaceName, columnFamily) logger.info("Column Family dropped.") except Exception as exep: logger.warning("Could not drop the column family '" + columnFamily + "'. | " + str(exep))
def _init_db(server): dbsys = SystemManager(server) dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY, {"replication_factor": "1"}) column_family_type = types.CompositeType( types.UTF8Type(), types.UTF8Type(), ) dbsys.create_column_family( KEY_SPACE, COLUMN_FAMILY, comparator_type=column_family_type, key_validation_class=types.UTF8Type(), default_validation_class=types.UTF8Type(), )
def cassandra_reset(): from feedly import settings hostname = settings.FEEDLY_CASSANDRA_HOSTS[0] keyspace = 'test_feedly' sys = SystemManager(hostname) # sys.drop_keyspace(keyspace) if keyspace not in sys.list_keyspaces(): sys.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) sys.create_column_family(keyspace, 'activity', comparator_type=UTF8_TYPE) sys.create_column_family(keyspace, 'timeline', comparator_type=IntegerType(reversed=True))
def setup_package(): sys = SystemManager() if TEST_KS in sys.list_keyspaces(): sys.drop_keyspace(TEST_KS) try: sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'}) sys.create_column_family(TEST_KS, 'Standard1') sys.create_column_family(TEST_KS, 'Super1', super=True) sys.create_column_family(TEST_KS, 'Indexed1') sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType') sys.create_column_family(TEST_KS, 'Counter1', default_validation_class='CounterColumnType') sys.create_column_family(TEST_KS, 'SuperCounter1', super=True, default_validation_class='CounterColumnType') except Exception, e: print e try: sys.drop_keyspace(TEST_KS) except: pass raise e
def setup_namespace(self, table_names): if self.pool: self.pool.dispose() del self.pool self.pool = None start_connect_time = time.time() super(CStorage, self).setup_namespace(table_names) sm = SystemManager(self._chosen_server) try: sm.create_keyspace( self._app_namespace, SIMPLE_STRATEGY, { 'replication_factor': str(self._config.get('replication_factor', '1')) }, ) except pycassa.InvalidRequestException, exc: if exc.why.startswith('Keyspace names must be case-insensitively unique'): pass else: raise exc
def __init__(self, engine, **kw): super(CassandraStore, self).__init__(engine, **kw) spliturl = urlsplit(engine) _, keyspace, column_family = spliturl.path.split('/') try: self._store = pycassa.ColumnFamily( pycassa.ConnectionPool(keyspace, [spliturl.hostname]), column_family, ) except pycassa.InvalidRequestException: from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8 system_manager = SystemManager(spliturl[1]) system_manager.create_keyspace( keyspace, pycassa.system_manager.SIMPLE_STRATEGY, dict(replication_factor=native(kw.get('replication', 1))), ) system_manager.create_column_family(keyspace, column_family) self._store = pycassa.ColumnFamily( pycassa.ConnectionPool(keyspace, [spliturl.netloc]), column_family, )
class ComedySysManagerConnection(object): """ This is a also a client connection of Cassandra, difference is it is only for System-Manager level """ def __init__(self, server='localhost:9160', credentials=None, framed_transport=True, timeout=_DEFAULT_TIMEOUT): self.conn = SystemManager(server,credentials,framed_transport,timeout) def create_keyspace(self, name, replication_strategy='SimpleStrategy', strategy_options=None, durable_writes=True, **ks_kwargs): """Create a database""" self.conn.create_keyspace(name, replication_strategy, strategy_options, durable_writes) create_database = create_keyspace def drop_keyspace(self, keyspace): """Drop a database""" self.conn.drop_keyspace(keyspace) drop_database = drop_keyspace def create_column_family(self, keyspace, name, column_validation_classes=None, **cf_kwargs): """Create a table""" self.conn.create_column_family(keyspace, name, column_validation_classes=None, **cf_kwargs) create_table = create_column_family def drop_column_family(self, keyspace, column_family): """Drop a table""" self.conn.drop_column_family(keyspace, column_family) drop_table = drop_column_family def close(self): pass
def test(host='localhost', keyspace_provided=None): from pycassa.types import BytesType, CompositeType from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY # Create a fake keyspace, if not provided if not keyspace_provided: import random keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000)) else: keyspace = keyspace_provided # Connect to cluster and create keyspace system_manager = SystemManager(host) system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) try: # Create CF with many CompositeFields comparator = CompositeType(BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType()) system_manager.create_column_family(keyspace, 'json', comparator_type=comparator) # Connect to the KS/CF and save samples db = CassandraJsonMapper.db(keyspace, 'json') tests = 0 # ---------------------------------------------------------------------- # Test a complicated structure sample_1 = { 'key1': { 'a': 1, 2: 'b', 'c': { 'd': 3, 'e': { 'f': True }, 'g': [ [ 'h', 'i', 'j', 4, 5 ], [ 'k', 'l', 'm', 'n', 'o' ], ], 'p': [ { 'id': 6, 'q': 'r' }, { 'id': 7, 's': 't' } ], 'u': [], 'v': None } } } db.save(sample_1) if db.get('key1') != sample_1['key1']: raise AssertionError('What was saved is not being equally returned.') tests += 1 # ---------------------------------------------------------------------- # Test improper format missing values sample_2 = { 'key1': 1 } try: db.save(sample_2) raise AssertionError('sample_2 should have thrown a KeyError.') except KeyError: pass tests += 1 # ---------------------------------------------------------------------- # Test saving multiple keys sample_3 = { 'key2': {2:2}, 'key3': {3:3}, 'key4': {4:4} } db.save(sample_3) if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \ db.get('key4') != {4:4}: raise AssertionError('Not all keys in a json_payload were saved.') tests += 1 # ---------------------------------------------------------------------- # Test delete db.delete('key1') if db.get('key1'): raise AssertionError('Not all keys in were delted.') tests += 1 # ---------------------------------------------------------------------- # Test deletes db.delete(['key2', 'key3', 'key4']) if db.get('key2') or db.get('key3') or db.get('key4'): raise AssertionError('Not all keys in were delted.') tests += 1 # ---------------------------------------------------------------------- # Test reading from fake keys if db.get('fake_key') != {}: raise AssertionError('A get(fake_key) should return {}.') tests += 1 # ---------------------------------------------------------------------- # Test multi-threaded save db.mt_save(sample_1) db.mt_finish() if db.get('key1') != sample_1['key1']: raise AssertionError('What was saved is not being equally returned.') tests += 1 # ---------------------------------------------------------------------- # print json_format(db.get('key1')) print '{0}/{0} tests passed!'.format(tests) except: raise finally: # For debugging purposes # raw_input('Press ENTER to continue...') if not keyspace_provided: # Delete the temporary KS's system_manager.drop_keyspace(keyspace) system_manager.close()
def test_composite_column_names(client): ''' examine the unique nature of cassandras "wide sorted rows"; Basically, C* is good at storing large OrderedDict objects and less good at the simpler kind of key=value storage that we were doing earlier. ''' config = client._config namespace = client._app_namespace chosen_server = client._chosen_server sm = SystemManager(chosen_server) sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'}) family = 'test' sm.create_column_family( namespace, family, super=False, key_validation_class = ASCII_TYPE, default_validation_class = BYTES_TYPE, comparator_type=CompositeType(UUIDType(), UUIDType()), #column_name_class = LEXICAL_UUID_TYPE ) logger.info( sm.describe_schema_versions() ) pool = ConnectionPool(namespace, config['storage_addresses'], max_retries=1000, pool_timeout=10, pool_size=2, timeout=120) cf = pycassa.ColumnFamily(pool, family) u1, u2, u3, u4 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1(), uuid.uuid1() ## insert four cf.insert('inbound', {(u1, u2): b''}) cf.insert('inbound', {(u1, u3): b''}) cf.insert('inbound', {(u1, u4): b''}) cf.insert('inbound', {(u1, u1): b'45'}) ## insert three with duplicates cf.insert('inbound', {(u3, u3): b'42'}) cf.insert('inbound', {(u3, u3): b'43'}) cf.insert('inbound', {(u3, u2): b''}) cf.insert('inbound', {(u3, u4): b''}) ## insert two cf.insert('inbound', {(u2, u3): b''}) cf.insert('inbound', {(u2, u4): b''}) ## verify start/finish parameters work as expected assert 4 == len(cf.get('inbound', column_start=(u1,), column_finish=(u1,))) assert 2 == len(cf.get('inbound', column_start=(u2,), column_finish=(u2,))) assert 3 == len(cf.get('inbound', column_start=(u3,), column_finish=(u3,))) ## check delete of a single column cf.remove('inbound', columns=[(u2, u3)]) assert 1 == len(cf.get('inbound', column_start=(u2,), column_finish=(u2,))) ## check that the last inserted value appears at the expected ## location in the OrderedDict assert '43' == cf.get('inbound', column_start=(u3,), column_finish=(u3,)).items()[1][1] rec1 = cf.get('inbound', column_start=(u3,u3), column_finish=(u3,u3)).items() assert len(rec1) == 1 start = uuid.UUID(int=0) finish = uuid.UUID(int=2**128-1) assert start < u1 < u2 < u3 < u4 < finish assert 8 == len(cf.get('inbound', column_start=(start,), column_finish=(finish,)).items()) ## test range searching start = uuid.UUID(int=u3.int - 1) finish = uuid.UUID(int=u3.int + 1) assert start.int < u3.int < finish.int rec2 = cf.get('inbound', column_start=(start,), column_finish=(finish,)).items() assert rec2[1][0] == rec1[0][0] assert rec2[1][1] == rec1[0][1] == '43' cf.insert('inbound', {(u3,u3): b''.join(map(lambda u: u.bytes, (u1,u2,u3,u4)))}) data = cf.get('inbound', column_start=(u3,u3), column_finish=(u3,u3)).items()[0][1] assert [u1,u2,u3,u4] == map(lambda b: uuid.UUID(bytes=b), grouper(data, 16)) sm.close()
def test_composite_column_names_second_level_range_query_with_decomposited_keys(client): ''' check that we can execute range queries on the second part of a CompositeType column name after we unpack the composite key into a long string of concatenated hex forms of the UUIDs ''' config = client._config namespace = client._app_namespace chosen_server = client._chosen_server sm = SystemManager(chosen_server) sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'}) family = 'test' sm.create_column_family( namespace, family, super=False, key_validation_class = ASCII_TYPE, default_validation_class = BYTES_TYPE, comparator_type=UTF8Type(), ) pool = ConnectionPool(namespace, config['storage_addresses'], max_retries=1000, pool_timeout=10, pool_size=2, timeout=120) cf = pycassa.ColumnFamily(pool, family) u1, u2, u3, u4 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1(), uuid.uuid1() cf.insert('inbound', {join_uuids(u1, u2): b''}) cf.insert('inbound', {join_uuids(u1, u3): b''}) cf.insert('inbound', {join_uuids(u1, u4): b''}) ## test range searching start = uuid.UUID(int=u3.int - 1) finish = uuid.UUID(int=u3.int + 1) assert start.int < u3.int < finish.int rec3 = cf.get('inbound', column_start =join_uuids(u1, start), column_finish=join_uuids(u1, finish)).items() assert len(rec3) == 1 assert split_uuids(rec3[0][0])[1] == u3 #### This assert above passes! #### This next part fails :-/ ## now insert many rows -- enough that some should fall in each ## subrange below for i in xrange(1000): cf.insert('inbound', {join_uuids(u1, uuid.uuid4()): b''}) ## do four ranges, and expect more than zero in each step_size = 2**(128 - 2) for i in range(2**2, 0, -1): start = uuid.UUID(int=(i-1) * step_size) finish = uuid.UUID(int=min(i * step_size, 2**128 - 1)) recs = cf.get('inbound', column_start =join_uuids(u1, start), column_finish=join_uuids(u1, finish)).items() for key, val in recs: key = split_uuids(key) assert val == b'' assert key[0] == u1 assert key[1] < finish assert start < key[1] ## this passes!! assert len(recs) > 0 logger.info( '%r for %r %r' % (len(recs), start, finish)) sm.close()
class DistributedCassandraLockTestCase(unittest.TestCase): def setUp(self): self.sysman = SystemManager() self.sysman.create_keyspace( TEST_KS, SIMPLE_STRATEGY, dict(replication_factor='1') ) self.sysman.create_column_family(TEST_KS, TEST_CF) self.pool = ConnectionPool(TEST_KS) self.cf = ColumnFamily(self.pool, TEST_CF) def tearDown(self): self.sysman.drop_keyspace(TEST_KS) del self.sysman def test_ttl(self): l = CassandraDistributedRowLock( self.pool, TEST_CF, "test_ttl", ttl=2.0, consistency_level=ConsistencyLevel.ONE, timeout=1.0 ) try: l.acquire() self.assertEqual(1, len(l.read_lock_columns())) time.sleep(3) self.assertEqual(0, len(l.read_lock_columns())) except: raise finally: l.release() self.assertEqual(0, len(l.read_lock_columns())) def test_stale_lock(self): l1 = CassandraDistributedRowLock( self.pool, TEST_CF, "test_stale_lock", ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE, timeout=1.0 ) l2 = CassandraDistributedRowLock( self.pool, TEST_CF, "test_stale_lock", ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE, timeout=9.0 ) try: l1.acquire() time.sleep(5) try: l2.acquire() except: raise finally: l2.release() except: raise finally: l1.release() def test_stale_lock_with_fail(self): l1 = CassandraDistributedRowLock( self.pool, TEST_CF, "test_stale_lock", ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE, timeout=1.0 ) l2 = CassandraDistributedRowLock( self.pool, TEST_CF, "test_stale_lock", ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE, timeout=9.0, fail_on_stale_lock=True ) try: l1.acquire() time.sleep(2) try: l2.acquire() assert False, "derp" except StaleLockException: print 'things are working smoothly' except: raise finally: l2.release() except: raise finally: l2.release()
def test(host='localhost', keyspace_provided=None): from pycassa.types import BytesType, CompositeType from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY # Create a fake keyspace, if not provided if not keyspace_provided: import random keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000)) else: keyspace = keyspace_provided # Connect to cluster and create keyspace system_manager = SystemManager(host) system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'}) try: # Create CF with many CompositeFields comparator = CompositeType(BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType(), BytesType()) system_manager.create_column_family(keyspace, 'json', comparator_type=comparator) # Connect to the KS/CF and save samples db = CassandraJsonMapper.db(keyspace, 'json') tests = 0 # ---------------------------------------------------------------------- # Test a complicated structure sample_1 = { 'key1': { 'a': 1, 2: 'b', 'c': { 'd': 3, 'e': { 'f': True }, 'g': [ ['h', 'i', 'j', 4, 5], ['k', 'l', 'm', 'n', 'o'], ], 'p': [{ 'id': 6, 'q': 'r' }, { 'id': 7, 's': 't' }], 'u': [], 'v': None } } } db.save(sample_1) if db.get('key1') != sample_1['key1']: raise AssertionError( 'What was saved is not being equally returned.') tests += 1 # ---------------------------------------------------------------------- # Test improper format missing values sample_2 = {'key1': 1} try: db.save(sample_2) raise AssertionError('sample_2 should have thrown a KeyError.') except KeyError: pass tests += 1 # ---------------------------------------------------------------------- # Test saving multiple keys sample_3 = {'key2': {2: 2}, 'key3': {3: 3}, 'key4': {4: 4}} db.save(sample_3) if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \ db.get('key4') != {4:4}: raise AssertionError('Not all keys in a json_payload were saved.') tests += 1 # ---------------------------------------------------------------------- # Test delete db.delete('key1') if db.get('key1'): raise AssertionError('Not all keys in were delted.') tests += 1 # ---------------------------------------------------------------------- # Test deletes db.delete(['key2', 'key3', 'key4']) if db.get('key2') or db.get('key3') or db.get('key4'): raise AssertionError('Not all keys in were delted.') tests += 1 # ---------------------------------------------------------------------- # Test reading from fake keys if db.get('fake_key') != {}: raise AssertionError('A get(fake_key) should return {}.') tests += 1 # ---------------------------------------------------------------------- # Test multi-threaded save db.mt_save(sample_1) db.mt_finish() if db.get('key1') != sample_1['key1']: raise AssertionError( 'What was saved is not being equally returned.') tests += 1 # ---------------------------------------------------------------------- # print json_format(db.get('key1')) print '{0}/{0} tests passed!'.format(tests) except: raise finally: # For debugging purposes # raw_input('Press ENTER to continue...') if not keyspace_provided: # Delete the temporary KS's system_manager.drop_keyspace(keyspace) system_manager.close()
from pycassa.system_manager import SystemManager from pycassa.system_manager import SIMPLE_STRATEGY from pycassa.system_manager import UTF8_TYPE from pycassa.system_manager import LONG_TYPE from storage.cassandra_settings import HOSTS from storage.cassandra_settings import KEYSPACE_NAME sys = SystemManager(HOSTS[0]) if KEYSPACE_NAME in sys.list_keyspaces(): sys.drop_keyspace(KEYSPACE_NAME) sys.create_keyspace(KEYSPACE_NAME, SIMPLE_STRATEGY, {'replication_factor': '1'}) sys.create_column_family(KEYSPACE_NAME, 'LoveActivity', comparator_type=UTF8_TYPE) sys.create_column_family(KEYSPACE_NAME, 'Feed', comparator_type=UTF8_TYPE) sys.create_column_family(KEYSPACE_NAME, 'AggregatedFeed', comparator_type=LONG_TYPE, super=True)
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY from pycassa import InvalidRequestException mgr = SystemManager() try: mgr.create_keyspace("craystack", SIMPLE_STRATEGY, {"replication_factor": "1"}) print "Created keyspace." except InvalidRequestException: print "Keyspace already exists!" try: mgr.create_column_family("craystack", "objects") print "Created column family." except InvalidRequestException: print "Column family already exists!"
def _create_namespace(self, namespace): sm = SystemManager(random.choice(self.server_list)) sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'}) sm.close()
from pycassa import ConnectionPool from pycassa.columnfamily import ColumnFamily from pycassa.system_manager import SystemManager from pycassa.system_manager import SIMPLE_STRATEGY from pycassa.system_manager import UTF8_TYPE from pycassa.system_manager import ASCII_TYPE import time ############################## Create Keyspace ################################ server_list = ['cassandra1:9160', 'cassandra2:9160', 'cassandra3:9160'] sys = SystemManager(server_list[0]) sys.list_keyspaces() if 'entries' in sys.list_keyspaces(): sys.drop_keyspace('entries') sys.create_keyspace('entries', SIMPLE_STRATEGY, {'replication_factor': '1'}) ############################## Connection Pooling ############################ #pool = ConnectionPool('entries', server_list=server_list, pool_size=20) pool = ConnectionPool('entries', server_list=server_list) ############################## Create Column Family ########################### sys.create_column_family('entries', 'Author', comparator_type=UTF8_TYPE) author_cf = ColumnFamily(pool, 'Author') ################################ INSERT ####################################### # Insert a row with a Column author_cf.insert('sacharya', {'first_name': 'Sudarshan'}) # Insert a row with multiple columns author_cf.insert('sacharya1', {'first_name': 'Sudarshan', 'last_name': 'Acharya'})
""" Create the Cassandra database. """ import pycassa from pycassa.system_manager import SystemManager mgr = SystemManager() mgr.create_keyspace('drought', strategy_options={'replication_factor': '1'}) mgr.create_column_family('drought', 'cmip5') mgr.create_column_family('drought', 'zipcodes') mgr.alter_column('drought', 'zipcodes', 'ZIPCODE', pycassa.types.IntegerType()) mgr.create_index('drought', 'zipcodes', 'ZIPCODE', pycassa.types.IntegerType()) mgr.alter_column('drought', 'zipcodes', 'CENTER_LATITUDE', pycassa.types.FloatType()) mgr.alter_column('drought', 'zipcodes', 'CENTER_LONGITUDE', pycassa.types.FloatType()) mgr.create_column_family('drought', 'counties') mgr.close()