def test_api_match(self):
cmanager = ManagedCassandraClientFactory(require_api_version=None)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
api_ver = cmanager._protos[0].api_version
yield cmanager.shutdown()
# try with the right version explicitly required
cmanager = ManagedCassandraClientFactory(require_api_version=api_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
yield cmanager.shutdown()
# try with a mismatching version explicitly required
bad_ver = [
v for (_, v) in translate.supported_versions if v != api_ver
][0]
cmanager = ManagedCassandraClientFactory(require_api_version=bad_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield self.assertFailure(d, translate.APIMismatch)
yield cmanager.shutdown()
def __init__(self, keyspace):
self._keyspace = keyspace
self.factory = ManagedCassandraClientFactory(keyspace)
host = settings.CASS_HOST.strip()
# Ensure that the host isn't blank
if host == '':
host = 'localhost'
addresses = socket.getaddrinfo(host, settings.CASS_PORT)
# If we are using IPv6, we need to provide an IPv6 address directly
# to twisted, as it doesn't support IPv6.
# addresses is a list of 5 tuple:
# (family, socktype, proto, cannonname, sockaddr)
shuffle(addresses)
if len(addresses) > 1 and addresses[0][0] == socket.AF_INET6:
# sockaddr is a 4 tuple:
# (address, port, flow, scope)
address = addresses[0][4][0]
else:
address = host
_log.debug("Cassandra is connecting to %s - for host %s", address,
settings.CASS_HOST)
reactor.connectTCP(address, settings.CASS_PORT, self.factory)
self.client = CassandraClient(self.factory)
def get(self):
# Attempt to connect to Cassandra (by asking for a non-existent key).
# We need this check as we've seen cases where telephus fails to
# connect to Cassandra, and requests sit on the queue forever without
# being processed.
clients = (CassandraClient(factory) for factory in self.cass_factories)
# If Cassandra is up, it will throw an exception (because we're asking
# for a nonexistent key). That's fine - it proves Cassandra is up and
# we have a connection to it. If Cassandra is down, this call will
# never return and Monit will time it out and kill the process for
# unresponsiveness.
# Catch the Twisted error made (as 'ping' isn't a configured column) -
# as with the Exception we don't care about the error, we just want to
# test if we can contact Cassandra
def ping_error(err): # pragma: no cover
pass
try:
_log.debug("Handling ping request")
gets = (client.get(key='ping', column_family='ping').addErrback(ping_error)
for client in clients)
yield defer.DeferredList(gets)
except Exception:
# We don't care about the result, just whether it returns
# in a timely fashion. Writing a log would be spammy.
pass
_log.debug("Handled ping request successfully")
self.finish("OK")
def test_initial_connection_failure(self):
cmanager = ManagedCassandraClientFactory()
client = CassandraClient(cmanager)
d = cmanager.deferred
reactor.connectTCP('nonexistent.example.com', PORT, cmanager)
yield self.assertFailure(d, error.DNSLookupError)
cmanager.shutdown()
def __init__(self, controller_name, host, port, username, password,
keyspace, instance_factory, sensor_item_factory,
prefix=''):
self.controller_name = str(controller_name)
# keep a set of known instances and sensors so we can save on
# unnecessary inserts to the controller instance/sensor lists
self.seen_instances = set()
self.seen_sensors = set()
self.instance_factory = instance_factory
self.sensor_item_factory = sensor_item_factory
authorization_dictionary = {'username': username, 'password': password}
self.manager = ManagedCassandraClientFactory(
credentials=authorization_dictionary,
check_api_version=True, keyspace=keyspace)
TCPConnection.__init__(self, host, port, self.manager)
self.client = CassandraClient(self.manager)
self.instance_cf = prefix + self.INSTANCE_CF_NAME
self.instance_id_cf = prefix + self.INSTANCE_ID_CF_NAME
self.sensor_cf = prefix + self.SENSOR_CF_NAME
self.sensor_id_cf = prefix + self.SENSOR_ID_CF_NAME
self.config_cf = prefix + self.CONFIG_CF_NAME
def keyspaceConnection(self, keyspace, consistency=ConsistencyLevel.ONE):
"""
Return a CassandraClient instance which uses this CassandraClusterPool
by way of a CassandraKeyspaceConnection, so that all requests made
through it are guaranteed to go to the given keyspace, no matter what
other consumers of this pool may do.
"""
return CassandraClient(CassandraKeyspaceConnection(self, keyspace),
consistency=consistency)
def initialize(self, factory_name, table, column):
"""
The factory_name, table and column are set as part of the Application router, see api/__init__.py
The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on
The row to operate on is passed to each function, while the value is in the request body, if relevant
"""
self.table = table
self.column = column
self.cass = CassandraClient(self.cass_factories[factory_name])
def connect(keyspace='system',
hosts=['localhost:9160'],
connections=5,
consistency=ConsistencyLevel.ONE):
manager = ManagedCassandraClientFactory(keyspace=keyspace)
client = CassandraClient(manager, consistency=consistency)
for i in xrange(connections):
for host, port in [x.split(':') for x in hosts]:
reactor.connectTCP(host, int(port), manager)
return client
def main():
sasl_kwargs = {
"host": "thobbs-laptop2",
"service": "host",
"mechanism": "GSSAPI"}
f = ManagedCassandraClientFactory(keyspace='system', sasl_kwargs=sasl_kwargs)
reactor.connectTCP(HOST, PORT, f)
yield f.deferred
c = CassandraClient(f)
yield setup_schema(c)
yield dostuff(c)
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz = {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True,
keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = thrift_api_ver_to_cassandra_ver(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'),
CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(name='col1',
validation_class=
'org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType'),
])
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
CfDef(keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
def connect(self, host=None, port=9160, username=None, password=None):
if not host:
host, port = get_host_port()
if username or password:
if not (username and password):
raise CassandraConfigurationError(
"Specify both username and password or neither")
else:
username, password = get_credentials()
authz = dict(username=username, password=password)
self.manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True)
self.connector = reactor.connectTCP(host, port, self.manager)
self.client = CassandraClient(self.manager)
def do_this():
for gzfilename in sys.argv[1:]:
n = 0
with gzip.GzipFile(gzfilename) as f:
csv_file = csv.DictReader(f)
first = True
last_key = ''
for row in csv_file:
if first:
factory = ManagedCassandraClientFactory(row["keyspace"])
reactor.connectTCP("localhost", 9160, factory)
client = CassandraClient(factory)
first = False
yield client.insert(column_family=row["cf"], key=uuidify(row["key"].decode("string_escape")), column=row["col"], value=uuidify(row["val"].decode("string_escape")))
if last_key != row["key"].decode("string_escape"):
n += 1
last_key = row["key"].decode("string_escape")
print "Successfully restored %d rows from %s" % (n, gzfilename)
reactor.stop()
def __init__(self,
seed_list,
keyspace=None,
creds=None,
thrift_port=None,
pool_size=None,
conn_timeout=10,
bind_address=None,
log_cb=log.msg,
reactor=None,
ssl_ctx_factory=None,
sasl_cred_factory=None,
auto_node_discovery=True,
fill_pool_throttle=0.5):
"""
Initialize a CassandraClusterPool.
@param keyspace: If given and not None, determines the keyspace to
which all connections in this pool will be made.
@param creds: Credentials to use to authenticate Cassandra connections
@type creds: A dict (or other mapping) of strings to strings
@param seed_list: An initial set of host addresses which, if
connectable, are part of this cluster.
@type seed_list: iterable
@param thrift_port: The port to use for connections to Cassandra nodes
@param pool_size: The target size for the connection pool. Naturally,
the actual size may be higher or lower as nodes connect and
disconnect, but an effort will be made to adjust toward this size.
@type pool_size: int
@param conn_timeout: The number of seconds before a pending connection
is deemed unsuccessful and aborted. Of course, when a connection
error can be detected before this time, the connection will be
aborted appropriately.
@type conn_timeout: float
@param bind_address: The local interface to which to bind when making
outbound connections. Default: determined by the system's socket
layer.
@type bind_address: str
@param log_cb: A callable which is expected to work like
L{twisted.python.log.msg}. Will be used when certain connection
and disconnection events occur. The default is log.msg.
@param reactor: The reactor instance to use when starting thrift
connections or setting timers.
@param ssl_ctx_factory: A L{twisted.internet.ssl.ClientContextFactory}
instance. If not None, SSL connections will be opened using
the provided context factory; if None, SSL will not be used.
@param sasl_cred_factory: A callable which, when called with two
parameters, a host and port, returns a dictionary of keyword
arguments to be used for the L{puresasl.client.SASLClient}
constructor. If supplied, the ThriftSASLClientProtocol will
be used.
@param auto_node_discovery: Bool that Enables/Disables node
auto-discovery, defaults to True
@param fill_pool_throttle: Float that indicates keeps fill_pool from
being called too fast in a failure situation
"""
self.describing_ring = False
self.seed_list = list(seed_list)
if thrift_port is None:
thrift_port = self.default_cassandra_thrift_port
self.thrift_port = thrift_port
if pool_size is None:
pool_size = len(self.seed_list)
self.target_pool_size = pool_size
self.log = log_cb
self.conn_timeout = conn_timeout
self.bind_address = bind_address
self.keyspace = keyspace
self.creds = creds
self.ssl_ctx_factory = ssl_ctx_factory
self.sasl_cred_factory = sasl_cred_factory
self.request_queue = defer.DeferredQueue()
self.future_fill_pool = None
self.removed_nodes = set()
self._client_instance = CassandraClient(self)
self.auto_node_discovery = auto_node_discovery
self.fill_pool_throttle = fill_pool_throttle
self.throttle_timer = None
self.fill_pool_last_called = None
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.retryables.extend(
(ttypes.TimedOutException, ttypes.UnavailableException))
# A set of CassandraNode instances representing known nodes. This
# includes nodes from the initial seed list, nodes seen in
# describe_ring calls to existing nodes, and nodes explicitly added
# by the addNode() method. Nodes are only removed from this set if
# no connections have been successful in self.forget_node_interval
# seconds, or by an explicit call to removeNode().
self.nodes = set()
# A set of CassandraPoolReconnectorFactory instances corresponding to
# connections which are either live or pending. Failed attempts to
# connect will remove a connector from this set. When connections are
# lost, an immediate reconnect will be attempted.
self.connectors = set()
# A collection of objects from self.connectors corresponding to
# existing, working (as far as we know) connections. This will be
# derivable from self.connectors, but hopefully will be maintained to
# present a good snapshot of what is alive, now, and what is not.
# This is stored in a deque so that it can be efficiently rotated
# to distribute requests.
self.good_conns = set()
# A set of CassandraPoolReconnectorFactory instances, formerly in
# self.connectors, the connections for which are draining. No new
# requests should be fed to these instances; they are tracked only so
# that they can be terminated more fully in case this service is shut
# down before they finish.
self.dying_conns = set()
def __init__(self,
seed_list,
keyspace=None,
creds=None,
thrift_port=None,
pool_size=None,
conn_timeout=10,
bind_address=None,
log_cb=log.msg,
reactor=None,
require_api_version=None):
"""
Initialize a CassandraClusterPool.
@param keyspace: If given and not None, determines the keyspace to
which all connections in this pool will be made.
@param creds: Credentials to use to authenticate Cassandra connections
@type creds: A dict (or other mapping) of strings to strings
@param seed_list: An initial set of host addresses which, if
connectable, are part of this cluster.
@type seed_list: iterable
@param thrift_port: The port to use for connections to Cassandra nodes
@param pool_size: The target size for the connection pool. Naturally,
the actual size may be higher or lower as nodes connect and
disconnect, but an effort will be made to adjust toward this size.
@type pool_size: int
@param conn_timeout: The number of seconds before a pending connection
is deemed unsuccessful and aborted. Of course, when a connection
error can be detected before this time, the connection will be
aborted appropriately.
@type conn_timeout: float
@param bind_address: The local interface to which to bind when making
outbound connections. Default: determined by the system's socket
layer.
@type bind_address: str
@param log_cb: A callable which is expected to work like
L{twisted.python.log.msg}. Will be used when certain connection
and disconnection events occur. The default is log.msg.
@param reactor: The reactor instance to use when starting thrift
connections or setting timers.
@param require_api_version: If not None, Telephus will require that
all connections conform to the API for the given Cassandra version.
Possible values are "0.7", "0.8", "1.0", etc.
If None, Telephus will consider all supported API versions to be
acceptable.
If the api version reported by a remote node is not compatible, the
connection to that node will be aborted. Default: None
"""
self.seed_list = list(seed_list)
if thrift_port is None:
thrift_port = self.default_cassandra_thrift_port
self.thrift_port = thrift_port
if pool_size is None:
pool_size = len(self.seed_list)
self.target_pool_size = pool_size
self.log = log_cb
self.conn_timeout = conn_timeout
self.bind_address = bind_address
self.keyspace = keyspace
self.creds = creds
self.request_queue = defer.DeferredQueue()
self.require_api_version = require_api_version
self.future_fill_pool = None
self.removed_nodes = set()
self._client_instance = CassandraClient(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
# A set of CassandraNode instances representing known nodes. This
# includes nodes from the initial seed list, nodes seen in
# describe_ring calls to existing nodes, and nodes explicitly added
# by the addNode() method. Nodes are only removed from this set if
# no connections have been successful in self.forget_node_interval
# seconds, or by an explicit call to removeNode().
self.nodes = set()
# A set of CassandraPoolReconnectorFactory instances corresponding to
# connections which are either live or pending. Failed attempts to
# connect will remove a connector from this set. When connections are
# lost, an immediate reconnect will be attempted.
self.connectors = set()
# A collection of objects from self.connectors corresponding to
# existing, working (as far as we know) connections. This will be
# derivable from self.connectors, but hopefully will be maintained to
# present a good snapshot of what is alive, now, and what is not.
# This is stored in a deque so that it can be efficiently rotated
# to distribute requests.
self.good_conns = set()
# A set of CassandraPoolReconnectorFactory instances, formerly in
# self.connectors, the connections for which are draining. No new
# requests should be fed to these instances; they are tracked only so
# that they can be terminated more fully in case this service is shut
# down before they finish.
self.dying_conns = set()
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = tuple(map(int, remote_ver.split('.')))
self.my_keyspace = ttypes.KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={},
cf_defs=[
ttypes.CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ttypes.ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=ttypes.IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version <= KS_RF_ATTRIBUTE:
self.my_keyspace.replication_factor = 1
else:
self.my_keyspace.strategy_options['replication_factor'] = '1'
if self.version >= COUNTERS_SUPPORTED_API:
self.my_keyspace.cf_defs.extend([
ttypes.CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
# from the data structure
res = yield client.batch_insert(key='test', column_family=CF, mapping={colname: 'bar'})
print "batch_insert", res
res = yield client.batch_insert(key='test', column_family=SCF, mapping={'foo': {colname: 'bar'}})
print "batch_insert", res
# with ttypes, you pass a list as you would for raw thrift
# this way you can set custom timestamps
cols = [Column(colname, 'bar', 1234), Column('bar', 'baz', 54321)]
res = yield client.batch_insert(key='test', column_family=CF, mapping=cols)
print "batch_insert", res
cols = [SuperColumn(name=colname, columns=cols)]
# of course you don't have to use kwargs if the order is correct
res = yield client.batch_insert('test', SCF, cols)
print "batch_insert", res
if __name__ == '__main__':
from twisted.internet import reactor
from twisted.python import log
import sys
log.startLogging(sys.stdout)
f = ManagedCassandraClientFactory(KEYSPACE)
c = CassandraClient(f)
dostuff(c)
reactor.connectTCP(HOST, PORT, f)
reactor.run()
