def test_read_timeout(self):
"""
Trigger and ensure read_timeouts are counted
Write a key, value pair. Force kill a node without waiting for the cluster to register the death.
Attempt a read at cl.ALL and receive a ReadTimeout.
"""
cluster = Cluster(metrics_enabled=True)
session = cluster.connect()
# Test write
session.execute("INSERT INTO test3rf.test (k, v) VALUES (1, 1)")
# Assert read
query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL)
results = session.execute(query, {'k': 1})
self.assertEqual(1, results[0].v)
# Force kill ccm node
get_node(1).stop(wait=False, gently=False)
try:
# Test read
query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL)
self.assertRaises(ReadTimeout, session.execute, query, {'k': 1})
self.assertEqual(1, cluster.metrics.stats.read_timeouts)
finally:
get_node(1).start(wait_other_notice=True, wait_for_binary_proto=True)
def setup_test_keyspace():
cluster = Cluster()
session = cluster.connect()
try:
results = session.execute("SELECT keyspace_name FROM system.schema_keyspaces")
existing_keyspaces = [row[0] for row in results]
for ksname in ('test1rf', 'test2rf', 'test3rf'):
if ksname in existing_keyspaces:
session.execute("DROP KEYSPACE %s" % ksname)
ddl = '''
CREATE KEYSPACE test3rf
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}'''
session.execute(ddl)
ddl = '''
CREATE KEYSPACE test2rf
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '2'}'''
session.execute(ddl)
ddl = '''
CREATE KEYSPACE test1rf
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}'''
session.execute(ddl)
ddl = '''
CREATE TABLE test3rf.test (
k int PRIMARY KEY,
v int )'''
session.execute(ddl)
finally:
cluster.shutdown()
def _main(keyspace, confKey, confVal, cmd):
cluster = Cluster()
session = cluster.connect(keyspace)
qResult = None
if cmd in ('UPDATE'):
cqlCmd = "update %s.conf set confVal='%s' where confKey='%s'" % (keyspace, confVal.replace('"', '"').replace('\'', ''').replace('\\', '\'), confKey)
try:
eResult = session.execute(cqlCmd)
filename = '/tmp/TQAlert/TQAlert.confchange'
try:
subprocess.call(['rm', '-f', filename])
with open(filename, 'w') as f:
f.write('%s\n' % datetime.now().strftime('%s'))
subprocess.call(['chmod', '0777', filename])
except:
pass
except:
return {'Result': 'Error! Failed to excute [%s]!'%cqlCmd}
return {'Result': 'OK'}
elif cmd in ('QUERY'):
queryStr = "select confVal from %s.conf where confKey='%s'" %(keyspace, confKey)
try:
qResult = session.execute(queryStr)
except:
return {'Result': 'Error! Failed to excute [%s]!'%queryStr}
if qResult is None or len(qResult.current_rows)<=0:
return {'Result': 'Error! No Such Data'}
return {'Result': 'OK', 'confVal':qResult[0][0].replace('"', '"').replace(''', '\'').replace('\', '\\')}
return {'Result': 'Error! Wrong cmd!'}
def test_unavailable(self):
"""
Trigger and ensure unavailables are counted
Write a key, value pair. Kill a node while waiting for the cluster to register the death.
Attempt an insert/read at cl.ALL and receive a Unavailable Exception.
"""
cluster = Cluster(metrics_enabled=True)
session = cluster.connect()
# Test write
session.execute("INSERT INTO test3rf.test (k, v) VALUES (1, 1)")
# Assert read
query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL)
results = session.execute(query, {'k': 1})
self.assertEqual(1, results[0].v)
# Force kill ccm node
get_node(1).stop(wait=True, gently=True)
try:
# Test write
query = SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL)
self.assertRaises(Unavailable, session.execute, query)
self.assertEqual(1, cluster.metrics.stats.unavailables)
# Test write
query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL)
self.assertRaises(Unavailable, session.execute, query, {'k': 1})
self.assertEqual(2, cluster.metrics.stats.unavailables)
finally:
get_node(1).start(wait_other_notice=True, wait_for_binary_proto=True)
def setup(hosts):
cluster = Cluster(hosts)
cluster.set_core_connections_per_host(HostDistance.LOCAL, 1)
session = cluster.connect()
rows = session.execute("SELECT keyspace_name FROM system.schema_keyspaces")
if KEYSPACE in [row[0] for row in rows]:
log.debug("dropping existing keyspace...")
session.execute("DROP KEYSPACE " + KEYSPACE)
log.debug("Creating keyspace...")
session.execute(
"""
CREATE KEYSPACE %s
WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': '2' }
"""
% KEYSPACE
)
log.debug("Setting keyspace...")
session.set_keyspace(KEYSPACE)
log.debug("Creating table...")
session.execute(
"""
CREATE TABLE %s (
thekey text,
col1 text,
col2 text,
PRIMARY KEY (thekey, col1)
)
"""
% TABLE
)
def connect(self):
"""
Connect to Cassandra cluster
:return:
"""
cluster = Cluster()
self.session = cluster.connect(SENSOR_DATA_KEYSPACE)
def handle_noargs(self, **options):
cluster = Cluster()
session = cluster.connect()
# Checking if keysapce exists
query = "SELECT * FROM system.schema_keyspaces WHERE keyspace_name='%s';" % KEYSPACE_NAME
result = session.execute(query)
if len(result) != 0:
msg = 'Looks like you already have a %s keyspace.\nDo you want to delete it and recreate it? All current data will be deleted! (y/n): ' % KEYSPACE_NAME
resp = raw_input(msg)
if not resp or resp[0] != 'y':
print "Ok, then we're done here."
return
query = "DROP KEYSPACE %s" % KEYSPACE_NAME
session.execute(query)
# Creating keysapce
query = "CREATE KEYSPACE tess WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};"
session.execute(query)
# Creating tables
query = "USE tess;"
session.execute(query)
query = "CREATE TABLE emotiv_eeg_record (test_id int, time double, AF3 double, F7 double, F3 double, FC5 double, T7 double, P7 double, O1 double, O2 double, P8 double, T8 double, FC6 double, F4 double, F8 double, AF4 double, PRIMARY KEY (test_id, time));"
session.execute(query)
cluster.shutdown()
print 'All done!'
def test_pool_management(self):
# Ensure that in_flight and request_ids quiesce after cluster operations
cluster = Cluster(protocol_version=PROTOCOL_VERSION, idle_heartbeat_interval=0) # no idle heartbeat here, pool management is tested in test_idle_heartbeat
session = cluster.connect()
session2 = cluster.connect()
# prepare
p = session.prepare("SELECT * FROM system.local WHERE key=?")
self.assertTrue(session.execute(p, ('local',)))
# simple
self.assertTrue(session.execute("SELECT * FROM system.local WHERE key='local'"))
# set keyspace
session.set_keyspace('system')
session.set_keyspace('system_traces')
# use keyspace
session.execute('USE system')
session.execute('USE system_traces')
# refresh schema
cluster.refresh_schema_metadata()
cluster.refresh_schema_metadata(max_schema_agreement_wait=0)
# submit schema refresh
future = cluster.submit_schema_refresh()
future.result()
assert_quiescent_pool_state(self, cluster)
cluster.shutdown()
class ConnectionTimeoutTest(unittest.TestCase):
def setUp(self):
self.defaultInFlight = Connection.max_in_flight
Connection.max_in_flight = 2
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.1']))
self.session = self.cluster.connect()
def tearDown(self):
Connection.max_in_flight = self.defaultInFlight
self.cluster.shutdown()
def test_in_flight_timeout(self):
"""
Test to ensure that connection id fetching will block when max_id is reached/
In previous versions of the driver this test will cause a
NoHostAvailable exception to be thrown, when the max_id is restricted
@since 3.3
@jira_ticket PYTHON-514
@expected_result When many requests are run on a single node connection acquisition should block
until connection is available or the request times out.
@test_category connection timeout
"""
futures = []
query = '''SELECT * FROM system.local'''
for i in range(100):
futures.append(self.session.execute_async(query))
for future in futures:
future.result()
def test_submit_schema_refresh(self):
"""
Ensure new new schema is refreshed after submit_schema_refresh()
"""
cluster = Cluster(protocol_version=PROTOCOL_VERSION)
cluster.connect()
self.assertNotIn("newkeyspace", cluster.metadata.keyspaces)
other_cluster = Cluster(protocol_version=PROTOCOL_VERSION)
session = other_cluster.connect()
session.execute(
"""
CREATE KEYSPACE newkeyspace
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}
""")
future = cluster.submit_schema_refresh()
future.result()
self.assertIn("newkeyspace", cluster.metadata.keyspaces)
session.execute("DROP KEYSPACE newkeyspace")
cluster.shutdown()
other_cluster.shutdown()
def test_refresh_schema_type(self):
if get_server_versions()[0] < (2, 1, 0):
raise unittest.SkipTest('UDTs were introduced in Cassandra 2.1')
if PROTOCOL_VERSION < 3:
raise unittest.SkipTest('UDTs are not specified in change events for protocol v2')
# We may want to refresh types on keyspace change events in that case(?)
cluster = Cluster(protocol_version=PROTOCOL_VERSION)
session = cluster.connect()
keyspace_name = 'test1rf'
type_name = self._testMethodName
session.execute('CREATE TYPE IF NOT EXISTS %s.%s (one int, two text)' % (keyspace_name, type_name))
original_meta = cluster.metadata.keyspaces
original_test1rf_meta = original_meta[keyspace_name]
original_type_meta = original_test1rf_meta.user_types[type_name]
# only refresh one type
cluster.refresh_user_type_metadata('test1rf', type_name)
current_meta = cluster.metadata.keyspaces
current_test1rf_meta = current_meta[keyspace_name]
current_type_meta = current_test1rf_meta.user_types[type_name]
self.assertIs(original_meta, current_meta)
self.assertIs(original_test1rf_meta, current_test1rf_meta)
self.assertIsNot(original_type_meta, current_type_meta)
self.assertEqual(original_type_meta.as_cql_query(), current_type_meta.as_cql_query())
session.shutdown()
def setup_cluster():
if args.host is None:
nodes = ['localhost']
else:
nodes = [args.host]
if args.port is None:
port = 9042
else:
port = args.port
cluster = None
if args.protocol_version is not None:
auth = None
if args.username is not None and args.password is not None:
if args.protocol_version == 1:
auth = get_credentials
elif args.protocol_version > 1:
auth = PlainTextAuthProvider(username=args.username, password=args.password)
cluster = Cluster(contact_points=nodes, protocol_version=args.protocol_version, auth_provider=auth, load_balancing_policy=cassandra.policies.WhiteListRoundRobinPolicy(nodes))
else:
cluster = Cluster(contact_points=nodes, port=port, load_balancing_policy=cassandra.policies.WhiteListRoundRobinPolicy(nodes))
session = cluster.connect()
session.default_timeout = TIMEOUT
session.default_fetch_size = FETCH_SIZE
session.row_factory = cassandra.query.ordered_dict_factory
return session
def test_connect_to_already_shutdown_cluster(self):
"""
Ensure you cannot connect to a cluster that's been shutdown
"""
cluster = Cluster(protocol_version=PROTOCOL_VERSION)
cluster.shutdown()
self.assertRaises(Exception, cluster.connect)
def test_session_no_cluster(self):
"""
Test session context without cluster context.
@since 3.4
@jira_ticket PYTHON-521
@expected_result session should be created correctly. Session should shutdown correctly outside of context
@test_category configuration
"""
cluster = Cluster(**self.cluster_kwargs)
unmanaged_session = cluster.connect()
with cluster.connect() as session:
self.assertFalse(cluster.is_shutdown)
self.assertFalse(session.is_shutdown)
self.assertFalse(unmanaged_session.is_shutdown)
self.assertTrue(session.execute('select release_version from system.local')[0])
self.assertTrue(session.is_shutdown)
self.assertFalse(cluster.is_shutdown)
self.assertFalse(unmanaged_session.is_shutdown)
unmanaged_session.shutdown()
self.assertTrue(unmanaged_session.is_shutdown)
self.assertFalse(cluster.is_shutdown)
cluster.shutdown()
self.assertTrue(cluster.is_shutdown)
class DuplicateRpcTest(unittest.TestCase):
load_balancing_policy = WhiteListRoundRobinPolicy(['127.0.0.1'])
def setUp(self):
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, load_balancing_policy=self.load_balancing_policy)
self.session = self.cluster.connect()
self.session.execute("UPDATE system.peers SET rpc_address = '127.0.0.1' WHERE peer='127.0.0.2'")
def tearDown(self):
self.session.execute("UPDATE system.peers SET rpc_address = '127.0.0.2' WHERE peer='127.0.0.2'")
self.cluster.shutdown()
def test_duplicate(self):
"""
Test duplicate RPC addresses.
Modifies the system.peers table to make hosts have the same rpc address. Ensures such hosts are filtered out and a message is logged
@since 3.4
@jira_ticket PYTHON-366
@expected_result only one hosts' metadata will be populated
@test_category metadata
"""
mock_handler = MockLoggingHandler()
logger = logging.getLogger(cassandra.cluster.__name__)
logger.addHandler(mock_handler)
test_cluster = self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, load_balancing_policy=self.load_balancing_policy)
test_cluster.connect()
warnings = mock_handler.messages.get("warning")
self.assertEqual(len(warnings), 1)
self.assertTrue('multiple' in warnings[0])
logger.removeHandler(mock_handler)
def get_conference_pictures(self, keyword):
json_data = []
# Cassandra initialization
cluster = Cluster([config.get("cassandra.host1"), config.get("cassandra.host2")])
session = cluster.connect('insight')
# Instagram initialization
instagram_api = InstagramAPI(client_id=config.get("instagram.client_id"), client_secret=config.get("instagram.client_secret"))
yymmdd = datetime.utcnow().strftime('%y') + datetime.utcnow().strftime('%m') + datetime.utcnow().strftime('%d')
rows = session.execute(self.TOP_10_QUERY % (yymmdd))
for (yymmdd, count, word) in rows:
img_arr = []
popular_media = instagram_api.media_popular(count=20)
for media in popular_media:
img_arr.append(media.images['standard_resolution'].url)
json_data.append({"word": word, "count": count, "pic_url": img_arr})
return json_data
def run_query(self, query, user):
connection = None
try:
if self.configuration.get('username', '') and self.configuration.get('password', ''):
auth_provider = PlainTextAuthProvider(username='******'.format(self.configuration.get('username', '')),
password='******'.format(self.configuration.get('password', '')))
connection = Cluster([self.configuration.get('host', '')],
auth_provider=auth_provider,
port=self.configuration.get('port', ''),
protocol_version=self.configuration.get('protocol', 3))
else:
connection = Cluster([self.configuration.get('host', '')],
port=self.configuration.get('port', ''),
protocol_version=self.configuration.get('protocol', 3))
session = connection.connect()
session.set_keyspace(self.configuration['keyspace'])
logger.debug("Cassandra running query: %s", query)
result = session.execute(query)
column_names = result.column_names
columns = self.fetch_columns(map(lambda c: (c, 'string'), column_names))
rows = [dict(zip(column_names, row)) for row in result]
data = {'columns': columns, 'rows': rows}
json_data = json.dumps(data, cls=CassandraJSONEncoder)
error = None
except KeyboardInterrupt:
error = "Query cancelled by user."
json_data = None
return json_data, error
def all_query(conn, field, clause, initial_set, contact_points, keyspace):
cluster = Cluster(contact_points)
session = cluster.connect(keyspace)
names = conn.recv()
if initial_set != "*":
results = set(initial_set)
else:
results = initial_set
for name in names:
if len(results) == 0:
break
query = "SELECT variant_id FROM variants_by_samples_%s WHERE sample_name = '%s' AND %s %s " % (field, name, field, clause)
if results == "*":
results = rows_as_set(session.execute(query))
elif not any (op in clause for op in ["<", ">"]):
in_clause = ",".join(map(str, results))
query += " AND variant_id IN (%s)" % in_clause
results = rows_as_set(session.execute(query))
else:
results = rows_as_set(session.execute(query)) & results
session.shutdown()
conn.send(results)
conn.close()
def _main(keyspace, sym):
cluster = Cluster()
session = cluster.connect(keyspace)
allData = {}
queryStr = "select * from %s.tick where symbol='%s' order by datetime limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['TickBeg'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.tick where symbol='%s' order by datetime desc limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['TickEnd'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.secbar where symbol='%s' order by datetime limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['SecBeg'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.secbar where symbol='%s' order by datetime desc limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['SecEnd'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.minbar where symbol='%s' order by datetime limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['MinBeg'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.minbar where symbol='%s' order by datetime desc limit 10;" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['MinEnd'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
queryStr = "select * from %s.symbol where symbol='%s';" % (keyspace, sym)
retData = _executeQurey(session, queryStr)
allData['SymbolInfo'] = retData['data'] if retData['Result'] == 'OK' else ['Exception!']
return allData
def test_tuples_with_nulls(self):
"""
Test tuples with null and empty string fields.
"""
if self._cass_version < (2, 1, 0):
raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1")
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect()
s.execute("""CREATE KEYSPACE test_tuples_with_nulls
WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""")
s.set_keyspace("test_tuples_with_nulls")
s.execute("CREATE TABLE mytable (k int PRIMARY KEY, t tuple<text, int, uuid, blob>)")
insert = s.prepare("INSERT INTO mytable (k, t) VALUES (0, ?)")
s.execute(insert, [(None, None, None, None)])
result = s.execute("SELECT * FROM mytable WHERE k=0")
self.assertEquals((None, None, None, None), result[0].t)
read = s.prepare("SELECT * FROM mytable WHERE k=0")
self.assertEquals((None, None, None, None), s.execute(read)[0].t)
# also test empty strings where compatible
s.execute(insert, [('', None, None, '')])
result = s.execute("SELECT * FROM mytable WHERE k=0")
self.assertEquals(('', None, None, ''), result[0].t)
self.assertEquals(('', None, None, ''), s.execute(read)[0].t)
c.shutdown()
def test_blob_type_as_bytearray(self):
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect()
s.execute("""
CREATE KEYSPACE typetests_blob2
WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}
""")
s.set_keyspace("typetests_blob2")
s.execute("""
CREATE TABLE mytable (
a ascii,
b blob,
PRIMARY KEY (a)
)
""")
params = [
'key1',
bytearray(b'blob1')
]
query = 'INSERT INTO mytable (a, b) VALUES (%s, %s);'
s.execute(query, params)
expected_vals = [
'key1',
bytearray(b'blob1')
]
results = s.execute("SELECT * FROM mytable")
for expected, actual in zip(expected_vals, results[0]):
self.assertEqual(expected, actual)
def test_timezone_aware_datetimes(self):
""" Ensure timezone-aware datetimes are converted to timestamps correctly """
try:
import pytz
except ImportError as exc:
raise unittest.SkipTest('pytz is not available: %r' % (exc,))
dt = datetime(1997, 8, 29, 11, 14)
eastern_tz = pytz.timezone('US/Eastern')
eastern_tz.localize(dt)
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect()
s.execute("""CREATE KEYSPACE tz_aware_test
WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""")
s.set_keyspace("tz_aware_test")
s.execute("CREATE TABLE mytable (a ascii PRIMARY KEY, b timestamp)")
# test non-prepared statement
s.execute("INSERT INTO mytable (a, b) VALUES ('key1', %s)", parameters=(dt,))
result = s.execute("SELECT b FROM mytable WHERE a='key1'")[0].b
self.assertEqual(dt.utctimetuple(), result.utctimetuple())
# test prepared statement
prepared = s.prepare("INSERT INTO mytable (a, b) VALUES ('key2', ?)")
s.execute(prepared, parameters=(dt,))
result = s.execute("SELECT b FROM mytable WHERE a='key2'")[0].b
self.assertEqual(dt.utctimetuple(), result.utctimetuple())
def test_for_schema_disagreement_attribute(self):
"""
Tests to ensure that schema disagreement is properly surfaced on the response future.
Creates and destroys keyspaces/tables with various schema agreement timeouts set.
First part runs cql create/drop cmds with schema agreement set in such away were it will be impossible for agreement to occur during timeout.
It then validates that the correct value is set on the result.
Second part ensures that when schema agreement occurs, that the result set reflects that appropriately
@since 3.1.0
@jira_ticket PYTHON-458
@expected_result is_schema_agreed is set appropriately on response thefuture
@test_category schema
"""
# This should yield a schema disagreement
cluster = Cluster(protocol_version=PROTOCOL_VERSION, max_schema_agreement_wait=0.001)
session = cluster.connect(wait_for_all_pools=True)
rs = session.execute("CREATE KEYSPACE test_schema_disagreement WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}")
self.check_and_wait_for_agreement(session, rs, False)
rs = session.execute("CREATE TABLE test_schema_disagreement.cf (key int PRIMARY KEY, value int)")
self.check_and_wait_for_agreement(session, rs, False)
rs = session.execute("DROP KEYSPACE test_schema_disagreement")
self.check_and_wait_for_agreement(session, rs, False)
# These should have schema agreement
cluster = Cluster(protocol_version=PROTOCOL_VERSION, max_schema_agreement_wait=100)
session = cluster.connect()
rs = session.execute("CREATE KEYSPACE test_schema_disagreement WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}")
self.check_and_wait_for_agreement(session, rs, True)
rs = session.execute("CREATE TABLE test_schema_disagreement.cf (key int PRIMARY KEY, value int)")
self.check_and_wait_for_agreement(session, rs, True)
rs = session.execute("DROP KEYSPACE test_schema_disagreement")
self.check_and_wait_for_agreement(session, rs, True)
def test_can_insert_tuples_all_primitive_datatypes(self):
"""
Ensure tuple subtypes are appropriately handled.
"""
if self.cass_version < (2, 1, 0):
raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1")
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect(self.keyspace_name)
s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple
s.execute("CREATE TABLE tuple_primitive ("
"k int PRIMARY KEY, "
"v frozen<tuple<%s>>)" % ','.join(PRIMITIVE_DATATYPES))
values = []
type_count = len(PRIMITIVE_DATATYPES)
for i, data_type in enumerate(PRIMITIVE_DATATYPES):
# create tuples to be written and ensure they match with the expected response
# responses have trailing None values for every element that has not been written
values.append(get_sample(data_type))
expected = tuple(values + [None] * (type_count - len(values)))
s.execute("INSERT INTO tuple_primitive (k, v) VALUES (%s, %s)", (i, tuple(values)))
result = s.execute("SELECT v FROM tuple_primitive WHERE k=%s", (i,))[0]
self.assertEqual(result.v, expected)
c.shutdown()
def hello_world():
cluster = Cluster(['172.17.0.2'])
session = cluster.connect()
session.execute("CREATE KEYSPACE IF NOT EXISTS results WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 3};")
session.execute("USE results;")
session.execute("create table IF NOT EXISTS result_table(translate_source text,translate_result text,analyze_result text,result_id text PRIMARY KEY);")
language_translation = LanguageTranslation(
username='******',
password='******')
f = open('translate_file.txt',"r")
line = f.readline()
translation = language_translation.translate(
text=line,
source='fr',
target='en')
translated_text = json.dumps(translation, indent=2, ensure_ascii=False)
tone_analyzer = ToneAnalyzerV3(
username='******',
password='******',
version='2016-05-19 ')
analyze = json.dumps(tone_analyzer.tone(text=translated_text), indent=2)
session.execute("INSERT INTO result_table (translate_source,translate_result,analyze_result,result_id) VALUES(%s,%s,%s,%s)",(line,translated_text,analyze,1))
session.execute("SELECT * FROM result_table;")
return(analyze)
def read_check(count):
from cassandra.cluster import Cluster
cluster = Cluster()
session = cluster.connect()
gen = 0
counter = 0
random.seed(0);
while counter <= count:
data, gen = mkdata(gen)
try:
query = "SELECT data FROM sha.test WHERE data='%s';" % (data)
rows = session.execute(query)
count = 0
for row in rows:
count +=1
if count != 1:
return False
except Exception, e:
print e
keep_going = False
continue
counter += 1
def main(ip):
cluster = Cluster([ip])
session = cluster.connect()
rows = session.execute("select keyspace_name from system_schema.columns")
if KEYSPACE in [row[0] for row in rows]:
log.info("dropping existing keyspace...")
session.execute("DROP KEYSPACE " + KEYSPACE)
log.info("creating keyspace...")
session.execute("""
CREATE KEYSPACE %s
WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': '1' }
""" % KEYSPACE)
log.info("setting keyspace...")
session.set_keyspace(KEYSPACE)
rows = session.execute("select table_name from system_schema.columns;")
if TABLE not in [row[0] for row in rows]:
log.info("creating table...")
session.execute("""
CREATE TABLE %s (
thekey text,
col1 text,
col2 text,
PRIMARY KEY (thekey, col1)
)
""" % TABLE)
def _test_downgrading_cl(self, keyspace, rf, accepted):
cluster = Cluster(
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
default_retry_policy=DowngradingConsistencyRetryPolicy(),
protocol_version=PROTOCOL_VERSION)
session = cluster.connect()
create_schema(session, keyspace, replication_factor=rf)
self._insert(session, keyspace, 1)
self._query(session, keyspace, 1)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 1)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
try:
force_stop(2)
wait_for_down(cluster, 2)
self._assert_writes_succeed(session, keyspace, accepted)
self._assert_reads_succeed(session, keyspace,
accepted - set([ConsistencyLevel.ANY]))
self._assert_writes_fail(session, keyspace,
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
self._assert_reads_fail(session, keyspace,
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
finally:
start(2)
wait_for_up(cluster, 2)
def writer():
"""thread writer function"""
from cassandra.cluster import Cluster
cluster = Cluster()
session = cluster.connect()
create_database(session);
global keep_going
global counter
global lock
counter = 0
gen = 0;
keep_going = True
random.seed(0);
lock.acquire()
while keep_going:
data, gen = mkdata(gen)
try:
query = "INSERT INTO sha.test (data) VALUES('%s');" % (data)
session.execute(query)
except Exception, e:
print e
keep_going = False
continue
counter += 1
# let a change to the other thread to acquire the lock
lock.release()
lock.acquire()
def test_tuple_primitive_subtypes(self):
"""
Ensure tuple subtypes are appropriately handled.
"""
if self._cass_version < (2, 1, 0):
raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1")
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect()
s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple
s.execute("""CREATE KEYSPACE test_tuple_primitive_subtypes
WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""")
s.set_keyspace("test_tuple_primitive_subtypes")
s.execute("CREATE TABLE mytable ("
"k int PRIMARY KEY, "
"v tuple<%s>)" % ','.join(DATA_TYPE_PRIMITIVES))
for i in range(len(DATA_TYPE_PRIMITIVES)):
# create tuples to be written and ensure they match with the expected response
# responses have trailing None values for every element that has not been written
created_tuple = [get_sample(DATA_TYPE_PRIMITIVES[j]) for j in range(i + 1)]
response_tuple = tuple(created_tuple + [None for j in range(len(DATA_TYPE_PRIMITIVES) - i - 1)])
written_tuple = tuple(created_tuple)
s.execute("INSERT INTO mytable (k, v) VALUES (%s, %s)", (i, written_tuple))
result = s.execute("SELECT v FROM mytable WHERE k=%s", (i,))[0]
self.assertEqual(response_tuple, result.v)
args = parser.parse_args()
def my_print(msg):
if args.verbose:
print msg
toolbar_width = 10
print ""
# setup toolbar
sys.stdout.write("CR10:[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['
cluster = Cluster(["n2"])
session = cluster.connect("tpcc")
i = 0
result = [True] * 100
districts = session.execute('SELECT d_id FROM district')
for district in districts:
d_id=district.d_id
customers = session.execute('select c_id,c_balance FROM customer where c_w_id=1 and c_d_id='+ str(d_id))
for customer in customers:
my_print ("")
c_id=customer.c_id
my_print ("c_id:"+str(c_id))
c_balance=customer.c_balance
my_print ("c_balanec:"+str(c_balance))
'''#############################################################################
Setup Cassandra, connect to a cluster and keyspace.
#############################################################################'''
from cassandra.cluster import Cluster
cluster = Cluster(['172.17.0.2'])
session = cluster.connect()
session.set_keyspace('beepboopbackend')
'''#############################################################################
Drop the articles table
#############################################################################'''
session.execute("""DROP TABLE IF EXISTS articles""")
print("articles table dropped", '\n')
'''#############################################################################
Drop the comments table
#############################################################################'''
session.execute("""DROP TABLE IF EXISTS comments""")
print("comments table dropped", '\n')
'''#############################################################################
Drop the tags table
#############################################################################'''
session.execute("""DROP TABLE IF EXISTS tags""")
print("tags table dropped", '\n')
'''#############################################################################
Drop the users table
#############################################################################'''
session.execute("""DROP TABLE IF EXISTS users""")
print("users table dropped", '\n')
session.execute("""DROP KEYSPACE beepboopbackend;""")
print("keyspace dropped", '\n')
from passlib.apps import custom_app_context as pwd_context
from app import app
auth_api = Flask(__name__)
# User blueprint function to call another file
auth_api.register_blueprint(app)
auth_api.secret_key = 'CLOUD_SECRET_KEY'
login_manager = LoginManager(auth_api)
cluster = Cluster(contact_points=['172.17.0.2'], port=9042)
session = cluster.connect()
template_page = """
<form action={} method='POST'>
<input type='text' name='email' id='email' placeholder='email'/>
<input type='password' name='password' id='password' placeholder='password'/>
<input type='submit' name='submit'/>
</form>
create_keyspace(session, keyspace)
# ustawienie używanego keyspace w sesji
session.set_keyspace(keyspace)
# użycie dict_factory pozwala na zwracanie słowników
# znanych z języka Python przy zapytaniach do bazy danych
session.row_factory = dict_factory
# tworzenie tabeli
create_table(session, keyspace, table_movies, True)
create_table(session, keyspace, table_profiles, False)
if __name__ == "__main__":
keyspace = "user_ratings"
table = "user_avg_rating"
# utworzenia połączenia z klastrem
cluster = Cluster(['127.0.0.1'], port=9042)
session = cluster.connect()
# utworzenie nowego keyspace
create_keyspace(session, keyspace)
# ustawienie używanego keyspace w sesji
session.set_keyspace(keyspace)
# użycie dict_factory pozwala na zwracanie słowników
# znanych z języka Python przy zapytaniach do bazy danych
session.row_factory = dict_factory
# tworzenie tabeli
col_type = {'user_id': 'int', 'avg_movie_rating': 'float'}
create_table(session, keyspace, table, False)
# umieszczanie danych w tabeli
push_data_table(session, keyspace, table, userId=1337, avgMovieRating=4.2)
# pobieranie zawartości tabeli i wyświetlanie danych
get_data_table(session, keyspace, table)
from cassandra.cluster import Cluster
import random
from itertools import count
from threading import Event
sentinel = object()
num_queries = 1000000
num_started = count()
num_finished = count()
finished_event = Event()
cluster = Cluster(['127.0.0.1'])
session = cluster.connect('test')
def random_str(size):
"""
Return random string (only [a-z])
"""
# size = random.randint(1, max_size)
text = ''
for i in range(0, size):
i_char = random.randint(1,26)
c = chr(96 + i_char)
text += c
return text
def random_date():
"""
Return random date (str, format YYYY-MM-DD)
"""
year = str(random.randint(2000, 2018))
def getGraph():
completeGraph = ANGCompleteGraph()
alertDict = dict()
# capture the alert data
# do this first so that nodes with alerts can be flagged to be displayed
# with multispeak nodes, in case node isn't already flagged as multispeak
try:
response = urllib2.urlopen(cfg.serviceUrl + '/alertcount')
html = response.read()
alerts = json.loads(html)
for alert in alerts:
if 'ipAddress' in alert:
alertDict[alert['ipAddress']] = list()
except Exception as e:
print "Unable to obtain alerts", e
auth_provider = PlainTextAuthProvider(
username=cfg.cassandraConfig["user"],
password=cfg.cassandraConfig["password"])
cluster = Cluster([cfg.cassandraConfig["host"]],
auth_provider=auth_provider)
session = cluster.connect(cfg.cassandraConfig["db"])
#first we find all of the multispeak nodes since thats a relatively easy task
multispeakNodes = set()
testNodes = set()
lastTimestamp = getTimestampFromDatabase()
nowMinusOneDay = datetime.datetime.now() - datetime.timedelta(1)
if lastTimestamp is None or lastTimestamp < nowMinusOneDay:
lastTimestamp = nowMinusOneDay
for currentRow in session.execute(
'select source_addr, dest_addr from packet where time_stamp > %s allow filtering',
(lastTimestamp, )):
multispeakNodes.add(currentRow[0])
multispeakNodes.add(currentRow[1])
lastRawPacketTimestamp = datetime.datetime(1, 1, 1)
for currentRow in session.execute(
'select source_addr, source_port, destination_addr, destination_port, protocol, time_stamp from generic_packet where time_stamp > %s allow filtering',
(lastTimestamp, )):
srcName = currentRow[0]
srcPort = currentRow[1]
dstName = currentRow[2]
dstPort = currentRow[3]
protocol = currentRow[4]
time_stamp = currentRow[5]
if time_stamp > lastRawPacketTimestamp:
lastRawPacketTimestamp = time_stamp
isMultiSpeakLink = False # true if at least one node is multispeak
srcNode = completeGraph.addNode(srcName, "blue", False)
dstNode = completeGraph.addNode(dstName, "orange", False)
if srcName in multispeakNodes:
isMultiSpeakLink = True
if srcNode.color != "red":
srcNode.color = "green"
srcNode.addProtocol("Multispeak")
# If only displaying multispeak, this node and any connected nodes should be displayed
srcNode.displayWithMultiSpeak = True
dstNode.displayWithMultiSpeak = True
if dstName in multispeakNodes:
isMultiSpeakLink = True
if dstNode.color != "red":
dstNode.color = "green"
dstNode.addProtocol("Multispeak")
# If only displaying multispeak, this node and any connected nodes should be displayed
srcNode.displayWithMultiSpeak = True
dstNode.displayWithMultiSpeak = True
if srcName in alertDict:
srcNode.color = "red"
srcNode.radius = 8
# If only displaying multispeak, display this node if it has alerts even if it isn't multispeak
isMultiSpeakLink = True
srcNode.displayWithMultiSpeak = True
dstNode.displayWithMultiSpeak = True
if dstName in alertDict and dstNode.color != "red":
dstNode.color = "red"
dstNode.radius = 8
# If only displaying multispeak, display this node if it has alerts even if it isn't multispeak
isMultiSpeakLink = True
srcNode.displayWithMultiSpeak = True
dstNode.displayWithMultiSpeak = True
completeGraph.addLink(srcName, srcPort, dstName, dstPort,
isMultiSpeakLink)
if (protocol == "tcp"):
dstNode.addTcpServerPort(dstPort)
srcNode.incrementTcpConnectionCount()
dstNode.incrementTcpConnectionCount()
else:
srcNode.addUdpPort(srcPort)
srcNode.incrementUdpConnectionCount()
dstNode.addUdpPort(dstPort)
dstNode.incrementUdpConnectionCount()
cluster.shutdown()
completeGraph.saveGraph()
if lastRawPacketTimestamp.year >= 1900:
saveTimestampToDatabase(
lastRawPacketTimestamp.strftime('%Y-%m-%d %H:%M:%S.%f'))
return dumps(ANGJSON(completeGraph, alertDict), sort_keys=True, indent=2)
import config
from cassandra.cluster import Cluster
# Connect to cassandra cluster
cluster = Cluster([config.cass_seedip])
session = cluster.connect()
# Create and set cassandra keyspace to work
session.execute("CREATE KEYSPACE" + config.cass_keyspace +
"WITH replication = {'class':" + config.cass_class +
", 'replication_factor':" + config.cass_rf + "};")
session.set_keyspace(config.cass_keyspace)
# Create tables to insert data
session.execute(
"CREATE TABLE user_song_log (timestamp int, user_id int, song_id int, primary key(timestamp, user_id));"
)
# Close the connection
#session.shutdown()
#cluster.shutdown()
#cassandra-prepare.py
#example 12.10
from cassandra.cluster import Cluster
from cassandra.query import PreparedStatement
clstr = Cluster()
session = clstr.connect('mykeyspace')
stmt = session.prepare(
"INSERT INTO customers (custID, name,GSTIN) VALUES (?,?,?)")
boundstmt = stmt.bind([11, 'HarishKumar', '12PQRDF22431ZN'])
session.execute(boundstmt)
class LightweightTransactionsTests(unittest.TestCase):
def setUp(self):
"""
Test is skipped if run with cql version < 2
"""
if PROTOCOL_VERSION < 2:
raise unittest.SkipTest(
"Protocol 2.0+ is required for Lightweight transactions, currently testing against %r"
% (PROTOCOL_VERSION, ))
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION)
self.session = self.cluster.connect()
ddl = '''
CREATE TABLE test3rf.lwt (
k int PRIMARY KEY,
v int )'''
self.session.execute(ddl)
def tearDown(self):
"""
Shutdown cluster
"""
self.session.execute("DROP TABLE test3rf.lwt")
self.cluster.shutdown()
def test_no_connection_refused_on_timeout(self):
"""
Test for PYTHON-91 "Connection closed after LWT timeout"
Verifies that connection to the cluster is not shut down when timeout occurs.
Number of iterations can be specified with LWT_ITERATIONS environment variable.
Default value is 1000
"""
insert_statement = self.session.prepare(
"INSERT INTO test3rf.lwt (k, v) VALUES (0, 0) IF NOT EXISTS")
delete_statement = self.session.prepare(
"DELETE FROM test3rf.lwt WHERE k = 0 IF EXISTS")
iterations = int(os.getenv("LWT_ITERATIONS", 1000))
print("Started test for %d iterations" % iterations)
# Prepare series of parallel statements
statements_and_params = []
for i in range(iterations):
statements_and_params.append((insert_statement, ()))
statements_and_params.append((delete_statement, ()))
received_timeout = False
results = execute_concurrent(self.session,
statements_and_params,
raise_on_first_error=False)
for (success, result) in results:
if success:
continue
# In this case result is an exception
if type(result).__name__ == "NoHostAvailable":
self.fail("PYTHON-91: Disconnected from Cassandra: %s" %
result.message)
break
if type(result).__name__ == "WriteTimeout":
print("Timeout: %s" % result.message)
received_timeout = True
continue
self.fail("Unexpected exception %s: %s" %
(type(result).__name__, result.message))
break
# Make sure test passed
self.assertTrue(received_timeout)
from index_elasticsearch_wikipedia import INDEX_NAME, normalize, \
load_visited, rd, index_wiki_algorithm_entry
from parseWikipedia import get_wiki_page, is_algorithm_page
from impl_languages_deduplication import get_standardized_lang
import json
from cassandra.cluster import Cluster
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
visitedwiki = load_visited()
indexedimpl = set(rd.hkeys('rosetta-mapping-success'))
cluster = Cluster(['127.0.0.1']) # localhost
session = cluster.connect() # default key space
session.set_keyspace('crosswikis')
FUZZY_THRESHOLD = 79
UPDATING = True
def safe_print(s):
try:
print s
except UnicodeEncodeError:
print s.encode('utf8')
except UnicodeDecodeError:
print s.decode('utf8')
from base64 import b64encode
from os import makedirs
from os.path import join, basename
from sys import argv
import json
import requests
import cassandra
import re
import pandas as pd
import datetime
from google.cloud import translate
import time
import os, ssl
import shutil
from cassandra.cluster import Cluster
cluster = Cluster(['192.168.0.131', '192.168.0.132'])
session = cluster.connect('electionanalysis')
session.execute("TRUNCATE TABLE electionanalysis.votelist_full_pdf_queue")
session.execute(
"TRUNCATE TABLE electionanalysis.votelist_individual_page_queue")
#session.execute("TRUNCATE TABLE electionanalysis.voter_data")
session.execute("TRUNCATE TABLE electionanalysis.img_file_reprocessing")
session.execute("TRUNCATE TABLE electionanalysis.googlevision_inbound_queue")
session.execute("TRUNCATE TABLE electionanalysis.img_file_reprocessing")
session.execute("TRUNCATE TABLE electionanalysis.election_process_audit")
def rem_folder(path):
for root, dirs, files in os.walk(path):
if files:
from cassandra.cluster import Cluster
from cassandra import ConsistencyLevel
from cassandra.query import SimpleStatement
cluster = Cluster()
connect = cluster.connect('keyspace1')
with open('drop_buts.cql', mode='r') as f:
text = f.read()
stmts = text.split(';')
for i in stmts:
stmt = i.strip()
if stmt != '':
print(f'Executing {stmt}')
query = SimpleStatement(stmt,
consistency_level=ConsistencyLevel.QUORUM)
connect.execute(query)
print(f'{stmt} - done')
with open('create_buts.cql', mode='r') as f:
text = f.read()
stmts = text.split(';')
for i in stmts:
stmt = i.strip()
if stmt != '':
print(f'Executing {stmt}')
query = SimpleStatement(stmt,
consistency_level=ConsistencyLevel.ALL)
connect.execute(query)
print(f'{stmt} - done')
with open('work_buts.cql', mode='r') as f:
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = os.getcwd()
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
#limit the upload files
#load the exist tensorflow graph
sess = tf.Session()
saver = tf.train.import_meta_graph("./checkpoint/model.ckpt.meta")
saver.restore(sess, './checkpoint/model.ckpt')
keep_prob = tf.get_default_graph().get_tensor_by_name('dropout/Placeholder:0')
x = tf.get_default_graph().get_tensor_by_name('Placeholder:0')
y_conv = tf.get_default_graph().get_tensor_by_name('fc2/add:0')
#connect to the cassandra data base
cluster = Cluster(['0.0.0.0'])
session = cluster.connect()
session.execute(
"create KEYSPACE if not exists mnist_database WITH replication = {'class':'SimpleStrategy', 'replication_factor': 2};"
)
session.execute("use mnist_database")
session.execute(
"create table if not exists images(id uuid, digits int, upload_image list<float>, upload_time timestamp, primary key(id));"
)
#judge the files is admited
def allowed_file(filename):
return '.' in filename and filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
class BatchStatementTests(unittest.TestCase):
def setUp(self):
if PROTOCOL_VERSION < 2:
raise unittest.SkipTest(
"Protocol 2.0+ is required for BATCH operations, currently testing against %r"
% (PROTOCOL_VERSION, ))
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION)
if PROTOCOL_VERSION < 3:
self.cluster.set_core_connections_per_host(HostDistance.LOCAL, 1)
self.session = self.cluster.connect()
self.session.execute("TRUNCATE test3rf.test")
def tearDown(self):
self.cluster.shutdown()
def confirm_results(self):
keys = set()
values = set()
results = self.session.execute("SELECT * FROM test3rf.test")
for result in results:
keys.add(result.k)
values.add(result.v)
self.assertEqual(set(range(10)), keys)
self.assertEqual(set(range(10)), values)
def test_string_statements(self):
batch = BatchStatement(BatchType.LOGGED)
for i in range(10):
batch.add("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)",
(i, i))
self.session.execute(batch)
self.session.execute_async(batch).result()
self.confirm_results()
def test_simple_statements(self):
batch = BatchStatement(BatchType.LOGGED)
for i in range(10):
batch.add(
SimpleStatement(
"INSERT INTO test3rf.test (k, v) VALUES (%s, %s)"), (i, i))
self.session.execute(batch)
self.session.execute_async(batch).result()
self.confirm_results()
def test_prepared_statements(self):
prepared = self.session.prepare(
"INSERT INTO test3rf.test (k, v) VALUES (?, ?)")
batch = BatchStatement(BatchType.LOGGED)
for i in range(10):
batch.add(prepared, (i, i))
self.session.execute(batch)
self.session.execute_async(batch).result()
self.confirm_results()
def test_bound_statements(self):
prepared = self.session.prepare(
"INSERT INTO test3rf.test (k, v) VALUES (?, ?)")
batch = BatchStatement(BatchType.LOGGED)
for i in range(10):
batch.add(prepared.bind((i, i)))
self.session.execute(batch)
self.session.execute_async(batch).result()
self.confirm_results()
def test_no_parameters(self):
batch = BatchStatement(BatchType.LOGGED)
batch.add("INSERT INTO test3rf.test (k, v) VALUES (0, 0)")
batch.add("INSERT INTO test3rf.test (k, v) VALUES (1, 1)", ())
batch.add(
SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (2, 2)"))
batch.add(
SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (3, 3)"),
())
prepared = self.session.prepare(
"INSERT INTO test3rf.test (k, v) VALUES (4, 4)")
batch.add(prepared)
batch.add(prepared, ())
batch.add(prepared.bind([]))
batch.add(prepared.bind([]), ())
batch.add("INSERT INTO test3rf.test (k, v) VALUES (5, 5)", ())
batch.add("INSERT INTO test3rf.test (k, v) VALUES (6, 6)", ())
batch.add("INSERT INTO test3rf.test (k, v) VALUES (7, 7)", ())
batch.add("INSERT INTO test3rf.test (k, v) VALUES (8, 8)", ())
batch.add("INSERT INTO test3rf.test (k, v) VALUES (9, 9)", ())
self.assertRaises(ValueError, batch.add, prepared.bind([]), (1))
self.assertRaises(ValueError, batch.add, prepared.bind([]), (1, 2))
self.assertRaises(ValueError, batch.add, prepared.bind([]), (1, 2, 3))
self.session.execute(batch)
self.confirm_results()
def benchmark(thread_class):
options, args = parse_options()
for conn_class in options.supported_reactors:
setup(options)
log.info("==== %s ====" % (conn_class.__name__, ))
kwargs = {
'metrics_enabled': options.enable_metrics,
'connection_class': conn_class
}
if options.protocol_version:
kwargs['protocol_version'] = options.protocol_version
cluster = Cluster(options.hosts, **kwargs)
session = cluster.connect(options.keyspace)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
# Generate the query
if options.read:
query = "SELECT * FROM {0} WHERE thekey = '{{key}}'".format(TABLE)
else:
query = "INSERT INTO {0} (thekey".format(TABLE)
for i in range(options.num_columns):
query += ", col{0}".format(i)
query += ") VALUES ('{key}'"
for i in range(options.num_columns):
query += ", {0}".format(COLUMN_VALUES[options.column_type])
query += ")"
values = None # we don't use that anymore. Keeping it in case we go back to prepared statements.
per_thread = options.num_ops // options.threads
threads = []
log.debug(
"Beginning {0}...".format('reads' if options.read else 'inserts'))
start = time.time()
try:
for i in range(options.threads):
thread = thread_class(i, session, query, values, per_thread,
cluster.protocol_version,
options.profile)
thread.daemon = True
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
while thread.is_alive():
thread.join(timeout=0.5)
end = time.time()
finally:
cluster.shutdown()
teardown(options)
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (options.num_ops / total))
if options.enable_metrics:
stats = scales.getStats()['cassandra']
log.info("Connection errors: %d", stats['connection_errors'])
log.info("Write timeouts: %d", stats['write_timeouts'])
log.info("Read timeouts: %d", stats['read_timeouts'])
log.info("Unavailables: %d", stats['unavailables'])
log.info("Other errors: %d", stats['other_errors'])
log.info("Retries: %d", stats['retries'])
request_timer = stats['request_timer']
log.info("Request latencies:")
log.info(" min: %0.4fs", request_timer['min'])
log.info(" max: %0.4fs", request_timer['max'])
log.info(" mean: %0.4fs", request_timer['mean'])
log.info(" stddev: %0.4fs", request_timer['stddev'])
log.info(" median: %0.4fs", request_timer['median'])
log.info(" 75th: %0.4fs", request_timer['75percentile'])
log.info(" 95th: %0.4fs", request_timer['95percentile'])
log.info(" 98th: %0.4fs", request_timer['98percentile'])
log.info(" 99th: %0.4fs", request_timer['99percentile'])
log.info(" 99.9th: %0.4fs", request_timer['999percentile'])
class Q3Entry(Model):
__keyspace__ = KEYSPACE
date_origin_dest1 = columns.Ascii(partition_key=True)
date1 = columns.Ascii()
origin = columns.Ascii()
dest1 = columns.Ascii()
dest2 = columns.Ascii(primary_key=True)
total_delay = columns.Decimal()
datetime1 = columns.Ascii()
flight1 = columns.Ascii()
datetime2 = columns.Ascii()
flight2 = columns.Ascii()
if __name__ == '__main__':
cluster = Cluster(['node7'])
connection.register_connection('con',
session=cluster.connect(),
default=True)
create_keyspace_simple(name=KEYSPACE,
replication_factor=1,
connections=['con'])
for model in [
Q11Entry, Q12Entry, Q13Entry, Q21Entry, Q22Entry, Q23Entry,
Q24Entry, Q3Entry
]:
sync_table(model, connections=['con'])
print("All tables created !")
# you should make sure you have spark in your python path as below
# export PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/build:$PYTHONPATH
# but if you don't it will append it automatically for this session
import platform, os, sys
from os.path import dirname
CASSANDRA_IP = os.getenv('CASSANDRA1')
print(CASSANDRA_IP)
if CASSANDRA_IP is None:
CASSANDRA_IP = '172.18.0.2'
from cassandra.cluster import Cluster
cluster = Cluster([CASSANDRA_IP])
if not 'SPARK_HOME' in os.environ and not os.environ['SPARK_HOME'] in sys.path:
sys.path.append(os.environ['SPARK_HOME'] + '/python')
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession, SQLContext
from pyspark.sql.types import *
def initspark(appname="Test",
servername="local",
cassandra="cassandra",
mongo="mongo"):
conf = SparkConf().set("spark.cassandra.connection.host",
cassandra).setAppName(appname).setMaster(servername)
sc = SparkContext(conf=conf)
from flask import Flask, flash, request, redirect, url_for, render_template, jsonify
from cassandra.cluster import Cluster
from cassandra.query import dict_factory
import os
cassandra_endpoint = os.environ['CASSANDRA_ENDPOINT']
cluster = Cluster([cassandra_endpoint])
session = cluster.connect()
session.row_factory = dict_factory
keyspace = os.environ['CASSANDRA_KEYSPACE']
session.set_keyspace(keyspace)
app = Flask(__name__, static_url_path='', static_folder='static')
@app.route('/', methods=['GET'])
def main_page():
return render_template("index.html")
@app.route('/getimages', methods=['GET'])
def get_images():
rows = session.execute("SELECT * FROM images")
data = []
for row in rows:
obj = {}
for key, value in row.items():
obj[key] = value
data.append(obj)
return jsonify(data)
import cassandra
# Create connection to database
from cassandra.cluster import Cluster
try:
cluster = Cluster(['127.0.0.1']) #If you have a locally installed Apache Cassandra instance
session = cluster.connect()
except Exception as e:
print(e)
# Create a keyspace to do the work in
try:
session.execute("""
CREATE KEYSPACE IF NOT EXISTS udacity
WITH REPLICATION =
{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"""
)
except Exception as e:
print(e)
# Connect to keyspace
try:
session.set_keyspace('udacity')
except Exception as e:
print(e)
# 2 questions to be asked of our data
# 1. Give every album in the music library that was released in a given year
'SELECT * FROM music_library WHERE YEAR=1970'
from cassandra.cluster import Cluster cluster = Cluster(['172.17.0.2'],port=9042) session = cluster.connect() def create_tables(): sql = CREATE TABLE IF NOT EXISTS tracks ( title text, album text, artist text, duration text, url text, PRIMARY KEY(album,title) );
def get_kind_averages(keys):
""" Get an average size for each kind.
Args:
keys: A list of dictionaries containing keys.
Returns:
A dictionary listing the average size of each kind.
"""
hosts = appscale_info.get_db_ips()
retry_policy = IdempotentRetryPolicy()
cluster = Cluster(hosts, default_retry_policy=retry_policy)
session = cluster.connect(KEYSPACE)
entities_by_kind = {}
for key_dict in keys:
key = key_dict['key']
if is_entity(key):
key_parts = key.split(KEY_DELIMITER)
kind = key_parts[2].split(':')[0]
kind_id = KEY_DELIMITER.join([key_parts[0], key_parts[1], kind])
if kind_id not in entities_by_kind:
entities_by_kind[kind_id] = {'keys': [], 'size': 0, 'fetched': 0}
entities_by_kind[kind_id]['keys'].append(key)
for kind_id, kind in entities_by_kind.iteritems():
shuffle(kind['keys'])
if not entities_by_kind:
return {}
futures = []
for _ in range(50):
kind = choice(entities_by_kind.keys())
try:
key = entities_by_kind[kind]['keys'].pop()
except IndexError:
continue
select = """
SELECT {value} FROM "{table}"
WHERE {key}=%(key)s AND {column}=%(column)s
""".format(value=ThriftColumn.VALUE, table=APP_ENTITY_TABLE,
key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME)
parameters = {'key': bytearray(key), 'column': APP_ENTITY_SCHEMA[0]}
future = session.execute_async(select, parameters)
futures.append({'future': future, 'kind': kind})
for future_dict in futures:
future = future_dict['future']
kind = future_dict['kind']
try:
entity = future.result()[0].value
except IndexError:
continue
entities_by_kind[kind]['size'] += len(entity)
entities_by_kind[kind]['fetched'] += 1
kind_averages = {}
for kind_id, kind in entities_by_kind.iteritems():
try:
kind_averages[kind_id] = int(kind['size'] / kind['fetched'])
except ZeroDivisionError:
kind_averages[kind_id] = 0
return kind_averages
def __init__(self, config):
self.cluster = Cluster(
contact_points=config.cassandra_endpoints.split(','),
port=9042,
)
self.session = None
#Build connection string
azure_db_domain = ".cassandra.cosmosdb.azure.com"
azure_db_endpoint_uri = db_account_name + azure_db_domain
#Cassandra connection options for the Azure CosmoDB with Cassandra API from the quickstart documentation on the portal page
#grabbed my CA.pem from Mozilla https://curl.haxx.se/docs/caextract.html
ssl_opts = {
'ca_certs': './cacert.pem',
'ssl_version': PROTOCOL_TLSv1_2,
'cert_reqs': CERT_REQUIRED # Certificates are required and validated
}
auth_provider = PlainTextAuthProvider(username=db_account_name,
password=db_account_key)
cluster = Cluster([azure_db_endpoint_uri],
port=10350,
auth_provider=auth_provider,
ssl_options=ssl_opts)
logging.debug(
"Attempting to connect to CosmosDB with the following credentials {0}, {1}"
.format(azure_db_endpoint_uri, db_account_name, cosmos_keyspace))
session = cluster.connect(cosmos_keyspace)
def imageHashAndCache(image_bytes):
'''
Methods to
'''
image_hash = hashlib.md5(image_bytes).hexdigest()
image_hash_path = os.path.join("./static/img/", image_hash + ".jpg")
image_url = url_for('static', filename="img/" + image_hash + ".jpg")
#!/usr/bin/python
import cgi, cgitb
from cassandra.cluster import Cluster
cluster = Cluster()
session = cluster.connect('smg_space')
result = session.execute(" select * from category ")
print "Content-type:text/html\r\n\r\n"
print "<!DOCTYPE html>"
print "<html>"
print '<head>'
#print '<style>'
print '<link rel="stylesheet" type="text/css" href="style1.css">'
print '<title>Demo++</title>'
#print "</style>"
print "</head>"
print "<body>"
print '<div class="container">'
print '<header>'
print '<h1>Dashboard</h1>'
print '</header>'
print '<nav>'
print '<ul>'
print '<li color:red><a href="category1.py">Menu</a></li>'
print '<li><a href="upload_videos5.py">Add Category</a></li>'
print '<li><a href="add_category2.py">Upload Videos</a></li>'
print '<li><a href="index.py">Log-Out </a></li>'
def test_can_insert_collection_datatypes(self):
"""
Test insertion of all collection types
"""
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect(self.keyspace_name)
# use tuple encoding, to convert native python tuple into raw CQL
s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple
# create table
alpha_type_list = ["zz int PRIMARY KEY"]
col_names = ["zz"]
start_index = ord('a')
for i, collection_type in enumerate(COLLECTION_TYPES):
for j, datatype in enumerate(PRIMITIVE_DATATYPES_KEYS):
if collection_type == "map":
type_string = "{0}_{1} {2}<{3}, {3}>".format(
chr(start_index + i), chr(start_index + j),
collection_type, datatype)
elif collection_type == "tuple":
type_string = "{0}_{1} frozen<{2}<{3}>>".format(
chr(start_index + i), chr(start_index + j),
collection_type, datatype)
else:
type_string = "{0}_{1} {2}<{3}>".format(
chr(start_index + i), chr(start_index + j),
collection_type, datatype)
alpha_type_list.append(type_string)
col_names.append("{0}_{1}".format(chr(start_index + i),
chr(start_index + j)))
s.execute("CREATE TABLE allcoltypes ({0})".format(
', '.join(alpha_type_list)))
columns_string = ', '.join(col_names)
# create the input for simple statement
params = [0]
for collection_type in COLLECTION_TYPES:
for datatype in PRIMITIVE_DATATYPES_KEYS:
params.append((get_collection_sample(collection_type,
datatype)))
# insert into table as a simple statement
placeholders = ', '.join(["%s"] * len(col_names))
s.execute(
"INSERT INTO allcoltypes ({0}) VALUES ({1})".format(
columns_string, placeholders), params)
# verify data
results = s.execute(
"SELECT {0} FROM allcoltypes WHERE zz=0".format(columns_string))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# create the input for prepared statement
params = [0]
for collection_type in COLLECTION_TYPES:
for datatype in PRIMITIVE_DATATYPES_KEYS:
params.append((get_collection_sample(collection_type,
datatype)))
# try the same thing with a prepared statement
placeholders = ','.join(["?"] * len(col_names))
insert = s.prepare("INSERT INTO allcoltypes ({0}) VALUES ({1})".format(
columns_string, placeholders))
s.execute(insert.bind(params))
# verify data
results = s.execute(
"SELECT {0} FROM allcoltypes WHERE zz=0".format(columns_string))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# verify data with prepared statement query
select = s.prepare(
"SELECT {0} FROM allcoltypes WHERE zz=?".format(columns_string))
results = s.execute(select.bind([0]))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# verify data with with prepared statement, use dictionary with no explicit columns
s.row_factory = ordered_dict_factory
select = s.prepare("SELECT * FROM allcoltypes")
results = s.execute(select)[0]
for expected, actual in zip(params, results.values()):
self.assertEqual(actual, expected)
c.shutdown()
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
cloud_config = {'secure_connect_bundle': './secure-connect-ineuron.zip'}
auth_provider = PlainTextAuthProvider(
'MwvGmEOlZFejSOvRDcUtJksv',
'DRhIUyflGsl4KYE+bPxWTsMEcC6sOEUjJmE.suKzN,.au2rK.UPzPzoSyMtkQ4+LZ4HijD6Y-ouH5azI.X6ikEWa,4s5GFqZmOInJQKCtsxcywOLKBrwr58XQXQEkTXm'
)
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
session = cluster.connect()
for i in range(10):
session.execute(
"insert into test.emp (emp_id,emp_name,emp_city,emp_sal,emp_phone) values (00"
+ str(i) + ",'mano','singapore'," + str(i) + "0000,65732569);").one()
print("Executed", i)
def run_inserts_at_version(self, proto_ver):
session = Cluster(protocol_version=proto_ver).connect(
self.keyspace_name)
try:
p = session.prepare('insert into t (k, v) values (?, ?)')
session.execute(p, (0, [{1, 2}, {3, 5}]))
p = session.prepare('insert into u (k, v) values (?, ?)')
session.execute(p, (0, {(1, 2), (3, 5)}))
p = session.prepare('insert into v (k, v, v1) values (?, ?, ?)')
session.execute(p, (0, {
(1, 2): [1, 2, 3],
(3, 5): [4, 5, 6]
}, (123, 'four')))
p = session.prepare('insert into w (k, v) values (?, ?)')
session.execute(p, (0, ({1: [1, 2, 3], 2: [4, 5, 6]}, [7, 8, 9])))
finally:
session.cluster.shutdown()
def test_can_insert_tuples_all_collection_datatypes(self):
"""
Ensure tuple subtypes are appropriately handled for maps, sets, and lists.
"""
if self.cass_version < (2, 1, 0):
raise unittest.SkipTest(
"The tuple type was introduced in Cassandra 2.1")
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect(self.keyspace_name)
# set the row_factory to dict_factory for programmatic access
# set the encoder for tuples for the ability to write tuples
s.row_factory = dict_factory
s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple
values = []
# create list values
for datatype in PRIMITIVE_DATATYPES_KEYS:
values.append('v_{0} frozen<tuple<list<{1}>>>'.format(
len(values), datatype))
# create set values
for datatype in PRIMITIVE_DATATYPES_KEYS:
values.append('v_{0} frozen<tuple<set<{1}>>>'.format(
len(values), datatype))
# create map values
for datatype in PRIMITIVE_DATATYPES_KEYS:
datatype_1 = datatype_2 = datatype
if datatype == 'blob':
# unhashable type: 'bytearray'
datatype_1 = 'ascii'
values.append('v_{0} frozen<tuple<map<{1}, {2}>>>'.format(
len(values), datatype_1, datatype_2))
# make sure we're testing all non primitive data types in the future
if set(COLLECTION_TYPES) != set(['tuple', 'list', 'map', 'set']):
raise NotImplemented('Missing datatype not implemented: {}'.format(
set(COLLECTION_TYPES) - set(['tuple', 'list', 'map', 'set'])))
# create table
s.execute("CREATE TABLE tuple_non_primative ("
"k int PRIMARY KEY, "
"%s)" % ', '.join(values))
i = 0
# test tuple<list<datatype>>
for datatype in PRIMITIVE_DATATYPES_KEYS:
created_tuple = tuple([[get_sample(datatype)]])
s.execute(
"INSERT INTO tuple_non_primative (k, v_%s) VALUES (0, %s)",
(i, created_tuple))
result = s.execute(
"SELECT v_%s FROM tuple_non_primative WHERE k=0", (i, ))[0]
self.assertEqual(created_tuple, result['v_%s' % i])
i += 1
# test tuple<set<datatype>>
for datatype in PRIMITIVE_DATATYPES_KEYS:
created_tuple = tuple([sortedset([get_sample(datatype)])])
s.execute(
"INSERT INTO tuple_non_primative (k, v_%s) VALUES (0, %s)",
(i, created_tuple))
result = s.execute(
"SELECT v_%s FROM tuple_non_primative WHERE k=0", (i, ))[0]
self.assertEqual(created_tuple, result['v_%s' % i])
i += 1
# test tuple<map<datatype, datatype>>
for datatype in PRIMITIVE_DATATYPES_KEYS:
if datatype == 'blob':
# unhashable type: 'bytearray'
created_tuple = tuple([{
get_sample('ascii'):
get_sample(datatype)
}])
else:
created_tuple = tuple([{
get_sample(datatype):
get_sample(datatype)
}])
s.execute(
"INSERT INTO tuple_non_primative (k, v_%s) VALUES (0, %s)",
(i, created_tuple))
result = s.execute(
"SELECT v_%s FROM tuple_non_primative WHERE k=0", (i, ))[0]
self.assertEqual(created_tuple, result['v_%s' % i])
i += 1
c.shutdown()
from flask import jsonify
from flask import redirect
from flask import render_template
from app import app
from cassandra.cluster import Cluster
import config
import id_info
CASSANDRA_DNS = config.STORAGE_CONFIG['PUBLIC_DNS']
KEYSPACE = 'cryptocoins'
cluster = Cluster([CASSANDRA_DNS])
session = cluster.connect()
session.set_keyspace(KEYSPACE)
ID_DICT = id_info.ID_DICT
INV_ID_DICT = id_info.INV_ID_DICT
ID_LIST = id_info.ID_LIST
@app.route('/')
@app.route('/index')
def index():
""" Render index webpage """
return render_template('index.html')
def test_can_insert_primitive_datatypes(self):
"""
Test insertion of all datatype primitives
"""
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect(self.keyspace_name)
# create table
alpha_type_list = ["zz int PRIMARY KEY"]
col_names = ["zz"]
start_index = ord('a')
for i, datatype in enumerate(PRIMITIVE_DATATYPES):
alpha_type_list.append("{0} {1}".format(chr(start_index + i),
datatype))
col_names.append(chr(start_index + i))
s.execute("CREATE TABLE alltypes ({0})".format(
', '.join(alpha_type_list)))
# create the input
params = [0]
for datatype in PRIMITIVE_DATATYPES:
params.append((get_sample(datatype)))
# insert into table as a simple statement
columns_string = ', '.join(col_names)
placeholders = ', '.join(["%s"] * len(col_names))
s.execute(
"INSERT INTO alltypes ({0}) VALUES ({1})".format(
columns_string, placeholders), params)
# verify data
results = s.execute(
"SELECT {0} FROM alltypes WHERE zz=0".format(columns_string))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# try the same thing sending one insert at the time
s.execute("TRUNCATE alltypes;")
for i, datatype in enumerate(PRIMITIVE_DATATYPES):
single_col_name = chr(start_index + i)
single_col_names = ["zz", single_col_name]
placeholders = ','.join(["%s"] * len(single_col_names))
single_columns_string = ', '.join(single_col_names)
for j, data_sample in enumerate(get_all_samples(datatype)):
key = i + 1000 * j
single_params = (key, data_sample)
s.execute(
"INSERT INTO alltypes ({0}) VALUES ({1})".format(
single_columns_string, placeholders), single_params)
# verify data
result = s.execute(
"SELECT {0} FROM alltypes WHERE zz=%s".format(
single_columns_string), (key, ))[0][1]
compare_value = data_sample
if six.PY3:
import ipaddress
if isinstance(data_sample,
ipaddress.IPv4Address) or isinstance(
data_sample, ipaddress.IPv6Address):
compare_value = str(data_sample)
self.assertEqual(result, compare_value)
# try the same thing with a prepared statement
placeholders = ','.join(["?"] * len(col_names))
s.execute("TRUNCATE alltypes;")
insert = s.prepare("INSERT INTO alltypes ({0}) VALUES ({1})".format(
columns_string, placeholders))
s.execute(insert.bind(params))
# verify data
results = s.execute(
"SELECT {0} FROM alltypes WHERE zz=0".format(columns_string))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# verify data with prepared statement query
select = s.prepare(
"SELECT {0} FROM alltypes WHERE zz=?".format(columns_string))
results = s.execute(select.bind([0]))[0]
for expected, actual in zip(params, results):
self.assertEqual(actual, expected)
# verify data with with prepared statement, use dictionary with no explicit columns
s.row_factory = ordered_dict_factory
select = s.prepare("SELECT * FROM alltypes")
results = s.execute(select)[0]
for expected, actual in zip(params, results.values()):
self.assertEqual(actual, expected)
c.shutdown()